diff --git a/.azure/pipelines/azure-pipelines-external-release.yml b/.azure/pipelines/azure-pipelines-external-release.yml index c69347a3bcb..d172ac2f5f7 100644 --- a/.azure/pipelines/azure-pipelines-external-release.yml +++ b/.azure/pipelines/azure-pipelines-external-release.yml @@ -11,6 +11,7 @@ trigger: paths: include: - Version.props + resources: repositories: - repository: self @@ -21,9 +22,28 @@ pool: vmImage: 'windows-latest' jobs: +- job: BuildNativeLinux + displayName: Build bftree native (Linux) + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + clean: False + - bash: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + cargo build --release --locked --manifest-path libs/native/bftree-garnet/Cargo.toml + displayName: 'Build bftree for Linux' + - task: PublishPipelineArtifact@1 + displayName: 'Upload libbftree_garnet.so' + inputs: + targetPath: libs/native/bftree-garnet/target/release/libbftree_garnet.so + artifactName: bftree-linux-x64 + - job: Phase_1 displayName: Assessment - condition: succeeded() # Ensures job execution stops if any step fails - need this step due to "condition" in GitHubRelease task + dependsOn: BuildNativeLinux + condition: succeeded() cancelTimeoutInMinutes: 1 pool: name: Azure Pipelines @@ -34,12 +54,30 @@ jobs: submodules: recursive persistCredentials: True + - task: DownloadPipelineArtifact@2 + displayName: 'Download freshly-built Linux bftree binary' + inputs: + artifactName: bftree-linux-x64 + targetPath: libs/native/bftree-garnet/runtimes/linux-x64/native + - task: PowerShell@2 displayName: 'Extract version number from Version.props' inputs: filePath: .azure/pipelines/extract_version.ps1 workingDirectory: .azure/pipelines + - powershell: | + $isPreRelease = "$(Build.BuildNumber)".Contains('-') + if ($isPreRelease) { + $releaseTitle = "Garnet PREVIEW v$(Build.BuildNumber)" + } else { + $releaseTitle = "Garnet v$(Build.BuildNumber)" + } + Write-Host "Build.BuildNumber=$(Build.BuildNumber); isPreRelease=$isPreRelease; releaseTitle=$releaseTitle" + Write-Host "##vso[task.setvariable variable=isPreRelease]$($isPreRelease.ToString().ToLower())" + Write-Host "##vso[task.setvariable variable=releaseTitle]$releaseTitle" + displayName: 'Derive isPreRelease and releaseTitle from version (PREVIEW prefix iff SemVer prerelease)' + - task: UseDotNet@2 displayName: Use .NET 6 SDK - needed for code signing inputs: @@ -49,6 +87,15 @@ jobs: inputs: version: 10.0.x + - task: PowerShell@2 + displayName: Install Rust toolchain + inputs: + targetType: inline + script: | + Invoke-WebRequest -Uri https://win.rustup.rs -OutFile rustup-init.exe + .\rustup-init.exe -y --default-toolchain stable + echo "##vso[task.prependpath]$env:USERPROFILE\.cargo\bin" + - task: DotNetCoreCLI@2 displayName: dotnet build inputs: @@ -74,7 +121,7 @@ jobs: AuthCertName: 'garnet-codesign-auth-cert' ServiceEndpointUrl: 'https://api.esrp.microsoft.com/api/v2' FolderPath: . - Pattern: Garnet*.dll,Tsavorite*.dll,Garnet*.exe,HdrHistogram.dll,native_device.dll + Pattern: Garnet*.dll,Tsavorite*.dll,Garnet*.exe,HdrHistogram.dll,native_device.dll,bftree_garnet.dll,*Lua.dll signConfigType: inlineSignParams inlineOperation: >- [ @@ -174,13 +221,13 @@ jobs: - task: GitHubRelease@1 displayName: 'Create the GitHub release' - condition: eq(variables['Build.SourceBranchName'], 'main') + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') inputs: action: 'create' gitHubConnection: ADO_to_Github_ServiceConnection tagSource: userSpecifiedTag tag: 'v$(Build.BuildNumber)' - title: 'Garnet v$(Build.BuildNumber)' + title: '$(releaseTitle)' releaseNotesSource: inline releaseNotesInline: | Get NuGet binaries at: @@ -197,10 +244,11 @@ jobs: $(Build.ArtifactStagingDirectory)/*.zip $(Build.ArtifactStagingDirectory)/*.tar.xz $(Build.ArtifactStagingDirectory)/*.7z + isPreRelease: $(isPreRelease) - task: NuGetCommand@2 displayName: 'Push both packages to NuGet.org' - condition: eq(variables['Build.SourceBranchName'], 'main') + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') inputs: command: push packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg' diff --git a/.azure/pipelines/azure-pipelines-internal-release.yml b/.azure/pipelines/azure-pipelines-internal-release.yml index 3ab15aa5c7b..852792d2c89 100644 --- a/.azure/pipelines/azure-pipelines-internal-release.yml +++ b/.azure/pipelines/azure-pipelines-internal-release.yml @@ -11,8 +11,27 @@ resources: type: git ref: refs/heads/main jobs: +- job: BuildNativeLinux + displayName: Build bftree native (Linux) + pool: + vmImage: 'ubuntu-latest' + steps: + - checkout: self + clean: False + - bash: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + cargo build --release --locked --manifest-path libs/native/bftree-garnet/Cargo.toml + displayName: 'Build bftree for Linux' + - task: PublishPipelineArtifact@1 + displayName: 'Upload libbftree_garnet.so' + inputs: + targetPath: libs/native/bftree-garnet/target/release/libbftree_garnet.so + artifactName: bftree-linux-x64 + - job: Phase_1 displayName: Assessment + dependsOn: BuildNativeLinux cancelTimeoutInMinutes: 1 pool: name: Azure Pipelines @@ -21,6 +40,11 @@ jobs: clean: False submodules: recursive persistCredentials: True + - task: DownloadPipelineArtifact@2 + displayName: 'Download freshly-built Linux bftree binary' + inputs: + artifactName: bftree-linux-x64 + targetPath: libs/native/bftree-garnet/runtimes/linux-x64/native - task: UseDotNet@2 displayName: Use .NET 10 SDK inputs: @@ -34,6 +58,14 @@ jobs: displayName: NuGet Authenticate - task: NuGetAuthenticate@1 displayName: 'NuGet Authenticate' + - task: PowerShell@2 + displayName: Install Rust toolchain + inputs: + targetType: inline + script: | + Invoke-WebRequest -Uri https://win.rustup.rs -OutFile rustup-init.exe + .\rustup-init.exe -y --default-toolchain stable + echo "##vso[task.prependpath]$env:USERPROFILE\.cargo\bin" - task: DotNetCoreCLI@2 displayName: dotnet build inputs: @@ -45,7 +77,7 @@ jobs: inputs: ConnectedServiceName: Garnet Code Signing FolderPath: . - Pattern: Garnet.server.dll,Garnet.client.dll,Garnet.common.dll,Garnet.cluster.dll,Garnet.host.dll,HdrHistogram.dll,Tsavorite.core.dll,Tsavorite.devices.AzureStorageDevice.dll,native_device.dll + Pattern: Garnet.server.dll,Garnet.client.dll,Garnet.common.dll,Garnet.cluster.dll,Garnet.host.dll,HdrHistogram.dll,Tsavorite.core.dll,Tsavorite.devices.AzureStorageDevice.dll,native_device.dll,bftree_garnet.dll,BfTreeInterop.dll signConfigType: inlineSignParams inlineOperation: >- [ diff --git a/.azure/pipelines/azure-pipelines.yml b/.azure/pipelines/azure-pipelines.yml index e58df029b0c..a6f4219d23a 100644 --- a/.azure/pipelines/azure-pipelines.yml +++ b/.azure/pipelines/azure-pipelines.yml @@ -42,6 +42,15 @@ jobs: packageType: 'sdk' version: '10.0.x' + - task: PowerShell@2 + displayName: Install Rust toolchain + inputs: + targetType: inline + script: | + Invoke-WebRequest -Uri https://win.rustup.rs -OutFile rustup-init.exe + .\rustup-init.exe -y --default-toolchain stable + echo "##vso[task.prependpath]$env:USERPROFILE\.cargo\bin" + - task: NodeTool@0 displayName: Node Tool inputs: @@ -164,6 +173,11 @@ jobs: packageType: 'sdk' version: '10.0.x' + - bash: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + echo "##vso[task.prependpath]$HOME/.cargo/bin" + displayName: 'Install Rust toolchain' + - bash: | sudo npm install -g azurite sudo mkdir azurite diff --git a/.azure/pipelines/createbinaries.ps1 b/.azure/pipelines/createbinaries.ps1 index 67138664087..aec3f08bca5 100644 --- a/.azure/pipelines/createbinaries.ps1 +++ b/.azure/pipelines/createbinaries.ps1 @@ -40,14 +40,17 @@ function CleanUpFiles { $publishPath = "$basePath/main/GarnetServer/bin/Release/$framework/publish/$publishFolder" $excludeGarnetServerPDB = 'GarnetServer.pdb' - # Native binary is different based on OS by default + # Native binaries are different based on OS by default $nativeFile = "libnative_device.so" + $bftreeFile = "libbftree_garnet.so" if ($platform -match "win-x64") { $nativeFile = "native_device.dll" + $bftreeFile = "bftree_garnet.dll" } $nativeRuntimePathFile = "$publishPath/runtimes/$platform/native/$nativeFile" + $bftreeRuntimePathFile = "$publishPath/runtimes/$platform/native/$bftreeFile" if (Test-Path -Path $publishPath) { Get-ChildItem -Path $publishPath -Filter '*.pfx' | Remove-Item -Force @@ -56,6 +59,9 @@ function CleanUpFiles { # Copy proper native run time to publish directory Copy-Item -Path $nativeRuntimePathFile -Destination $publishPath + if (Test-Path -Path $bftreeRuntimePathFile) { + Copy-Item -Path $bftreeRuntimePathFile -Destination $publishPath + } } else { Write-Host "Publish Path not found: $publishPath" } diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000000..802815e3b82 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,300 @@ +# Garnet - Copilot Instructions + +Garnet is a high-performance remote cache-store from Microsoft Research implementing the Redis RESP wire protocol in C#/.NET. It uses Tsavorite as its storage engine. Full developer docs: https://microsoft.github.io/garnet/docs/dev/onboarding + +> **Note**: Some website docs may reference the older two-store architecture (separate main store and object store). This branch uses a **unified single-store** design — see the Architecture section below for the current model. + +## Build, Test, and Lint + +```bash +# Build the entire solution +dotnet build + +# Run all Garnet tests +dotnet test test/Garnet.test -f net10.0 -c Debug -l "console;verbosity=detailed" + +# Run all cluster tests +dotnet test test/Garnet.test.cluster -f net10.0 -c Debug -l "console;verbosity=detailed" + +# Run a single test by fully qualified name +dotnet test test/Garnet.test -f net10.0 -c Debug --filter "FullyQualifiedName~RespTests.PingTest" + +# Run all tests in a single test class +dotnet test test/Garnet.test -f net10.0 -c Debug --filter "FullyQualifiedName~RespTests" + +# Build and test Tsavorite independently (has its own solution) +dotnet build libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj +dotnet test libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj -f net10.0 -c Debug -l "console;verbosity=detailed" + +# Check formatting (CI enforces this) +dotnet format Garnet.slnx --verify-no-changes +dotnet format libs/storage/Tsavorite/cs/Tsavorite.slnx --verify-no-changes + +# Run the server locally (from repo root) +cd main/GarnetServer && dotnet run -c Debug -f net10.0 -- --logger-level Trace -m 4g -i 64m +``` + +Target frameworks are `net8.0` and `net10.0`. CI runs tests on both, in Debug and Release, on Ubuntu and Windows. + +## Architecture + +### Unified Single-Store Design + +Garnet uses a **single Tsavorite key-value store** instance (`TsavoriteKV`) that holds both raw strings and complex objects. The store is accessed through three different **context types**, each with its own input/output types and session functions: + +| Context | Input/Output Types | Session Functions | Used For | +|---------|-------------------|-------------------|----------| +| **String context** | `StringInput` / `StringOutput` | `MainSessionFunctions` | Raw string commands (GET, SET, APPEND, INCR, etc.) | +| **Object context** | `ObjectInput` / `ObjectOutput` | `ObjectSessionFunctions` | Collection commands (HSET, LPUSH, ZADD, SADD, etc.) | +| **Unified context** | `UnifiedInput` / `UnifiedOutput` | `UnifiedSessionFunctions` | Type-agnostic commands (EXISTS, DELETE, TYPE, TTL, EXPIRE, RENAME, etc.) | + +All three contexts operate on the **same underlying store**. At the storage level, each record's `RecordInfo` has a `ValueIsObject` bit that indicates whether the value is a raw string (inline bytes) or a heap object reference, enabling the unified store to differentiate between the two value types. The `GarnetApi` struct is generic over all three context types: + +```csharp +public partial struct GarnetApi +``` + +Two concrete instantiations are used: `BasicGarnetApi` (normal operations) and `TransactionalGarnetApi` (within transactions). Type aliases for all context variants are defined in `libs/GlobalUsings.cs`. + +The single store is held by `GarnetDatabase` (`libs/server/GarnetDatabase.cs`) and managed by `StoreWrapper` (`libs/server/StoreWrapper.cs`). Each record carries a `ValueIsObject` bit in its `RecordInfo` header to distinguish raw string values from serialized object values. + +#### Storage layer organization + +Each context type has parallel directory structures: + +- **Functions** (Tsavorite callbacks for RMW, Read, Upsert, Delete): + - `libs/server/Storage/Functions/MainStore/` — string operations + - `libs/server/Storage/Functions/ObjectStore/` — collection operations + - `libs/server/Storage/Functions/UnifiedStore/` — type-agnostic operations +- **Session ops** (StorageSession methods wrapping Tsavorite API): + - `libs/server/Storage/Session/MainStore/` — string ops (MainStoreOps.cs, BitmapOps.cs, HyperLogLogOps.cs) + - `libs/server/Storage/Session/ObjectStore/` — collection ops ([ObjectName]Ops.cs) + - `libs/server/Storage/Session/UnifiedStore/` — unified ops (UnifiedStoreOps.cs) +- **Object implementations**: `libs/server/Objects/[ObjectName]/` — per-type logic (Hash, List, Set, SortedSet, SortedSetGeo) + +### Key Layers + +- **Network/Session** (`libs/common/Networking/`, `libs/server/Sessions/`) — Shared-memory network design where TLS and storage ops run on IO completion threads. `GarnetServerTcp` accepts connections, creates `ServerTcpNetworkHandler` per client. `GarnetProvider` creates `RespServerSession` instances to handle RESP messages. +- **RESP Command Processing** (`libs/server/Resp/`) — Commands are defined as `RespCommand` enum values and dispatched via switch expressions in `ProcessBasicCommands`/`ProcessArrayCommands`. The `RespServerSession` class is split across multiple partial `.cs` files organized by command category. +- **Storage API** (`libs/server/API/`) — Narrow-waist API (`IGarnetApi` inherits `IGarnetReadApi` + `IGarnetAdvancedApi`) with read, upsert, delete, and atomic read-modify-write operations. Command handlers are generic over `TGarnetApi` for testability. `StorageSession` wraps Tsavorite API calls. API methods for string, object, and unified commands are split across `GarnetApi.cs`, `GarnetApiObjectCommands.cs`, and `GarnetApiUnifiedCommands.cs`. +- **Tsavorite Engine** (`libs/storage/Tsavorite/cs/src/core/`) — Has its own solution (`Tsavorite.slnx`) and test project. Provides concurrent key-value storage with checkpointing, tiered storage, recovery, and epoch-based memory reclamation. Relies heavily on `Span` and `SpanByte` for zero-copy memory management. +- **Cluster** (`libs/cluster/`) — Sharding, replication, gossip protocol, key migration. Interface defined in `libs/server/Cluster/IClusterProvider.cs`, implementation in `libs/cluster/Server/`. +- **Database Management** (`libs/server/Databases/`) — Factory pattern with `SingleDatabaseManager` and `MultiDatabaseManager` implementations behind `IDatabaseManager`. Multi-database only available when cluster mode is off. Each `RespServerSession` manages a map of `GarnetDatabaseSession` instances (one per database index). + +### Type Aliases + +The codebase uses `using` aliases extensively for complex generic store types. `libs/GlobalUsings.cs` defines the key aliases: `BasicGarnetApi`, `TransactionalGarnetApi`, `StringBasicContext`, `ObjectBasicContext`, `UnifiedBasicContext`, `StoreAllocator`, and their transactional variants. See also the top of `RespServerSession.cs` and `StoreWrapper.cs`. + +## Adding a New RESP Command + +Full guide: https://microsoft.github.io/garnet/docs/dev/garnet-api + +### Steps for a new built-in command: + +1. **Define the command**: Add enum value to `RespCommand` in `libs/server/Resp/Parser/RespCommand.cs`. For object commands (List, SortedSet, Hash, Set), also add a value to the `[ObjectName]Operation` enum in `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs`. +2. **Add parsing logic**: In `libs/server/Resp/Parser/RespCommand.cs`, add to `FastParseCommand` (fixed arg count) or `FastParseArrayCommand` (variable args). +3. **Declare the API method**: Add method signature to `IGarnetReadApi` (read-only) or `IGarnetApi` (read-write) in `libs/server/API/IGarnetApi.cs`. +4. **Implement the network handler**: Add a method to `RespServerSession` (the class is split across ~22 partial `.cs` files — object commands go in `libs/server/Resp/Objects/[ObjectName]Commands.cs`, others in `libs/server/Resp/BasicCommands.cs`, `ArrayCommands.cs`, `AdminCommands.cs`, `KeyAdminCommands.cs`, etc.). The handler parses arguments from the network buffer via `parseState.GetArgSliceByRef(i)` (returns `ref PinnedSpanByte`), calls the storage API, and writes the RESP response using `RespWriteUtils` helper methods, then calls `SendAndReset()` to flush the response buffer. +5. **Add dispatch route**: In `libs/server/Resp/RespServerSession.cs`, add a case to `ProcessBasicCommands` or `ProcessArrayCommands` calling the handler from step 4. +6. **Implement storage logic**: Add method to `StorageSession`. Choose the appropriate context based on the command type: + - **String commands**: Add to `libs/server/Storage/Session/MainStore/MainStoreOps.cs`. Call Tsavorite's `Read` or `RMW` via the string context. For RMW, implement init/update logic in `libs/server/Storage/Functions/MainStore/RMWMethods.cs`. + - **Object (collection) commands**: Add to `libs/server/Storage/Session/ObjectStore/[ObjectName]Ops.cs`. Call `ReadObjectStoreOperation` or `RMWObjectStoreOperation` via the object context, then implement the case in `libs/server/Objects/[ObjectName]/[ObjectName]ObjectImpl.cs`. + - **Type-agnostic commands** (EXISTS, DELETE, TTL, EXPIRE, TYPE, etc.): Add to `libs/server/Storage/Session/UnifiedStore/UnifiedStoreOps.cs`. Use the unified context. Implement callbacks in `libs/server/Storage/Functions/UnifiedStore/RMWMethods.cs`. +7. **Transaction support**: For standard commands, define `KeySpecs` in the command's metadata — the framework automatically handles key locking via `TxnKeyManager.LockKeys()`. For custom multi-key operations, manually call `txnManager.SaveKeyEntryToLock(key, lockType)` in `libs/server/Transaction/TxnKeyManager.cs`; the key is a `PinnedSpanByte` in the unified key-space, so object-vs-string handling is managed internally by the transaction layer. +8. **Tests**: Add tests using both `StackExchange.Redis` and `LightClient` where applicable. Object command tests go in `test/Garnet.test/Resp[ObjectName]Tests.cs`, others in `test/Garnet.test/RespTests.cs` or similar. +9. **Documentation**: Update the appropriate markdown file under `website/docs/commands/` and mark the command as supported in `website/docs/commands/api-compatibility.md`. +10. **Command info metadata**: Add the command to `playground/CommandInfoUpdater/SupportedCommand.cs`, then run the updater tool: + ```bash + cd playground/CommandInfoUpdater + dotnet run -- --output ../../libs/resources + ``` + +> **Tip**: Write a basic test calling the new command first, then implement missing logic as you debug. + +## Custom Extensions + +Four extensibility points, all in C#. See https://microsoft.github.io/garnet/docs/dev/custom-commands + +- **`CustomRawStringFunctions`** — Custom operations on raw strings (example: `main/GarnetServer/Extensions/DeleteIfMatch.cs`) +- **`CustomObjectBase` + `CustomObjectFactory`** — Custom data types for the object store (example: `main/GarnetServer/Extensions/MyDictObject.cs`) +- **`CustomTransactionProcedure`** — Server-side multi-key transactions (example: `main/GarnetServer/Extensions/ReadWriteTxn.cs`) +- **`CustomProcedure`** — Non-transactional server-side stored procedures (example: `main/GarnetServer/Extensions/Sum.cs`) + +Register server-side via `server.Register` (a `RegisterApi` instance on `GarnetServer`) with `.NewCommand()`, `.NewTransactionProc()`, `.NewProcedure()`, or `.NewType()`. Client-side registration uses the `REGISTERCS` admin command with assemblies on the server. + +## Key Conventions + +### File Headers + +All C# files require this header (enforced by .editorconfig `file_header_template`): + +```csharp +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +``` + +This same header is used throughout the entire codebase, including Tsavorite files. + +### Test Structure + +- **Framework**: NUnit with `[TestFixture]`, `[Test]`, `[SetUp]`, `[TearDown]` +- **Test base class**: All test fixtures must inherit from `TestBase` (defined in `test/standalone/Garnet.test/TestBase.cs`). This tracks currently running tests for diagnostics. +- **Server lifecycle**: Create in `[SetUp]` via `TestUtils.CreateGarnetServer(TestUtils.MethodTestDir)`, call `.Start()`, then `.Dispose()` in `[TearDown]`. Common optional parameters include `enableAOF`, `lowMemory`, `enableTLS`, `enableCluster`, `tryRecover`, `disableObjects`, `useAcl`, and `defaultPassword`. +- **Teardown**: Always call `TestUtils.OnTearDown()` (checks for leaked `LightEpoch` instances) +- **Test directory cleanup**: `TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true)` at the start of `[SetUp]` +- **Namespace**: Test files use `Garnet.test` namespace (even files in subdirectories like `DiskANN/`) +- **Clients in tests**: Use `StackExchange.Redis` for high-level operations, `LightClient` for raw RESP protocol testing + +```csharp +[TestFixture] +public class MyTests : TestBase +{ + GarnetServer server; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + [Test] + public void MyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + // ... test using StackExchange.Redis + } +} +``` + +### Adding Configuration Settings + +To add a new Garnet server setting: +1. Add property to `Options` class in `libs/host/Configuration/Options.cs` with `[Option]` attribute +2. Add default value in `libs/host/defaults.conf` +3. If needed in core code, add matching property to `GarnetServerOptions` (`libs/server/Servers/GarnetServerOptions.cs`) and map it in `Options.GetServerOptions()` +4. Add tests in `test/Garnet.test/GarnetServerConfigTests.cs` + +### Coding Style + +- 4-space indentation, Allman braces (opening brace on new line) +- `var` preferred when type is apparent +- `unsafe` and `AllowUnsafeBlocks` enabled globally +- Private/internal fields: camelCase; constants and statics: PascalCase +- `TreatWarningsAsErrors` is enabled — all warnings must be resolved +- Central package version management via `Directory.Packages.props` +- XML doc comments (`/// `) are strongly recommended on public methods, with `` tags for each parameter; analyzer rules for missing docs are currently configured as suggestions (see `.editorconfig`) +- Comment format: `// Comment starting with a capital letter` (one space after `//`) + +### Performance Conventions + +- Use `Span` and `SpanByte` extensively for zero-copy memory management — avoid allocations on hot paths +- Use `[MethodImpl(MethodImplOptions.AggressiveInlining)]` for hot-path methods +- Use `[MethodImpl(MethodImplOptions.NoInlining)]` for cold-path and exception-throwing methods +- Tsavorite uses `LightEpoch` for epoch-based safe memory reclamation — acquire epoch protection before store operations, release after +- `LightEpoch` instances track ownership — only dispose if owned +- In parallel tests, share a `LightEpoch` instance across `GarnetClient` instances + +### Epoch Protection and Log Address Invariants + +Tsavorite uses **epoch-based memory reclamation** (`LightEpoch`) so writers can publish new values and reclaim old memory only after every reader has moved past it. Any change to the allocator, recovery, scan iterators, transient locking, or callbacks fired from the drain list must respect the rules below. + +**Key files**: `libs/storage/Tsavorite/cs/src/core/Epochs/LightEpoch.cs`, `libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs`, `libs/storage/Tsavorite/cs/src/core/ClientSession/ClientSession.cs`. + +#### Epoch protection model (`LightEpoch`) + +- Each thread acquires a per-instance entry via `epoch.Resume()` (calls `Acquire`) and releases it via `epoch.Suspend()` (calls `Release`). Inside the protected region, the thread's `localCurrentEpoch` is advanced on every `ProtectAndDrain()` call and on entry. +- `Resume()` is **non-reentrant** — `Acquire` asserts if the thread is already protected on this instance. Use `ResumeIfNotProtected()` (returns `true` if it acquired) when code may be entered under an existing hold; pair with a matching `Suspend()` only on the path that took it. +- `BasicContext.{RMW, Upsert, Read, Delete}` wrap the call in `UnsafeResumeThread()` / `UnsafeSuspendThread()` (in `ClientSession`) via try/finally. Custom code that calls `epoch.ProtectAndDrain()` (e.g., spin-waiters in `EpochOperations.SpinWaitUntilClosed`/`SpinWaitUntilRecordIsClosed`, `TransientLocking.LockForScan`) **must already hold the epoch** — the `Debug.Assert(entry > 0, "Trying to refresh unacquired epoch")` in `LightEpoch.ProtectAndDrain` fires otherwise. +- `BumpCurrentEpoch(Action)` increments the global epoch and queues `Action` against the *prior* epoch. The action fires on whatever thread next observes that epoch as safe-to-reclaim — typically inside `ProtectAndDrain` → `Drain`. Therefore actions must be **thread-agnostic** (no thread-affine state) and **safe to fire synchronously** from the bumping thread itself: `BumpCurrentEpoch(Action)` calls `ProtectAndDrain` internally and may execute the action it just queued. + +#### Log address layout and invariants + +The seven log addresses on `AllocatorBase` advance monotonically and obey: + +``` +BeginAddress <= ClosedUntilAddress <= SafeHeadAddress <= HeadAddress + <= FlushedUntilAddress + <= SafeReadOnlyAddress <= ReadOnlyAddress <= TailAddress +``` + +| Address | Meaning | +|---------|---------| +| `BeginAddress` | Lowest valid address. Advancing it logically retires older addresses but **does not delete on-disk files** by itself; physical truncation only happens when a `ShiftBeginAddress` caller passes `truncateLog: true` (typically a checkpoint commit), and even then the device may defer file removal. | +| `ClosedUntilAddress` | Highest address whose page buffer has been freed (`pagePointers[idx] = 0`). | +| `SafeHeadAddress` | High-water set by `OnPagesClosed` *before* freeing — readers see it lead `ClosedUntilAddress`. | +| `HeadAddress` | Lowest in-memory address. May advance while you hold the epoch, **but any address that was `>= HeadAddress` at any point during your protected region cannot be evicted until you `Suspend`**. Capped at `FlushedUntilAddress` — eviction never gets ahead of disk durability. | +| `FlushedUntilAddress` | All bytes below have been written to disk. Updated by flush completion callbacks invoked from `AsyncFlushPagesForReadOnly`. Lags `SafeReadOnlyAddress` (a page is only flushed once it has become safely read-only). | +| `SafeReadOnlyAddress` | Below this, no writer can be in-place mutating. Set by `OnPagesMarkedReadOnly` after writers have drained; same call also kicks off the flush that will later advance `FlushedUntilAddress`. | +| `ReadOnlyAddress` | Maximum address of the immutable region. Records below are flushed/in-flush. | +| `TailAddress` | Next address to allocate; published via the `PageOffset` CAS in `HandlePageOverflow`. | + +#### Cascade pattern: publish → epoch barrier → post-drain action + +Address advancement uses a **publish → bump → action** cascade so that the post-barrier work runs only after every prior holder has observed the new value: + +1. **Publish** the new address into the visible field via `MonotonicUpdate`. +2. **`BumpCurrentEpoch(Action)`** queues the post-barrier work against the prior epoch; it fires once every thread that observed the old value has either `Suspend`ed or `ProtectAndDrain`ed. +3. The **action** does the work that requires "all prior holders have moved past": flush pages, advance the `Safe*` companion, close pages, free buffers, truncate disk segments. + +The two cascades you encounter on the runtime hot path: + +- **Read-only / flush cycle** — `ShiftReadOnlyAddress(newRO)` publishes `ReadOnlyAddress`, then `BumpCurrentEpoch(OnPagesMarkedReadOnly)`. The action advances `SafeReadOnlyAddress` and issues `AsyncFlushPagesForReadOnly`; flush completion later advances `FlushedUntilAddress` via `FlushCallback`. Triggered by `PageAlignedShiftReadOnlyAddress` whenever the tail moves far enough past the read-only region. +- **Eviction / close cycle** — `ShiftHeadAddress(desiredHA)` publishes `HeadAddress`, then `BumpCurrentEpoch(OnPagesClosed)`. The action advances `SafeHeadAddress` and `ClosedUntilAddress`, and frees page buffers via the per-allocator `FreePage` (defined in `SpanByteAllocatorImpl` / `ObjectAllocatorImpl`). Triggered when `FlushedUntilAddress` moves past `HeadAddress + (some delta)`, or explicitly via `ShiftHeadAddressToBlocking`. + +Other cascades: + +- **`ShiftBeginAddress(newBA, truncateLog)`** — publishes `BeginAddress` (and cascades through `ShiftReadOnlyAddress` + `ShiftHeadAddress` if needed). When `truncateLog: true`, also bumps with `TruncateUntilAddress` to drop on-disk segments below the new begin; when `false` (the common case) on-disk segments are left in place to be reclaimed at the next checkpoint commit. Disk file removal itself is asynchronous — even after `TruncateUntilAddress` returns, the device may defer the actual unlink. +- **`ShiftReadOnlyAddressWithWait(newRO, wait)`** — convenience wrapper that uses `ResumeIfNotProtected`/`Suspend` to launch the shift and (optionally) blocks the caller on `FlushedUntilAddress < newRO`. + +#### Rules when changing allocator/iterator/callback code + +1. **Holding the epoch implies stability**: an address observed `>= HeadAddress` during the protected region cannot be evicted before `Suspend()`. Re-acquire after suspend and re-validate. +2. **`Suspend` and `Resume` must be balanced** on every code path. The only suspend inside the basic op path is the `ALLOCATE_FAILED` retry in `HandleRetryStatus`, balanced via try/finally. +3. **Drain-list actions run on arbitrary threads** that hold the epoch. Do not capture thread-static state; do not call code that asserts on a specific thread. +4. **Multi-phase mutations** that need to advance several addresses with barriers between them should use one `BumpCurrentEpoch(Action)` per phase with a `ManualResetEventSlim` to wait. **Drop the prior epoch before waiting** on the MRE — otherwise the drain list cannot make progress (the action you queued cannot fire while you hold the epoch it is gating on). Re-acquire to issue the next bump. `AllocatorBase.Reset` is an example: phase 1 publishes `ReadOnlyAddress` and waits for writers to drain before advancing `SafeReadOnlyAddress`/`FlushedUntilAddress`; phase 2 publishes `HeadAddress` and waits for readers to drain before closing/freeing pages. +5. **Address publication ordering**: when one operation advances multiple addresses, advance the more permissive ones (`HeadAddress`, `ReadOnlyAddress`) before the more restrictive ones (`BeginAddress`). The full invariant `BeginAddress <= ClosedUntilAddress <= SafeHeadAddress <= HeadAddress <= FlushedUntilAddress <= SafeReadOnlyAddress <= ReadOnlyAddress <= TailAddress` must hold throughout, and stale readers caching the older value will route through safer paths (e.g., disk-frame branch in `LoadPageIfNeeded` rather than dereferencing freed `pagePointers`). `AllocatorBase.Reset` publishes `BeginAddress` last for this reason — an iterator with a stale `nextAddress` then routes through the disk-frame path instead of the in-memory page that has just been freed. +6. **Page pointers**: after `OnPagesClosed` → `FreePage`, `pagePointers[idx] = 0`. Iterators must not dereference a page pointer outside the epoch protection that observed `addr >= HeadAddress`. +7. **Scan iterators and `BufferAndLoad`**: `ScanIteratorBase.BufferAndLoad` may internally call `BumpCurrentEpoch`, `ProtectAndDrain`, or `Suspend`+`Resume` on IO, any of which advances the iterator thread's `localCurrentEpoch` and may synchronously fire deferred drain-list actions. Reads stay safe because the IO frame is iterator-owned (allocated in the iterator's constructor) and `headAddress` advances monotonically — `LoadPageIfNeeded` only routes a record to the in-log path when it was `>= HeadAddress` at the time of sampling, so the snapshot's routing decision is always conservative. + +#### Tests that exercise these paths + +- `BasicLockTests.FunctionsLockTest` (in `libs/storage/Tsavorite/cs/test/BasicLockTests.cs`) — multi-threaded RMW/Upsert under contention; exercises Resume/Suspend balance and `ProtectAndDrain`. +- Cluster checkpoint/flush tests under `test/Garnet.test.cluster/` — exercise the full address cascade with live clients. + +### Scratch Buffer Conventions + +`StorageSession` has two scratch buffer types — use the right one: + +- **`ScratchBufferBuilder` (SBB)** — Single contiguous buffer for temporary workspace. All data is laid out sequentially in one buffer. On expansion, the previous data is copied into a new larger buffer and the old buffer is freed — **any existing pointers into the old buffer become invalid**. Use for building command inputs, Lua serialization, or any data that is consumed immediately and then rewound. **Do not** return `PinnedSpanByte` from SBB to callers — it may be invalidated by subsequent allocations. Always rewind after use. Debug builds enforce single-outstanding-slice discipline via asserts. + - Key APIs: `CreateArgSlice` (returns `PinnedSpanByte`, must rewind), `CreateArgSliceAsOffset` (returns `(Offset, Length)`, safe for multi-alloc since offsets survive reallocation), `ViewRemainingArgSlice`/`ViewFullArgSlice` (immediate-use views, do not store), `MoveOffset`, `Reset`, `RewindScratchBuffer`. + +- **`ScratchBufferAllocator` (SBA)** — Maintains a collection of fragmented pinned buffers (via `GC.AllocateArray(_, true)`). When the current buffer fills, a new one is allocated and the old buffer is kept rooted in a stack — so previously returned `PinnedSpanByte` values remain valid. Use for `PinnedSpanByte` values returned via `out` parameters or `IGarnetApi` that callers retain across multiple API calls. Reset between batches. + - Key APIs: `CreateArgSlice`, `ViewRemainingArgSlice`, `Reset`. + +**Rules:** +1. Any `StorageSession` or `IGarnetApi` method returning `PinnedSpanByte` via `out` must use SBA, not SBB. +2. When using SBB's `CreateArgSlice`, always `RewindScratchBuffer` after use — debug asserts enforce at most one outstanding slice. +3. For multiple allocations without rewind, use `CreateArgSliceAsOffset` (returns offsets that survive reallocation). +4. When copying from `IMemoryOwner` (e.g., `ObjectOutput.SpanByteAndMemory.Memory`), always `Dispose()` after copying — do not leak pooled buffers. +5. `ViewFullArgSlice` and `ViewRemainingArgSlice` return immediate-use views — do not store or return them. + +### Releasing a New Version + +To update the version and make a new release, increment the `VersionPrefix` in `Version.props` at the repo root and submit a PR with that change. + +### PR Protocol + +1. Create a GitHub Issue (Enhancement / Bug / Task) +2. Branch naming: `/branch-name` +3. Include unit tests (see Test Structure above) +4. Link PR to the issue in the development section diff --git a/.github/skills/add-garnet-command/SKILL.md b/.github/skills/add-garnet-command/SKILL.md new file mode 100644 index 00000000000..fce040a5773 --- /dev/null +++ b/.github/skills/add-garnet-command/SKILL.md @@ -0,0 +1,796 @@ +--- +name: add-garnet-command +description: Adds a new built-in RESP command to Garnet end-to-end. Covers enum registration, parsing, dispatch, RESP handler, API surface, storage session, RMW callbacks, command metadata JSON, ACL tests, and integration tests. Use when asked to "add a command", "implement RI.SET", "add RESP command", or any new server command. Do NOT use for custom extension commands (CustomRawStringFunctions) or object-type sub-operations. +--- + +# Add a New Built-in RESP Command to Garnet + +Step-by-step guide for implementing a new built-in RESP command in Garnet. This covers every file that must be created or modified, the tools that must be run, caveats discovered during implementation, and how to verify correctness. + +**Scope:** Built-in commands that are part of the Garnet server (not custom extensions registered via `REGISTERCS`). + +--- + +## Overview: What Must Change + +Adding a single new command touches **at minimum** these areas: + +| # | Area | Files | Required? | +|---|------|-------|-----------| +| 1 | RespCommand enum | `libs/server/Resp/Parser/RespCommand.cs` | ✅ Always | +| 2 | Command parsing | `libs/server/Resp/Parser/RespCommand.cs` | ✅ Always | +| 3 | Command dispatch | `libs/server/Resp/RespServerSession.cs` | ✅ Always | +| 4 | RESP handler | `libs/server/Resp//*.cs` | ✅ Always | +| 5 | API interface | `libs/server/API/IGarnetApi.cs` | If key-value command (not blocking/admin) | +| 6 | API delegation | `libs/server/API/GarnetApi*.cs` | If key-value command (not blocking/admin) | +| 7 | Storage session | `libs/server/Storage/Session/[Main\|Object\|Unified]Store/*Ops.cs` | If key-value command (not blocking/admin) | +| 8 | RMW/Read callbacks | `libs/server/Storage/Functions/[Main\|Unified]Store/[RMW\|Read]Methods.cs` | If string/unified command using RMW/Read | +| 8b | Read response | `libs/server/Storage/Functions/MainStore/PrivateMethods.cs` | If string command using Read (add to `CopyRespToWithInput`) | +| 9 | VarLen methods | `libs/server/Storage/Functions/[Main\|Unified]Store/VarLenInputMethods.cs` | If string/unified command using RMW | +| 10 | Object operation enum | `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs` | If new object sub-operation | +| 11 | Object implementation | `libs/server/Objects/[ObjectName]/[ObjectName]ObjectImpl.cs` | If new object sub-operation | +| 12 | ItemBroker | `libs/server/Objects/ItemBroker/CollectionItemBroker.cs` | If blocking command | +| 13 | Command info JSON | `libs/resources/RespCommandsInfo.json` | ✅ Always (generated) | +| 14 | Command docs JSON | `libs/resources/RespCommandsDocs.json` | ✅ Always (generated) | +| 15 | Supported commands | `playground/CommandInfoUpdater/SupportedCommand.cs` | ✅ Always | +| 16 | Garnet command info | `playground/CommandInfoUpdater/GarnetCommandsInfo.json` | If Garnet-only command | +| 17 | Garnet command docs | `playground/CommandInfoUpdater/GarnetCommandsDocs.json` | If Garnet-only command | +| 18 | ACL test | `test/Garnet.test/Resp/ACL/RespCommandTests.cs` | ✅ Always | +| 19 | Integration tests | `test/Garnet.test/Resp*.cs` | ✅ Always | +| 20 | Website documentation | `website/docs/commands/` | ✅ Always | +| 21 | Configuration settings | `Options.cs`, `GarnetServerOptions.cs`, `defaults.conf` | If command is optional/gated | + +--- + +## Step 1: Add the RespCommand Enum Value + +**File:** `libs/server/Resp/Parser/RespCommand.cs` + +The `RespCommand` enum is divided into sections with **ordering that matters**: + +``` +Read commands: BITCOUNT ... ZUNION (before APPEND) +Write commands: APPEND ... BITOP_DIFF (after APPEND) +Script commands: EVAL, EVALSHA +Non-key commands: PING, SUBSCRIBE, etc. +Admin commands: AUTH, CONFIG, etc. +``` + +**Read/write classification uses enum ordering:** +- `cmd < RespCommand.APPEND` → read-only +- `cmd >= RespCommand.APPEND && cmd <= RespCommand.BITOP_DIFF` → write + +**Rules:** +- Read-only commands go **before** `APPEND` +- Write commands go **between** `APPEND` and `BITOP_DIFF` +- Update the boundary comments if you add before `APPEND` or after `BITOP_DIFF` +- Place alphabetically within the appropriate section + +**Boundary markers to watch (search for these comments):** +```csharp +ZUNION, // Note: Last read command is determined by APPEND - 1 +APPEND, // Note: Update FirstWriteCommand if adding new write commands before this +BITOP_DIFF, // Note: Update LastWriteCommand if adding new write commands after this +EVALSHA, // Note: Update LastDataCommand if adding new data commands after this +``` + +**⚠️ Caveat:** The boundary comments in the source may not be on the actual last/first entry (e.g., `ZSCORE` has the comment but `ZUNION` follows it). The real boundary is determined by code: `LastReadCommand = RespCommand.APPEND - 1`. Always check the actual enum ordering, not just the comments. + +--- + +## Step 2: Add Command Parsing + +**File:** `libs/server/Resp/Parser/RespCommand.cs` + +Two parsing paths exist: + +### Fast path: `FastParseCommand()` / `FastParseArrayCommand()` +Two fast-path methods exist with different constraints: +- **`FastParseCommand()`**: For commands with a fixed number of arguments and command names up to **9 characters**. Uses `ulong` pointer comparisons on `(count << 4) | length` patterns. +- **`FastParseArrayCommand()`**: For commands with a variable number of arguments and command names up to **16 characters**. Uses similar `ulong` comparison patterns but accommodates longer names. + +Only add here if the command name is a simple word (no dots or special characters). + +### Slow path: `SlowParseCommand()` +For longer names, dot-prefixed names (like `RI.CREATE`), or names that don't fit the fast-path pattern. + +**⚠️ Convention:** Define the command name string in **`libs/server/Resp/CmdStrings.cs`** and reference it from the parser, rather than using inline `"..."u8` literals. This keeps command name strings centralized and reusable (e.g., for error messages). + +```csharp +// In CmdStrings.cs: +public static ReadOnlySpan DELIFGREATER => "DELIFGREATER"u8; +``` + +**Pattern for slow-path commands:** +```csharp +else if (command.SequenceEqual(CmdStrings.DELIFGREATER)) +{ + return RespCommand.DELIFGREATER; +} +``` + +**Pattern for dot-prefixed commands (e.g., `RI.CREATE`):** +```csharp +else if (command.SequenceEqual(CmdStrings.RICREATE)) +{ + return RespCommand.RICREATE; +} +``` + +Add this before the final `return RespCommand.INVALID;` at the end of `SlowParseCommand`. + +**⚠️ Caveat: Dot-prefixed commands and ACL** +If your command uses a dot (e.g., `RI.CREATE`), you must also update **`libs/server/ACL/ACLParser.cs`** so that the ACL system can map the dotted wire name to the enum name. Search for how existing dot-handling works (look for `Replace(".", "")` or similar normalization). + +--- + +## Step 3: Add Command Dispatch + +**File:** `libs/server/Resp/RespServerSession.cs` + +Three dispatch methods exist, and which one you use matters for latency tracking: + +| Method | For | Latency | +|--------|-----|---------| +| `ProcessBasicCommands` | Fast single/dual-arg commands | @fast only | +| `ProcessArrayCommands` | Fast multi-arg commands | @fast only | +| `ProcessOtherCommands` | Slow commands, admin commands | @slow OK | + +**⚠️ WARNING:** Do NOT add `@slow`-classified commands to `ProcessBasicCommands` or `ProcessArrayCommands`. This breaks latency tracking. If in doubt, use `ProcessOtherCommands`. + +**Pattern:** +```csharp +RespCommand.MYCMD => NetworkMYCMD(ref storageApi), +``` + +Add before the `_ => ...` fallthrough in the appropriate method. + +--- + +## Step 4: Implement the RESP Handler + +**New file or existing file** in `libs/server/Resp/` + +Command handlers are methods on the `RespServerSession` partial class: + +```csharp +private bool NetworkMYCMD(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi +{ + // 1. Validate argument count + if (parseState.Count != N) + return AbortWithWrongNumberOfArguments(nameof(RespCommand.MYCMD)); + + // 2. Validate other inputs (short-circuit before going to storage) + var key = parseState.GetArgSliceByRef(0); + // e.g., parse and validate optional flags, numeric arguments, etc. + if (!parseState.TryGetInt(1, out var _)) + { + WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER); + return true; + } + + // 3. Build input/output and call storage API + // Note: To avoid double-parsing a parameter, you can pass a pre-parsed + // value in the input struct's auxiliary arguments (e.g., input.arg1). + var input = new StringInput(RespCommand.MYCMD, ref parseState, startIdx: 1); + var output = GetStringOutput(); + var status = storageApi.MyOperation(key, ref input, ref output); + + // 4. Write RESP response + if (status == GarnetStatus.OK) + { + ProcessOutput(output); + } + else + { + WriteError(CmdStrings.RESP_ERR_MY_MESSAGE); + } + + return true; +} +``` + +**Key patterns:** +- Arguments: `parseState.GetArgSliceByRef(i)` returns `ref PinnedSpanByte` +- Input/Output: Instantiate `StringInput`/`StringOutput` (for string commands), `ObjectInput`/`ObjectOutput` (for object commands), or `UnifiedInput`/`UnifiedOutput` (for unified commands) before calling the storage API +- Response (happy path): Use `ProcessOutput(output)` in the common case — this handles writing the RESP response from the output struct +- Response (errors/special cases): Use `RespServerSession` extension methods (e.g., `WriteError(...)`, `WriteDirect(...)`, `WriteInt64(...)`, etc.) — these handle `SendAndReset()` internally +- Error strings: Store as `u8` literals in `CmdStrings` (e.g., `CmdStrings.RESP_ERR_MY_MESSAGE`) rather than inline +- Always return `true` — there are no partial executions + +### Object command RESP handler pattern + +Object commands (Hash, List, Set, SortedSet) follow a similar pattern to string commands. The main difference is that the RESP handler uses `ObjectInput`/`ObjectOutput` with the appropriate operation enum and must handle `WRONGTYPE` errors: + +**File:** `libs/server/Resp/Objects/[ObjectName]Commands.cs` (e.g., `SortedSetCommands.cs`) + +```csharp +private unsafe bool SortedSetAdd(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi +{ + if (parseState.Count < 3) + return AbortWithWrongNumberOfArguments("ZADD"); + + var key = parseState.GetArgSliceByRef(0); + + var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZADD }; + var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = GetObjectOutput(); + + var status = storageApi.SortedSetAdd(key, ref input, ref output); + + switch (status) + { + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + default: + ProcessOutput(output.SpanByteAndMemory); + break; + } + + return true; +} +``` + +**Key differences from string commands:** +- Uses `ObjectInput` with a `RespInputHeader(GarnetObjectType.XXX)` and an operation enum (e.g., `SortedSetOperation.ZADD`) +- Must handle `GarnetStatus.WRONGTYPE` — object commands can fail if the key holds a different object type +- The actual data operation logic lives in `libs/server/Objects/[ObjectName]/[ObjectName]ObjectImpl.cs`, dispatched via the operation enum + +**For new object sub-operations:** +Add a value to the `[ObjectName]Operation` enum in `libs/server/Objects/[ObjectName]/[ObjectName]Object.cs` and handle it in the `Operate` method's switch statement. + +### Unified command note + +Unified commands (EXISTS, DELETE, TYPE, TTL, EXPIRE, RENAME, etc.) are type-agnostic — they work on both raw string and object values. The RESP handler pattern is the same as string and object commands, but uses `UnifiedInput`/`UnifiedOutput`. The storage session layer uses the unified context (`unifiedBasicContext`), and the callbacks go in `libs/server/Storage/Functions/UnifiedStore/`. + +### Blocking command RESP handler pattern + +Blocking commands (`BLPOP`, `BRPOP`, `BLMOVE`, `BLMPOP`, `BZPOPMIN`, `BZPOPMAX`, `BZMPOP`) follow a distinct pattern. They do **not** use `ref storageApi` and instead interact with the `CollectionItemBroker` (`libs/server/Objects/ItemBroker/CollectionItemBroker.cs`), which manages blocking/waiting behavior: + +```csharp +private unsafe bool SortedSetBlockingPop(RespCommand command) +{ + if (parseState.Count < 2) + return AbortWithWrongNumberOfArguments(command.ToString()); + + if (!parseState.TryGetTimeout(parseState.Count - 1, out var timeout, out var error)) + return AbortWithErrorMessage(error); + + var keysBytes = new byte[parseState.Count - 1][]; + for (var i = 0; i < keysBytes.Length; i++) + keysBytes[i] = parseState.GetArgSliceByRef(i).ToArray(); + + var result = storeWrapper.itemBroker.GetCollectionItemAsync(command, keysBytes, this, timeout).Result; + + if (!result.Found) + { + WriteNull(); + } + else + { + // Write RESP response with result.Key, result.Item, result.Score, etc. + } + + return true; +} +``` + +**Key differences from regular commands:** +- The dispatch in `RespServerSession.cs` does NOT pass `ref storageApi`: `RespCommand.BZMPOP => SortedSetBlockingMPop(),` +- The handler calls `storeWrapper.itemBroker.GetCollectionItemAsync()` which blocks (with timeout) until data is available +- No `IGarnetApi` method, no storage session method, and no RMW callbacks are needed for the blocking command itself (Steps 5-7 are skipped) +- The `CollectionItemBroker` is notified when data is added to a collection (e.g., `ZADD` calls `itemBroker.HandleCollectionUpdate(key)`), which wakes up blocked clients +- When adding a new blocking command, you must also update the `TryGetResult` method in `CollectionItemBroker.cs` to map your `RespCommand` to the correct `GarnetObjectType` and implement the retrieval logic + +--- + +## Steps 5–7: Storage Layer (skip for admin/non-key commands) + +> **Note:** Steps 5, 6, and 7 apply only to commands that read or write key-value data through the store (e.g., `SET`, `GET`, `DELIFGREATER`). Admin commands like `DEBUG`, `PING`, `CONFIG`, etc. handle their logic entirely in the RESP handler (Step 4) and do **not** need API interface methods, storage session ops, or RMW callbacks. Skip to Step 8 for those. Blocking commands (e.g., `BZMPOP`) also skip Steps 5-7 — see the blocking command pattern in Step 4. +> +> **Note on context types:** The unified single-store has three context types: **string context** (for raw string commands like GET/SET), **object context** (for collection commands like ZADD/LPUSH), and **unified context** (for type-agnostic commands like EXISTS/DELETE/TTL/EXPIRE). Most new commands use either the string or object context — the unified context is only for commands that must work across both value types. + +## Step 5: Add API Interface Method + +**File:** `libs/server/API/IGarnetApi.cs` + +Add method signature to `IGarnetApi` (read-write) or `IGarnetReadApi` (read-only): + +```csharp +// String command: +GarnetStatus MyOperation(PinnedSpanByte key, ref StringInput input, ref StringOutput output); + +// Object command: +GarnetStatus MyOperation(PinnedSpanByte key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + +// Unified command: +GarnetStatus MyOperation(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); +``` + +**File:** `libs/server/API/GarnetApi*.cs` + +Add delegation in the `GarnetApi` partial struct. The implementation goes in the appropriate partial file based on the context type: +- `GarnetApi.cs` — string commands +- `GarnetApiObjectCommands.cs` — object commands +- `GarnetApiUnifiedCommands.cs` — unified commands + +```csharp +public GarnetStatus MyOperation(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.MyOperation(key, ref input, ref output); +``` + +**⚠️ Caveat:** `GarnetApi` is a generic partial struct: `GarnetApi`. Always add your method to the correct partial file for the context type you're using. + +**Overloads for programmatic callers:** In addition to the primary signature (used by the network handler), you can add simpler overloads for programmatic/embedded callers that avoid forcing them to create the Input/Output structs. For example: + +```csharp +public GarnetStatus MyOperation(PinnedSpanByte key, double val, out double output) +``` + +This overload internally creates the appropriate input/output structs and only returns the desired value to the caller, instead of writing to the output buffer. + +--- + +## Step 6: Implement Storage Session Layer + +**File:** New or existing file in `libs/server/Storage/Session/MainStore/` (for string-context ops), `libs/server/Storage/Session/ObjectStore/` (for object-context ops), or `libs/server/Storage/Session/UnifiedStore/` (for unified-context ops) + +### String command pattern + +This layer wraps Tsavorite API calls. The network path uses a generic context parameter: + +```csharp +public GarnetStatus MyOperation(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext<...> +{ + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); + if (status.IsPending) + CompletePendingForSession(ref status, ref output, ref context); + + return GarnetStatus.OK; +} +``` + +Object and unified commands follow the same pattern — just substitute the appropriate context, input, and output types: + +| Context type | Input type | Output type | Helper method | +|-------------|-----------|------------|---------------| +| String | `StringInput` | `StringOutput` | `context.RMW(...)` / `context.Read(...)` | +| Object | `ObjectInput` | `GarnetObjectStoreOutput` | `RMWObjectStoreOperation(...)` / `ReadObjectStoreOperation(...)` | +| Unified | `UnifiedInput` | `UnifiedOutput` | `context.RMW(...)` / `context.Read(...)` | + +**Programmatic overloads:** You can also add simpler overloads for programmatic callers (see Step 5 note). These internally create the input/output structs and return only the desired value. + +### Object-specific: HandleCollectionUpdate + +Object commands use the same pattern as above with `ObjectInput`/`GarnetObjectStoreOutput` and the object context. + +**⚠️ Caveat — `HandleCollectionUpdate`:** If your object command modifies a collection (adds/removes elements), call `itemBroker.HandleCollectionUpdate(key)` after the store operation. This wakes up any clients blocked on that key (e.g., via `BZPOPMIN`). The actual data logic is implemented in the object class, not in the storage session. + +### Object implementation + +**File:** `libs/server/Objects/[ObjectName]/[ObjectName]ObjectImpl.cs` + +For object commands, the core logic lives in the object implementation. The `Operate` method in `[ObjectName]Object.cs` dispatches to implementation methods based on the operation enum: + +```csharp +case SortedSetOperation.ZADD: + SortedSetAdd(ref input, ref output.SpanByteAndMemory); + break; +``` + +The implementation methods in `[ObjectName]ObjectImpl.cs` directly manipulate the object's internal data structures (e.g., `sortedSet`, `sortedSetDict` for SortedSet). + +--- + +## Step 7: Add RMW/Read Callbacks (if applicable) + +**File:** `libs/server/Storage/Functions/MainStore/RMWMethods.cs` (string commands) or `libs/server/Storage/Functions/UnifiedStore/RMWMethods.cs` (unified commands) + +If your command uses `RMW`, you must handle these callbacks: + +| Callback | When | Purpose | +|----------|------|---------| +| `NeedInitialUpdate` | Key doesn't exist | Return `true` to create record | +| `InitialUpdater` | Creating new record | Write initial value | +| `NeedCopyUpdate` | Key exists, record needs copy | Return `true` to copy-update, `false` to skip | +| `InPlaceUpdater` | Key exists, update in place | Modify existing value | +| `CopyUpdater` | Key exists, copy to new record | Write updated value to new record | + +Add a `case RespCommand.MYCMD:` to each relevant switch statement. + +**For Read commands (MainStore):** If your command uses `Read` (not RMW), the read response logic lives in `libs/server/Storage/Functions/MainStore/PrivateMethods.cs` — add a case to the `CopyRespToWithInput` method. + +**File:** `libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs` (string commands) or `libs/server/Storage/Functions/UnifiedStore/VarLenInputMethods.cs` (unified commands) + +If your command writes a value, you must specify the value length: + +| Method | Purpose | +|--------|---------| +| `GetRMWInitialFieldInfo` | Size of value for new records | +| `GetRMWModifiedFieldInfo` | Size of value for updated records | + +**⚠️ Caveat — RecordType:** +If your command creates records with a custom `RecordType` (e.g., for type discrimination), you can set it in `InitialUpdater` after record initialization: + +```csharp +var header = logRecord.RecordDataHeader; +header.RecordType = MyManager.MyRecordType; +``` + +This works because `RecordDataHeader.RecordType` has a setter that writes through a raw pointer. No Tsavorite infrastructure changes needed (despite the TODO comments in `LogRecord.cs`). + +--- + +## Step 8: Command Metadata Registration + +### 8a. Add to SupportedCommand.cs + +**File:** `playground/CommandInfoUpdater/SupportedCommand.cs` + +Add entry following the existing ordering/grouping in the file: +```csharp +new("MY.CMD", RespCommand.MYCMD, StoreType.Main), +``` + +> **Note:** The file is not strictly alphabetical — entries are grouped by category (e.g., script commands at the end). Follow the existing grouping conventions rather than inserting strictly alphabetically. + +For admin/non-key commands (e.g., `DEBUG`, `PING`), omit `StoreType` or use `StoreType.None`: +```csharp +new("DEBUG", RespCommand.DEBUG), +``` + +`StoreType` values: `Main` (string store), `Object` (object store), `All` (both), `None` (no keys). + +### 8b. Add to GarnetCommandsInfo.json (Garnet-only commands) + +**File:** `playground/CommandInfoUpdater/GarnetCommandsInfo.json` + +Needed for commands that don't exist in standard Redis (e.g., `DELIFGREATER`, `SETIFMATCH`), or standard Redis commands whose info you need to override. Standard Redis commands (e.g., `DEBUG`, `GETDEL`) normally get their metadata from a running RESP server automatically via the CommandInfoUpdater tool — skip this step and Step 8c for those unless you need to override their info. + +Add a JSON entry: + +```json +{ + "Command": "MYCMD", + "Name": "MY.CMD", + "IsInternal": false, + "Arity": -2, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { "TypeDiscriminator": "BeginSearchIndex", "Index": 1 }, + "FindKeys": { "TypeDiscriminator": "FindKeysRange", "LastKey": 0, "KeyStep": 1, "Limit": 0 }, + "Flags": "RW, Insert" + } + ], + "StoreType": "Main" +} +``` + +**Key fields:** +- `Arity`: Positive = exact arg count (including command name); Negative = minimum +- `Flags`: `ReadOnly`, `Write`, `DenyOom`, `Fast`, `Admin`, `NoAuth`, `Module`, etc. +- `AclCategories`: Used for ACL permission checks. Use `Garnet` for Garnet-specific commands +- `KeySpecifications`: Drives automatic transaction key locking — no per-command switch needed +- `KeySpec.Flags`: `RO` (read-only), `RW` (read-write), `Access`, `Insert`, `Update`, `Delete` + +### 8c. Add to GarnetCommandsDocs.json (Garnet-only commands) + +**File:** `playground/CommandInfoUpdater/GarnetCommandsDocs.json` + +> **Note:** This step is not necessary for internal commands. The main purpose of command docs is to enable client auto-complete for the command. + +Add documentation entry: + +```json +{ + "Command": "MYCMD", + "Name": "MY.CMD", + "Summary": "Description of what the command does.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] +} +``` + +**⚠️ Caveat — Group must be a valid `RespCommandGroup` enum value:** +`Bitmap`, `Cluster`, `Connection`, `Generic`, `Geo`, `Hash`, `HyperLogLog`, `List`, `Module`, `PubSub`, `Scripting`, `Sentinel`, `Server`, `Set`, `SortedSet`, `Stream`, `String`, `Transactions` + +Do NOT invent new group names — the JSON deserializer will fail. + +### 8d. Generate the resource JSON files + +**⚠️ CRITICAL: Never edit `libs/resources/RespCommandsInfo.json` or `libs/resources/RespCommandsDocs.json` directly.** These are generated by the CommandInfoUpdater tool. + +**Steps:** +1. Start a local RESP-compatible server (e.g., **Valkey** or Redis) — the tool queries it for standard Redis command metadata: + ```bash + valkey-server --port 6399 + ``` +2. Build and run the tool: + ```bash + cd playground/CommandInfoUpdater + dotnet build -f net10.0 + dotnet run -f net10.0 --no-build -- --port 6399 --output ../../libs/resources + ``` + (The `--port` must match the port of the local RESP server.) +3. The tool will prompt `Would you like to continue? (Y/N)` **twice** (once for info, once for docs). Press `Y` for both. +4. Kill the local RESP server afterward. + +**⚠️ Caveat:** The tool uses `Console.ReadKey()` which does NOT work with piped input. You must run it interactively (not via `echo "Y" | dotnet run ...`). For AI agents, use an async shell session and send `Y` keystrokes via interactive input (e.g., `write_bash`). + +**⚠️ Caveat:** The tool requires a running RESP-compatible server (e.g., Valkey or Redis — **not** Garnet) to query standard command metadata. For Garnet-only commands, the tool reads from `GarnetCommandsInfo.json` and `GarnetCommandsDocs.json` instead. + +--- + +## Step 9: Add ACL Test + +**File:** `test/Garnet.test/Resp/ACL/RespCommandTests.cs` + +**⚠️ CRITICAL:** The `AllCommandsCovered` test automatically validates that **every** `RespCommand` enum value has a corresponding ACL test. If you add a command without an ACL test, `AllCommandsCovered` will fail. + +**Test naming convention:** +- Method name must end with `ACLs` or `ACLsAsync` +- The name (minus the suffix) must match the command name with dots and underscores removed +- Example: `RI.CREATE` → `RICreateACLsAsync` + +**Pattern (follow SADD for non-idempotent commands):** +```csharp +[Test] +public async Task MyCommandACLsAsync() +{ + int count = 0; + + await CheckCommandsAsync( + "MY.CMD", + [DoMyCommandAsync] + ).ConfigureAwait(false); + + async Task DoMyCommandAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("MY.CMD", + [$"key-{count}", "arg1"]).ConfigureAwait(false); + count++; + ClassicAssert.AreEqual("OK", val); + } +} +``` + +**⚠️ Caveat — Idempotency:** The ACL framework calls your command multiple times (for different user/permission combinations). If your command is NOT idempotent (e.g., `RI.CREATE` fails on duplicate), use a counter to generate unique keys per invocation (see the SADD ACL test pattern). + +--- + +## Step 10: Add Integration Tests + +**File:** Add tests to an existing or new file in `test/Garnet.test/`: +- **String / Unified commands**: Add to `test/Garnet.test/RespTests.cs` +- **Object commands**: Add to `test/Garnet.test/Resp[ObjectName]Tests.cs` (e.g., `RespSortedSetTests.cs`) +- **New feature area**: Create `test/Garnet.test/RespTests.cs` if the command doesn't fit existing test files + +**Required structure:** +```csharp +[TestFixture] +public class RespMyFeatureTests : TestBase +{ + GarnetServer server; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + [Test] + public void MyBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var result = db.Execute("MY.CMD", "key", "value"); + ClassicAssert.AreEqual("OK", (string)result); + } +} +``` + +**Note:** Test fixtures must inherit from `TestBase`. + +**Recommended test cases:** +- Basic success case +- Error cases (wrong args, wrong type) +- Duplicate/idempotency behavior +- DEL/UNLINK interaction (if applicable) +- Type safety (e.g., string command on object key → WRONGTYPE) + +--- + +## Step 11: Update Website Documentation + +**File:** `website/docs/commands/` — choose the appropriate markdown file based on the command category (e.g., `garnet-specific.md` for Garnet-only commands, `api-compatibility.md` to mark a standard Redis command as supported). + +Add a section documenting the command syntax, description, and response format: + +```markdown +### **MY.CMD** + +#### **Syntax** + +```bash +MY.CMD key value +``` + +Description of what the command does. + +#### **Response** + +- **Type reply**: Description of the response. +``` + +Also mark the command as supported in `website/docs/commands/api-compatibility.md` if it corresponds to a standard Redis command. + +--- + +## Step 11b: Add Configuration Settings (if needed) + +If the command is optional, gated behind a feature flag, or needs a server-side configuration parameter (e.g., `DEBUG` requires `--enable-debug-command`), you must wire up a configuration option across four files: + +### 1. Add property to `Options` class + +**File:** `libs/host/Configuration/Options.cs` + +Add a property with the `[Option]` attribute (from CommandLineParser): + +```csharp +[OptionValidation] +[Option("enable-my-feature", Required = false, HelpText = "Enable MY.CMD for 'no', 'local' or 'all' connections")] +public ConnectionProtectionOption EnableMyFeature { get; set; } +``` + +The `[Option]` attribute defines the CLI flag name (kebab-case). Use `Required = false` for optional settings. Common types: `bool`, `int`, `string`, `ConnectionProtectionOption` (for no/local/yes connection gating), or custom enums. + +### 2. Map to `GarnetServerOptions` + +**File:** `libs/server/Servers/GarnetServerOptions.cs` + +Add a matching field: + +```csharp +/// +/// Enables MY.CMD +/// +public ConnectionProtectionOption EnableMyFeature; +``` + +Then in `Options.GetServerOptions()` (in `Options.cs`), map the property: + +```csharp +EnableMyFeature = EnableMyFeature, +``` + +### 3. Add default value + +**File:** `libs/host/defaults.conf` + +Add the default in the appropriate section: + +```json +/* Enable MY.CMD for clients - no/local/yes */ +"EnableMyFeature": "no", +``` + +### 4. Check the setting in your RESP handler + +Access the setting via `storeWrapper.serverOptions`: + +```csharp +if (storeWrapper.serverOptions.EnableMyFeature == ConnectionProtectionOption.No) +{ + while (!RespWriteUtils.TryWriteError("ERR command not enabled"u8, ref dcurr, dend)) + SendAndReset(); + return true; +} +``` + +### 5. Add config tests + +**File:** `test/Garnet.test/GarnetServerConfigTests.cs` + +Test that the setting is parsed correctly from CLI args and config files. + +--- + +## Step 12: Verify Everything + +### Build +```bash +dotnet build Garnet.slnx -c Debug +``` + +### Format check +```bash +dotnet format Garnet.slnx --verify-no-changes +``` + +**⚠️ Caveat:** New files commonly fail with `FINALNEWLINE` errors. Ensure files do NOT have a trailing newline at the very end. Fix with: `perl -pi -e 'chomp if eof' path/to/file.cs` + +### Run your tests +```bash +dotnet test test/Garnet.test -f net10.0 -c Debug --filter "FullyQualifiedName~RespMyFeatureTests" +``` + +### Run ACL coverage test +```bash +dotnet test test/Garnet.test -f net10.0 -c Debug --filter "FullyQualifiedName~AllCommandsCovered" +``` + +### Run broader regression tests +```bash +dotnet test test/Garnet.test -f net10.0 -c Debug --filter "FullyQualifiedName~RespTests" +``` + +--- + +## Transaction Support + +For standard commands, transaction key locking is **automatic** — driven by `KeySpecifications` in the command metadata JSON. No per-command code is needed in `TxnKeyManager.cs`. + +For custom multi-key operations that don't fit the standard key spec pattern, manually call `txnManager.SaveKeyEntryToLock(key, lockType)`. + +--- + +## Common Caveats and Gotchas + +1. **Dot-prefixed commands (e.g., `RI.CREATE`):** The RESP wire name uses a dot, but the enum name cannot. The ACL parser, `AllCommandsCovered` test, and `SupportedCommand.cs` all need to handle the dot-to-enum mapping. Check `ACLParser.cs` for normalization logic. + +2. **`AllCommandsCovered` is strict:** It reflects over ALL `RespCommand` enum values and ALL entries in `RespCommandsInfo.json`. Missing either an ACL test or a JSON entry will fail this test. + +3. **`NeedCopyUpdate` for create-only commands:** If your command should NOT overwrite existing records (like `SETNX`), return `false` from `NeedCopyUpdate` for your command. Otherwise Tsavorite will attempt a copy-update when the record can't be updated in-place. + +4. **`VarLenInputMethods.cs` is easy to forget:** If your command creates or modifies records via RMW, you must add cases to `GetRMWInitialFieldInfo` and `GetRMWModifiedFieldInfo`. Without this, Tsavorite won't allocate the right amount of space for your value. + +5. **Resource JSON files are generated, not hand-edited:** `libs/resources/RespCommandsInfo.json` and `libs/resources/RespCommandsDocs.json` are generated by `playground/CommandInfoUpdater`. Edit the source files (`GarnetCommandsInfo.json`, `GarnetCommandsDocs.json`, `SupportedCommand.cs`) and run the tool. + +6. **`RespCommandGroup` enum is closed:** The `Group` field in docs JSON must match a value in the `RespCommandGroup` enum (`libs/server/Resp/RespCommandDocs.cs`). Use `Generic` if no existing group fits. + +7. **File headers:** All C# files require `// Copyright (c) Microsoft Corporation.` / `// Licensed under the MIT license.` + +8. **Test resource usage:** Use small values for buffer sizes, cache sizes, etc. in tests. Don't allocate 16MB+ buffers when 64KB will do. + +--- + +## Reference: RI.CREATE Implementation + +The `RI.CREATE` command was the first command implemented following this guide. Key files for reference: + +| File | Purpose | +|------|---------| +| `libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs` | RESP handler with option parsing | +| `libs/server/Resp/RangeIndex/RangeIndexManager.cs` | Manager pattern for external data | +| `libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs` | Fixed-size stub struct in store | +| `libs/server/Storage/Session/MainStore/RangeIndexOps.cs` | StorageSession → RMW flow | +| `test/Garnet.test/RespRangeIndexTests.cs` | Integration tests | +| `test/Garnet.test/Resp/ACL/RespCommandTests.cs` | ACL test (search for `RICreateACLsAsync`) | diff --git a/.github/skills/pr-finalize/SKILL.md b/.github/skills/pr-finalize/SKILL.md new file mode 100644 index 00000000000..085a26decb7 --- /dev/null +++ b/.github/skills/pr-finalize/SKILL.md @@ -0,0 +1,361 @@ +--- +name: pr-finalize +description: Finalizes any PR for merge by verifying title/description match implementation AND performing code review for best practices. Use when asked to "finalize PR", "check PR description", "review commit message", before merging any PR, or when PR implementation changed during review. Do NOT use for extracting lessons or investigating build failures. +--- + +# PR Finalize + +Ensures PR title and description accurately reflect the implementation, and performs a **code review** for Garnet best practices before merge. + +**Standalone skill** — Can be used on any PR. + +## Two-Phase Workflow + +1. **Title & Description Review** — Verify PR metadata matches implementation +2. **Code Review** — Review code for Garnet-specific best practices and potential issues + +--- + +## 🚨 CRITICAL RULES + +### 1. NEVER Approve or Request Changes + +**AI agents must NEVER use `--approve` or `--request-changes` flags.** + +| Action | Allowed? | Why | +|--------|----------|-----| +| `gh pr review --approve` | ❌ **NEVER** | Approval is a human decision | +| `gh pr review --request-changes` | ❌ **NEVER** | Blocking PRs is a human decision | + +### 2. NEVER Post Comments Directly + +**This skill is ANALYSIS ONLY.** Never post comments using `gh` commands. + +| Action | Allowed? | Why | +|--------|----------|-----| +| `gh pr review --comment` | ❌ **NEVER** | Present findings to the user instead | +| `gh pr comment` | ❌ **NEVER** | Present findings to the user instead | +| Analyze and report findings | ✅ **YES** | This is the skill's purpose | + +**Only humans control when comments are posted.** Your job is to analyze and present findings. + +--- + +## Phase 1: Title & Description + +### Core Principle: Preserve Quality + +**Review existing description BEFORE suggesting changes.** Many PR authors write excellent, detailed descriptions. Your job is to: + +1. **Evaluate first** — Is the existing description good? Better than a template? +2. **Preserve quality** — Don't replace a thorough description with a generic template +3. **Enhance, don't replace** — Add missing required elements (issue links, test info) without rewriting good content +4. **Only rewrite if needed** — When description is stale, inaccurate, or missing key information + +## Usage + +```bash +# Get current state (no local checkout required) +gh pr view XXXXX --json title,body +gh pr view XXXXX --json files --jq '.files[].path' + +# Review commit messages (helpful for squash/merge commit quality) +gh pr view XXXXX --json commits --jq '.commits[].messageHeadline' + +# Review actual code changes +gh pr diff XXXXX + +# Optional: if the PR branch is checked out locally +git diff origin/main...HEAD +``` + +## Evaluation Workflow + +### Step 1: Review Existing Description Quality + +Before suggesting changes, evaluate the current description: + +| Quality Indicator | Look For | +|-------------------|----------| +| **Structure** | Clear sections, headers, organized flow | +| **Technical depth** | File-by-file changes, specific code references | +| **Scannability** | Easy to find what changed and where | +| **Accuracy** | Matches actual diff — not stale or incorrect | +| **Completeness** | Breaking changes, performance impact, testing info | + +### Step 2: Compare to Template + +Ask: "Is the existing description better than what my template would produce?" + +- **If YES**: Keep existing, only add missing required elements +- **If NO**: Suggest improvements or replacement + +### Step 3: Produce Output + +- Recommended PR title (if change needed) +- Assessment of existing description +- Specific additions needed (e.g., "Add issue link", "Mention breaking change") +- Only full replacement if description is inadequate + +## Title Requirements + +**The title becomes the commit message headline.** Make it searchable and informative. + +| Requirement | Good | Bad | +|-------------|------|-----| +| Component prefix (if specific) | `[Cluster] Fix gossip protocol timeout` | `Fix timeout` | +| Describes behavior, not issue | `[RESP] ZADD: Support GT/LT flags` | `Fix #123` | +| Captures the "what" | `[Tsavorite] Reduce lock contention in RMW` | `Fix perf bug` | +| Notes breaking change if applicable | `(breaking)` | (omitted) | +| No noise prefixes | `[Storage] Fix...` | `[PR agent] Fix...` | + +### Title Formula + +``` +[Component] What changed (breaking if applicable) +``` + +Component prefixes (use when change is scoped): +- `[RESP]` — RESP command parsing/dispatch (`libs/server/Resp/`) +- `[Storage]` — Storage session/functions (`libs/server/Storage/`) +- `[Tsavorite]` — Tsavorite engine (`libs/storage/Tsavorite/`) +- `[Cluster]` — Cluster/replication/sharding (`libs/cluster/`) +- `[Objects]` — Object types: Hash, List, Set, SortedSet (`libs/server/Objects/`) +- `[API]` — Garnet API surface (`libs/server/API/`) +- `[Network]` — Networking/TLS (`libs/common/Networking/`) +- `[Config]` — Configuration/options (`libs/host/Configuration/`) +- `[Tests]` — Test-only changes +- `[Docs]` — Documentation-only changes +- Omit prefix for cross-cutting changes + +Examples: +- `[RESP] ZADD: Support GT/LT flags for conditional updates` +- `[Tsavorite] Reduce epoch protection overhead in hot-path RMW` +- `[Cluster] Fix replication lag during key migration` +- `Add multi-database support for standalone mode` + +## Description Requirements + +PR description should: +1. Link to the GitHub Issue +2. Describe what changed and why +3. Match the actual implementation + +```markdown +### Description of Change + +[Must match actual implementation] + +### Issues Fixed + +Fixes #XXXXX +``` + +## Content for Future Agents + +**The title and description become the commit message.** Future agents searching git history will use this to understand: +- What changed and why +- What patterns to follow or avoid +- How this change affects related code + +### Required Elements for Agent Success + +| Element | Purpose | Example | +|---------|---------|---------| +| Component in title | Scoped search | `[Tsavorite] ...` | +| Root cause (bug fixes) | Understand failure mode | "Epoch was not released on error path" | +| Description of change | What code does now | "Added GT/LT flag parsing in ZADD handler" | +| Key types/interfaces | API surface awareness | `IGarnetApi`, `StorageSession`, `CustomRawStringFunctions` | +| What NOT to do | Prevent repeat mistakes | "Don't allocate on RMW hot path" | + +### Recommended Elements + +| Element | When to Include | +|---------|----------------| +| **Root cause** | Bug fixes — explain why the bug occurred | +| **Key technical details** | Complex changes — list affected types and interfaces | +| **What NOT to do** | When failed approaches were attempted | +| **Edge cases** | When behavior differs across scenarios | +| **Performance impact** | When change affects hot paths or memory allocation | +| **Breaking changes** | When API or behavior changes affect consumers | +| **Migration guide** | When users/extensions need to update | + +## Description Template (for Inadequate Descriptions) + +Use this only when the existing description is stale, inaccurate, or missing key information: + +```markdown +### Root Cause + +[Why the bug occurred — be specific about the code path] + +### Description of Change + +[What the code now does] + +**Key changes:** +- [Change 1] +- [Change 2] + +### Key Technical Details + +**Affected types/interfaces:** +- `TypeA` — [What it does] +- `TypeB` — [What it does] + +### What NOT to Do (for future agents) + +- ❌ **Don't [approach 1]** — [Why it fails] +- ❌ **Don't [approach 2]** — [Why it's wrong] + +### Edge Cases + +| Scenario | Risk | Mitigation | +|----------|------|------------| +| [Case 1] | Low/Medium/High | [How to handle] | + +### Issues Fixed + +Fixes #XXXXX +``` + +## Quality Comparison Examples + +### Good Existing Description (KEEP) + +```markdown +## Changes + +### `libs/server/Resp/Objects/SortedSetCommands.cs` +- Added GT/LT flag parsing in ZADD command handler +- Flag validation against NX (mutually exclusive) + +### `libs/server/Objects/SortedSet/SortedSetObjectImpl.cs` +- Implemented conditional update logic in SortedSetAdd +- GT: only update if new score > current; LT: only if new score < current + +### `libs/server/Storage/Session/ObjectStore/SortedSetOps.cs` +- Passed flags through ObjectInput to the object implementation + +## Tests Added +- `RespSortedSetTests.ZAddWithGTFlag` — verifies GT-only updates +- `RespSortedSetTests.ZAddWithLTFlag` — verifies LT-only updates +- `RespSortedSetTests.ZAddGTNXMutuallyExclusive` — verifies error on GT+NX +``` + +**Verdict:** Excellent — file-by-file breakdown, specific changes, tests listed. Keep it. + +### Poor Existing Description (REWRITE) + +```markdown +Fixed the issue mentioned in #456 +``` + +**Verdict:** Inadequate — no detail on what changed. Use template. + +--- + +## Phase 2: Code Review + +After verifying title/description, perform a **code review** to catch Garnet-specific issues and general best practice violations before merge. + +### Review Focus Areas + +When reviewing code changes in Garnet, focus on: + +1. **Performance and memory safety** + - Unnecessary heap allocations on hot paths (prefer `Span`, `SpanByte`, stack allocation) + - Missing `[MethodImpl(MethodImplOptions.AggressiveInlining)]` on hot-path methods + - Missing `[MethodImpl(MethodImplOptions.NoInlining)]` on cold/exception-throwing methods + - Blocking calls or unnecessary copies in RESP command handlers + +2. **Epoch management** + - `LightEpoch` acquired but not released on error paths + - Epoch ownership — only dispose if owned + - Shared epochs in parallel test scenarios + +3. **RESP protocol correctness** + - Argument parsing via `parseState.GetArgSliceByRef(i)` returning `ref PinnedSpanByte` + - Correct RESP response format (using `RespWriteUtils` helpers) + - Proper `SendAndReset()` calls to flush response buffer + - Command dispatch wired in `ProcessBasicCommands`/`ProcessArrayCommands` + +4. **Thread safety and concurrency** + - Proper lock usage (`TryWriteLock()` in spin loops, not `CloseLock()`) + - Safe concurrent access to shared state + - Session-local vs shared state boundaries + +5. **Test quality** + - `TestBase` inheritance on test fixtures + - `TestUtils.OnTearDown()` called in `[TearDown]` (checks for leaked epochs) + - `TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true)` in `[SetUp]` + - Both `StackExchange.Redis` and `LightClient` coverage where applicable + +6. **Code conventions** + - File header: `// Copyright (c) Microsoft Corporation.` / `// Licensed under the MIT license.` + - `TreatWarningsAsErrors` — no new warnings introduced + - XML doc comments on public methods + - Consistent naming (camelCase private fields, PascalCase constants/statics) + +7. **Breaking changes and API surface** + - Changes to `IGarnetApi` / `IGarnetReadApi` / `IGarnetAdvancedApi` + - Changes to custom extension base classes (`CustomRawStringFunctions`, `CustomObjectBase`, etc.) + - Configuration option changes in `GarnetServerOptions` + +### How to Review + +```bash +# Get the PR diff +gh pr diff XXXXX + +# Review specific files +gh pr diff XXXXX -- path/to/file.cs + +# Check CI status +gh pr view XXXXX --json statusCheckRollup +``` + +### Output Format + +```markdown +## Code Review Findings + +### 🔴 Critical Issues + +**[Issue Title]** +- **File:** [path/to/file.cs] +- **Problem:** [Description] +- **Recommendation:** [Code fix or approach] + +### 🟡 Suggestions + +- [Suggestion 1] +- [Suggestion 2] + +### ✅ Looks Good + +- [Positive observation 1] +- [Positive observation 2] +``` + +### 🚨 CRITICAL: Do NOT Post Comments Directly + +**The pr-finalize skill is ANALYSIS ONLY.** Never post comments using `gh pr review` or `gh pr comment`. + +| Action | Allowed? | Why | +|--------|----------|-----| +| `gh pr review --comment` | ❌ **NEVER** | Present findings to the user instead | +| `gh pr comment` | ❌ **NEVER** | Present findings to the user instead | +| Analyze and report findings | ✅ **YES** | This is the skill's purpose | + +**Workflow:** +1. **This skill**: Analyze PR, produce findings in your response +2. **User asks to post**: User decides whether and how to share feedback + +The user controls when comments are posted. Your job is to analyze and present findings. + +--- + +## Complete Example + +See [references/complete-example.md](references/complete-example.md) for a full agent-optimized PR description showing all elements above applied to a real Garnet change. diff --git a/.github/skills/pr-finalize/references/complete-example.md b/.github/skills/pr-finalize/references/complete-example.md new file mode 100644 index 00000000000..10c5b48f8ff --- /dev/null +++ b/.github/skills/pr-finalize/references/complete-example.md @@ -0,0 +1,71 @@ +# Complete Example: Agent-Optimized PR + +This example shows a PR description optimized for future agent success in the Garnet codebase. + +## Title + +``` +[RESP] ZADD: Support GT/LT flags for conditional score updates +``` + +## Description + +```markdown +### Root Cause + +The ZADD command handler in `SortedSetCommands.cs` only supported the `NX`, `XX`, and `CH` flags. The Redis-compatible `GT` (only update if new score is greater) and `LT` (only update if new score is less) flags were missing, causing compatibility failures with clients that rely on these flags for leaderboard and rate-limiting patterns. + +### Description of Change + +Added `GT` and `LT` flag parsing and conditional update logic to the ZADD command pipeline. + +**Key changes:** +- Parse `GT`/`LT` flags in `SortedSetCommands.cs` alongside existing `NX`/`XX`/`CH` flags +- Added validation: `GT` and `LT` are mutually exclusive with `NX` (returns error per Redis spec) +- Passed flags through `ObjectInput` to `SortedSetObjectImpl.SortedSetAdd()` +- Implemented conditional score comparison before update in the sorted set object + +### Key Technical Details + +**Affected types:** +- `RespServerSession` (partial in `SortedSetCommands.cs`) — flag parsing from RESP buffer +- `SortedSetObject` (`SortedSetObjectImpl.cs`) — conditional update logic +- `StorageSession` (`SortedSetOps.cs`) — flag propagation via `ObjectInput` + +**Flag interaction matrix:** + +| Flags | Behavior | +|-------|----------| +| `GT` alone | Update only if new score > current score | +| `LT` alone | Update only if new score < current score | +| `GT XX` | Update existing members only if new score > current | +| `GT NX` | ❌ Error — mutually exclusive | +| `LT NX` | ❌ Error — mutually exclusive | + +### What NOT to Do (for future agents) + +- ❌ **Don't allocate strings for flag comparison** — Use `SpanByte` comparison against known byte sequences; the RESP hot path must stay allocation-free +- ❌ **Don't modify `SortedSetOperation` enum for flags** — Flags modify behavior of existing operations, they are not separate operations. Pass via `ObjectInput` fields instead +- ❌ **Don't skip the NX mutual exclusion check** — Redis returns an error for `GT NX` and `LT NX` combinations; clients depend on this + +### Edge Cases + +| Scenario | Risk | Mitigation | +|----------|------|------------| +| `GT` with equal scores | Low | Strict `>` comparison, no update on equal | +| `LT` with `NaN` score | Low | `NaN` comparison returns false, no update | +| Mixed `GT CH` counting | Medium | `CH` counts "changed" — includes not-updated members; tested explicitly | +| Cluster mode with key migration | Low | Flags are per-command, no cross-node impact | + +### Issues Fixed + +Fixes #456 +``` + +## Why This Works for Agents + +- **Searchable title** — Agents searching "ZADD GT LT" or "conditional score" will find this +- **Flag interaction matrix** — Agents know exactly how flags combine +- **What NOT to do** — Agents won't allocate on the hot path or misuse the operation enum +- **Edge cases** — Agents know the risk profile for NaN, equal scores, and CH interaction +- **Affected types listed** — Agents know which files to modify for similar flag additions diff --git a/.github/workflows/ci-bdnbenchmark.yml b/.github/workflows/ci-bdnbenchmark.yml index 5524a3e758d..0c16a859669 100644 --- a/.github/workflows/ci-bdnbenchmark.yml +++ b/.github/workflows/ci-bdnbenchmark.yml @@ -36,6 +36,9 @@ jobs: name: BDNBenchmark needs: [changes] runs-on: ${{ matrix.os }} + concurrency: + group: bdn-benchmark-${{ matrix.gh-pages-branch }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + cancel-in-progress: false strategy: fail-fast: false matrix: @@ -51,8 +54,10 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 + - name: Setup .NET uses: actions/setup-dotnet@v4 + - name: Install dependencies run: dotnet restore @@ -60,28 +65,19 @@ jobs: run: ./test/BDNPerfTests/run_bdnperftest.ps1 ${{ matrix.test }} ${{ matrix.framework }} shell: pwsh continue-on-error: false - - - name: Random pause between tasks so multiple inserts don't run at the same time - run: | - $delay = Get-Random -Minimum 1 -Maximum 600 - Write-Host "Sleeping for $delay seconds..." - Start-Sleep -Seconds $delay - shell: pwsh - name: Upload test results to artifacts uses: actions/upload-artifact@v4 with: name: Results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} - path: | - ./test/BDNPerfTests/results + path: ./test/BDNPerfTests/results if: ${{ always() }} - name: Upload Error Log to artifacts uses: actions/upload-artifact@v4 with: name: ErrorLog-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} - path: | - ./test/BDNPerfTests/errorlog + path: ./test/BDNPerfTests/errorlog if: ${{ always() }} # Run `github-action-benchmark` action for the Continuous Benchmarking Charts (https://microsoft.github.io/garnet/charts/) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16aceef2f21..d95425d8b11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,12 @@ jobs: uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- - name: Install dependencies run: dotnet restore Garnet.slnx - name: Check style format @@ -68,97 +74,147 @@ jobs: uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- - name: Install dependencies run: dotnet restore libs/storage/Tsavorite/cs/Tsavorite.slnx - name: Check style format run: dotnet format libs/storage/Tsavorite/cs/Tsavorite.slnx --no-restore --verify-no-changes --verbosity diagnostic - # Job to build and test Garnet code - build-test-garnet: - name: Garnet - needs: [changes, format-garnet] + # Job to build Garnet code (once per os/configuration) + build-garnet: + name: Build Garnet + needs: [changes] runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ ubuntu-latest, windows-latest ] - framework: [ 'net8.0' , 'net10.0'] configuration: [ 'Debug', 'Release' ] - test: [ 'Garnet.test', 'Garnet.test.cluster' ] if: needs.changes.outputs.garnet == 'true' steps: - name: Check out code uses: actions/checkout@v4 - name: Setup .NET uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- - name: Install dependencies run: dotnet restore - name: Build Garnet - run: dotnet build --configuration ${{ matrix.configuration }} - - - name: Verify Allure wiring (${{ matrix.test }}) - shell: pwsh - run: | - $asmPath = "${{ github.workspace }}/test/${{ matrix.test }}/bin/${{ matrix.configuration }}/${{ matrix.framework }}/${{ matrix.test }}.dll" - $asm = [System.Reflection.Assembly]::LoadFrom($asmPath) + run: dotnet build --configuration ${{ matrix.configuration }} --no-restore - # When using .Net 9.0/10.0 it tries to resolve every referenced type in that assembly and some dependencies are missing by design so pops exception. - # Since we don't want to fail on every 9.0/10.0 run (even when has Allure inheritance), we catch it here. - try { - $types = $asm.GetTypes() - } catch [System.Reflection.ReflectionTypeLoadException] { - # Keep only successfully loaded types, skip nulls - $types = $_.Exception.Types | Where-Object { $_ -ne $null } - } - $allureBase = $types | Where-Object { $_.Name -eq "AllureTestBase" } - - $bad = @() - foreach ($t in $types) { - # Detect NUnit test fixtures by attribute names - $isFixture = $t.GetCustomAttributes($true) | - Where-Object { $_.GetType().FullName -eq "NUnit.Framework.TestFixtureAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - if ($isFixture -eq 0) { - $isFixture = $t.GetMethods() | - ForEach-Object { $_.GetCustomAttributes($true) } | - Where-Object { $_.GetType().FullName -eq "NUnit.Framework.TestAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - } - - if ($isFixture -gt 0) { - $inheritsAllure = $allureBase -and $allureBase.IsAssignableFrom($t) - $hasAttr = $t.GetCustomAttributes($true) | - Where-Object { $_.GetType().FullName -eq "Allure.NUnit.AllureNUnitAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - - if (-not $inheritsAllure -or $hasAttr -eq 0) { - $bad += $t.FullName - } - } - } + # Job to test Garnet standalone code + test-garnet-standalone: + name: Garnet Standalone + needs: [changes, build-garnet] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] + framework: [ 'net8.0' , 'net10.0'] + configuration: [ 'Debug', 'Release' ] + test: [ 'Garnet.test', 'Garnet.test.collections', 'Garnet.test.acl', 'Garnet.test.scripting', 'Garnet.test.complexstring', 'Garnet.test.vectorset', 'Garnet.test.rangeindex', 'Garnet.test.extensions' ] + if: needs.changes.outputs.garnet == 'true' + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- + - name: Install dependencies + run: dotnet restore + - name: Run tests ${{ matrix.test }} + run: dotnet test test/standalone/${{ matrix.test }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "GarnetTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}" -- NUnit.DisplayName=FullName + timeout-minutes: 45 + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: dotnet-standalone-results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + path: GarnetTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + if: ${{ always() }} - if ($bad.Count -gt 0) { - Write-Host "These test fixtures are missing Allure setup from $asmPath. See Pull Request Protocol for details on how to set up Allure for new tests. https://microsoft.github.io/garnet/docs/dev/onboarding" - $bad | ForEach-Object { Write-Host $_ } - exit 1 - } else { - Write-Host "All test fixtures wired to Allure in $asmPath" - } - + # Job to test Garnet cluster code + test-garnet-cluster: + name: Garnet Cluster + needs: [changes, build-garnet] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] + framework: [ 'net8.0' , 'net10.0'] + configuration: [ 'Debug', 'Release' ] + test: [ 'Garnet.test.cluster', 'Garnet.test.cluster.migrate', 'Garnet.test.cluster.replication', 'Garnet.test.cluster.replication.tls', 'Garnet.test.cluster.replication.disklesssync', 'Garnet.test.cluster.vectorsets', 'Garnet.test.cluster.multilog' ] + if: needs.changes.outputs.garnet == 'true' + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-${{ runner.os }}-${{ hashFiles('**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-${{ runner.os }}- + - name: Install dependencies + run: dotnet restore - name: Run tests ${{ matrix.test }} - run: dotnet test test/${{ matrix.test }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "GarnetTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}" -- NUnit.DisplayName=FullName + run: dotnet test test/cluster/${{ matrix.test }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "GarnetTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}" -- NUnit.DisplayName=FullName timeout-minutes: 45 - name: Upload test results uses: actions/upload-artifact@v4 with: - name: dotnet-garnet-results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + name: dotnet-cluster-results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} path: GarnetTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} if: ${{ always() }} - # Job to build and test Tsavorite code (only if there were changes to it) - build-test-tsavorite: + # Job to build Tsavorite code (once per os/configuration) + build-tsavorite: + name: Build Tsavorite + needs: [changes] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest, windows-latest ] + configuration: [ 'Debug', 'Release' ] + if: needs.changes.outputs.tsavorite == 'true' + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-tsavorite-${{ runner.os }}-${{ hashFiles('libs/storage/Tsavorite/**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-tsavorite-${{ runner.os }}- + - name: Install dependencies + run: dotnet restore libs/storage/Tsavorite/cs/Tsavorite.slnx + - name: Build Tsavorite + run: dotnet build libs/storage/Tsavorite/cs/Tsavorite.slnx --configuration ${{ matrix.configuration }} --no-restore + + # Job to test Tsavorite code + test-tsavorite: name: Tsavorite - needs: [changes, format-tsavorite] + needs: [changes, build-tsavorite] runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -166,87 +222,67 @@ jobs: os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0', 'net10.0' ] configuration: [ 'Debug', 'Release' ] - if: needs.changes.outputs.tsavorite == 'true' + test: [ 'Tsavorite.test', 'Tsavorite.test.recordops', 'Tsavorite.test.session', 'Tsavorite.test.session.context', 'Tsavorite.test.hlog', 'Tsavorite.test.recovery' ] + if: needs.changes.outputs.tsavorite == 'true' steps: - name: Check out code uses: actions/checkout@v4 - name: Set environment variable for Linux run: echo "RunAzureTests=yes" >> $GITHUB_ENV - if: ${{ matrix.os == 'ubuntu-latest' }} + if: ${{ matrix.os == 'ubuntu-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }} - name: Set environment variable for Windows run: echo ("RunAzureTests=yes") >> $env:GITHUB_ENV - if: ${{ matrix.os == 'windows-latest' }} + if: ${{ matrix.os == 'windows-latest' && matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }} - name: Setup .NET uses: actions/setup-dotnet@v4 - name: Setup Node.js for Azurite + if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }} uses: actions/setup-node@v4 with: node-version: 22 + - name: Cache Azurite + if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }} + uses: actions/cache@v4 + with: + path: ${{ runner.os == 'Windows' && '%APPDATA%\npm-cache' || '~/.npm' }} + key: azurite-${{ runner.os }} - name: Install and Run Azurite + if: ${{ matrix.test != 'Tsavorite.test.recordops' && matrix.test != 'Tsavorite.test.session' && matrix.test != 'Tsavorite.test.session.context' && matrix.test != 'Tsavorite.test.recovery' }} shell: bash run: | npm install -g azurite azurite --skipApiVersionCheck & + - name: Cache NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: nuget-tsavorite-${{ runner.os }}-${{ hashFiles('libs/storage/Tsavorite/**/*.csproj', 'Directory.Packages.props') }} + restore-keys: nuget-tsavorite-${{ runner.os }}- - name: Install dependencies - run: dotnet restore - - name: Build Tsavorite - run: dotnet build libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj --configuration ${{ matrix.configuration }} + run: dotnet restore libs/storage/Tsavorite/cs/Tsavorite.slnx - - name: Verify Allure wiring (Tsavorite) + - name: Resolve Tsavorite test directory shell: pwsh run: | - $asmPath = "${{ github.workspace }}/libs/storage/Tsavorite/cs/test/bin/${{ matrix.configuration }}/${{ matrix.framework }}/Tsavorite.test.dll" - $asm = [System.Reflection.Assembly]::LoadFrom($asmPath) - - # when using .Net 9.0/10.0 it tries to resolve every referenced type in that assembly and some dependencies are missing by design so pops exception (we don't need those anyways) - try { - $types = $asm.GetTypes() - } catch [System.Reflection.ReflectionTypeLoadException] { - $types = $_.Exception.Types | Where-Object { $_ -ne $null } + $tsavoriteDirMap = @{ + 'Tsavorite.test' = 'libs/storage/Tsavorite/cs/test' + 'Tsavorite.test.recordops' = 'libs/storage/Tsavorite/cs/test/test.recordops' + 'Tsavorite.test.session' = 'libs/storage/Tsavorite/cs/test/test.session' + 'Tsavorite.test.session.context' = 'libs/storage/Tsavorite/cs/test/test.session.context' + 'Tsavorite.test.hlog' = 'libs/storage/Tsavorite/cs/test/test.hlog' + 'Tsavorite.test.recovery' = 'libs/storage/Tsavorite/cs/test/test.recovery' } - $allureBase = $types | Where-Object { $_.Name -eq "AllureTestBase" } - - $bad = @() - foreach ($t in $types) { - # Detect NUnit test fixtures by attribute names - $isFixture = $t.GetCustomAttributes($true) | - Where-Object { $_.GetType().FullName -eq "NUnit.Framework.TestFixtureAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - if ($isFixture -eq 0) { - $isFixture = $t.GetMethods() | - ForEach-Object { $_.GetCustomAttributes($true) } | - Where-Object { $_.GetType().FullName -eq "NUnit.Framework.TestAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - } + $dir = $tsavoriteDirMap['${{ matrix.test }}'] + echo "TSAVORITE_TEST_DIR=$dir" >> $env:GITHUB_ENV - if ($isFixture -gt 0) { - $inheritsAllure = $allureBase -and $allureBase.IsAssignableFrom($t) - $hasAttr = $t.GetCustomAttributes($true) | - Where-Object { $_.GetType().FullName -eq "Allure.NUnit.AllureNUnitAttribute" } | - Measure-Object | Select-Object -ExpandProperty Count - - if (-not $inheritsAllure -or $hasAttr -eq 0) { - $bad += $t.FullName - } - } - } - - if ($bad.Count -gt 0) { - Write-Host "These test fixtures are missing Allure setup from $asmPath" - $bad | ForEach-Object { Write-Host $_ } - exit 1 - } else { - Write-Host "All test fixtures wired to Allure in $asmPath" - } - - - name: Run Tsavorite tests - run: dotnet test libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "TsavoriteTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}" - timeout-minutes: 45 + - name: Run tests ${{ matrix.test }} + run: dotnet test ${{ env.TSAVORITE_TEST_DIR }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "TsavoriteTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }}" -- NUnit.DisplayName=FullName + timeout-minutes: 45 - name: Upload test results uses: actions/upload-artifact@v4 with: - name: dotnet-tsavorite-results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} - path: TsavoriteTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} + name: dotnet-tsavorite-results-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} + path: TsavoriteTestResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} if: ${{ always() }} build-website: @@ -269,10 +305,150 @@ jobs: - name: Build website run: yarn build + # Job to generate combined test summaries per category + test-summary: + name: Test Summary + runs-on: ubuntu-latest + needs: [changes, test-garnet-standalone, test-garnet-cluster, test-tsavorite] + if: always() && (needs.test-garnet-standalone.result != 'skipped' || needs.test-garnet-cluster.result != 'skipped' || needs.test-tsavorite.result != 'skipped') + steps: + - name: Download standalone test results + if: needs.test-garnet-standalone.result != 'skipped' + uses: actions/download-artifact@v4 + with: + pattern: dotnet-standalone-results-* + path: results/standalone + merge-multiple: true + - name: Download cluster test results + if: needs.test-garnet-cluster.result != 'skipped' + uses: actions/download-artifact@v4 + with: + pattern: dotnet-cluster-results-* + path: results/cluster + merge-multiple: true + - name: Download Tsavorite test results + if: needs.test-tsavorite.result != 'skipped' + uses: actions/download-artifact@v4 + with: + pattern: dotnet-tsavorite-results-* + path: results/tsavorite + merge-multiple: true + - name: Generate summaries + shell: pwsh + run: | + function Write-CategorySummary($category, $dir) { + $trxFiles = Get-ChildItem -Path $dir -Filter "*.trx" -Recurse -ErrorAction SilentlyContinue + if (-not $trxFiles -or $trxFiles.Count -eq 0) { return } + $totalTests = 0; $passed = 0; $failed = 0; $skipped = 0 + $durations = @() + $failedTests = @() + foreach ($trx in $trxFiles) { + $xml = [System.Xml.XmlDocument]::new() + try { $xml.Load($trx.FullName) } + catch { + Write-Warning "Skipping malformed TRX file: $($trx.FullName) - $_" + continue + } + $ns = @{ t = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010" } + $results = Select-Xml -Xml $xml -XPath "//t:UnitTestResult" -Namespace $ns + foreach ($r in $results) { + $totalTests++ + $outcome = $r.Node.outcome + if ($outcome -eq "Passed") { $passed++ } + elseif ($outcome -eq "Failed") { + $failed++ + $dur = $r.Node.duration + $durSec = 0 + if ($dur) { try { $durSec = [TimeSpan]::Parse($dur).TotalSeconds } catch { } } + $failedTests += @{ Name = $r.Node.testName; Seconds = $durSec } + } + else { $skipped++ } + $dur = $r.Node.duration + if ($dur) { + try { $ts = [TimeSpan]::Parse($dur); $durations += @{ Name = $r.Node.testName; Seconds = $ts.TotalSeconds } } + catch { } + } + } + } + # Compute total duration as sum of all individual test durations + $totalSec = ($durations | ForEach-Object { $_.Seconds } | Measure-Object -Sum).Sum + if ($totalSec -gt 0) { + $totalDurStr = "{0}m {1:D2}s" -f [int][math]::Floor($totalSec / 60), [int]($totalSec % 60) + } else { $totalDurStr = "N/A" } + $buckets = @( + @{ Label = "< 1s"; Min = 0; Max = 1 }, + @{ Label = "1s - 5s"; Min = 1; Max = 5 }, + @{ Label = "5s - 15s"; Min = 5; Max = 15 }, + @{ Label = "15s - 30s"; Min = 15; Max = 30 }, + @{ Label = "30s - 60s"; Min = 30; Max = 60 }, + @{ Label = "> 60s"; Min = 60; Max = [double]::MaxValue } + ) + $counts = @(0,0,0,0,0,0) + foreach ($d in $durations) { + for ($i = 0; $i -lt $buckets.Count; $i++) { + if ($d.Seconds -ge $buckets[$i].Min -and $d.Seconds -lt $buckets[$i].Max) { $counts[$i]++; break } + } + } + $maxCount = ($counts | Measure-Object -Maximum).Maximum + if ($maxCount -eq 0) { $maxCount = 1 } + $sb = [System.Text.StringBuilder]::new() + [void]$sb.AppendLine("## 🧪 $category") + [void]$sb.AppendLine("") + [void]$sb.AppendLine("⏱️ **Total: $totalDurStr** · $totalTests tests (✅ $passed · ❌ $failed · ⏭️ $skipped)") + [void]$sb.AppendLine("") + # Failed tests section (only if there are failures) + if ($failedTests.Count -gt 0) { + [void]$sb.AppendLine("### ❌ Failed Tests") + [void]$sb.AppendLine("") + [void]$sb.AppendLine("| Test | Duration |") + [void]$sb.AppendLine("|------|----------|") + foreach ($t in $failedTests) { + $durStr = "{0:F1}s" -f $t.Seconds + [void]$sb.AppendLine("| $($t.Name) | $durStr |") + } + [void]$sb.AppendLine("") + } + [void]$sb.AppendLine("### Distribution") + [void]$sb.AppendLine("") + [void]$sb.AppendLine("| Duration | Count | |") + [void]$sb.AppendLine("|----------|------:|--|") + for ($i = 0; $i -lt $buckets.Count; $i++) { + $barLen = [int][math]::Round(($counts[$i] / $maxCount) * 20) + if ($barLen -eq 0 -and $counts[$i] -gt 0) { $barLen = 1 } + $bar = "█" * $barLen + [void]$sb.AppendLine("| $($buckets[$i].Label) | $($counts[$i]) | $bar |") + } + $top10 = $durations | Sort-Object { $_.Seconds } -Descending | Select-Object -First 10 + if ($top10.Count -gt 0) { + [void]$sb.AppendLine("") + [void]$sb.AppendLine("### 🐢 Top 10 Slowest") + [void]$sb.AppendLine("") + [void]$sb.AppendLine("| Test | Duration |") + [void]$sb.AppendLine("|------|----------|") + foreach ($t in $top10) { + $durStr = "{0:F1}s" -f $t.Seconds + [void]$sb.AppendLine("| $($t.Name) | $durStr |") + } + } + [void]$sb.AppendLine("") + $sb.ToString() >> $env:GITHUB_STEP_SUMMARY + } + + # Generate per-category summaries + if (Test-Path "results/standalone") { + Write-CategorySummary "Garnet Standalone" "results/standalone" + } + if (Test-Path "results/cluster") { + Write-CategorySummary "Garnet Cluster" "results/cluster" + } + if (Test-Path "results/tsavorite") { + Write-CategorySummary "Tsavorite" "results/tsavorite" + } + pipeline-success: name: Garnet CI (Complete) runs-on: ubuntu-latest - needs: [ build-test-garnet, build-test-tsavorite, build-website ] + needs: [changes, format-garnet, format-tsavorite, build-garnet, test-garnet-standalone, test-garnet-cluster, build-tsavorite, test-tsavorite, build-website ] steps: - run: echo Done! if: ${{ !(failure() || cancelled()) }} diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 2f2033c2efa..ed2780dc708 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -42,17 +42,6 @@ jobs: sparse-checkout: | website/static/charts path: continuousbenchmark_net80 - - uses: actions/checkout@v4 - with: - ref: allure_data_history - path: allure_data_history - - # Checkout gh-pages to preserve existing Allure content when not updating from Nightly - - uses: actions/checkout@v4 - with: - ref: gh-pages - path: gh-pages-current - - name: Copy charts run: | mkdir -p static/charts @@ -62,52 +51,6 @@ jobs: - name: DEBUG Show triggering workflow name run: echo ${{ github.event.workflow_run.name }} - # When NOT triggered by Nightly, preserve existing Allure content from gh-pages - - name: Preserve existing Allure from gh-pages - if: ${{ github.event.workflow_run.name != 'Garnet Nightly Tests' }} - run: | - if [ -d "../gh-pages-current/allure" ]; then - mkdir -p static/allure - cp -R ../gh-pages-current/allure/* static/allure/ - echo "Preserved existing Allure content from gh-pages" - else - echo "No existing Allure content found on gh-pages" - fi - - # Download the current allure_report artifact which is big with A LOT of test result files - # Nightly run does not put full allure_report up in the allure_data_history branch to keep that branch small - - name: Download Allure artifact - if: ${{ github.event.workflow_run.name == 'Garnet Nightly Tests' }} - uses: actions/download-artifact@v4 - with: - name: allure-report - path: allure_artifact - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ github.token }} - # The one thing that is pulled from allure_data_history branch is the historical test result data - # This data is then copied into the downloaded allure_report to create a full report with history - - name: Copy Allure report - if: ${{ github.event.workflow_run.name == 'Garnet Nightly Tests' }} - run: | - mkdir -p static/allure - - # Base: Allure full report from Nightly Run artifact - # Note: artifact downloads to workspace root, not working-directory, so use ../ - # The artifact contents are directly in allure_artifact/ (not in a nested allure-report folder) - if [ -d "../allure_artifact" ]; then - cp -R ../allure_artifact/* static/allure - echo "Copied Allure report from artifact" - else - echo "Allure artifact missing; skipping base report copy." - fi - - # Overlay: history from branch - if [ -d "../allure_data_history/test/Allure/history" ]; then - cp -R ../allure_data_history/test/Allure/history static/allure/history - else - echo "Allure history missing; skipping history overlay." - fi - - uses: actions/setup-node@v4 with: node-version: 22 @@ -129,7 +72,7 @@ jobs: publish_dir: ./website/build # Deploy as an orphan commit (single commit, no history) to prevent # the gh-pages branch from accumulating large generated files (charts, - # allure reports, search index) across hundreds of commits, which + # search index) across hundreds of commits, which # bloats the repository pack file and slows down git clone. force_orphan: true # The following lines assign commit authorship to the official diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e47bf03cebc..4f40b27b31f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -12,94 +12,31 @@ permissions: contents: write jobs: - build-test-all: - name: Test + # Garnet standalone tests + test-garnet-standalone: + name: Garnet Standalone runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - test: [ 'Garnet.test', 'Garnet.test.cluster', 'Tsavorite.test' ] + test: [ 'Garnet.test', 'Garnet.test.collections', 'Garnet.test.acl', 'Garnet.test.scripting', 'Garnet.test.complexstring', 'Garnet.test.vectorset', 'Garnet.test.rangeindex', 'Garnet.test.extensions' ] os: [ ubuntu-latest, windows-latest ] framework: [ 'net8.0', 'net10.0' ] configuration: [ 'Debug', 'Release' ] - steps: - - name: Install Allure CLI - shell: pwsh - run: | - if ($env:RUNNER_OS -eq "Windows") { - Set-ExecutionPolicy RemoteSigned -Scope Process -Force - iwr -useb get.scoop.sh | iex - scoop install allure - $shimPath = "$env:USERPROFILE\scoop\shims" - echo "Adding Scoop shims to PATH: $shimPath" - echo "$shimPath" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - } elseif ($env:RUNNER_OS -eq "Linux") { - npm install -g allure-commandline --save-dev - } else { - Write-Host "Unsupported OS: $env:RUNNER_OS" - exit 1 - } - name: Check out code uses: actions/checkout@v4 - - - name: Set environment variable for Linux - run: echo "RunAzureTests=yes" >> $GITHUB_ENV - if: ${{ matrix.os == 'ubuntu-latest' }} - - - name: Set environment variable for Windows - run: echo ("RunAzureTests=yes") >> $env:GITHUB_ENV - if: ${{ matrix.os == 'windows-latest' }} - - name: Setup .NET uses: actions/setup-dotnet@v4 - - - name: Setup Node.js for Azurite - uses: actions/setup-node@v4 - with: - node-version: '22' - - - name: Install and Run Azurite - shell: bash - run: | - npm install -g azurite - azurite & - - name: Install dependencies run: dotnet restore - - name: Check style format run: dotnet format --verify-no-changes --verbosity diagnostic - - - name: Build Tsavorite.test - if: ${{ matrix.test == 'Tsavorite.test' }} - run: dotnet build libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj --configuration ${{ matrix.configuration }} - - name: Build Garnet - if: ${{ matrix.test != 'Tsavorite.test' }} - run: dotnet build --configuration ${{ matrix.configuration }} - + run: dotnet build --configuration ${{ matrix.configuration }} --no-restore - name: Run tests ${{ matrix.test }} - shell: pwsh - run: | - $resultsDir = "${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}" - if ("${{ matrix.test }}" -eq "Tsavorite.test") { - dotnet test "libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj" ` - -f "${{ matrix.framework }}" ` - --configuration "${{ matrix.configuration }}" ` - --logger "console;verbosity=detailed" ` - --logger "trx" ` - --results-directory "$resultsDir" - } else { - dotnet test "test/${{ matrix.test }}" ` - -f "${{ matrix.framework }}" ` - --configuration "${{ matrix.configuration }}" ` - --logger "console;verbosity=detailed" ` - --logger "trx" ` - --results-directory "$resultsDir" - } - timeout-minutes: 55 - + run: dotnet test test/standalone/${{ matrix.test }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}" -- NUnit.DisplayName=FullName + timeout-minutes: 55 - name: Upload test results uses: actions/upload-artifact@v4 with: @@ -107,185 +44,96 @@ jobs: path: ${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} if: ${{ always() }} - - name: Stage allure-results into test/Allure/AllResults - shell: pwsh - run: | - $base = $env:GITHUB_WORKSPACE - - if ("${{ matrix.test }}" -eq "Tsavorite.test") { - $source = Join-Path $base "libs/storage/Tsavorite/cs/test/bin/${{ matrix.configuration }}/${{ matrix.framework }}/allure-results" - } else { - $source = Join-Path $base "test/${{ matrix.test }}/bin/${{ matrix.configuration }}/${{ matrix.framework }}/allure-results" - } - - $target = Join-Path $base "test/Allure/AllResults" - Write-Host "Staging allure-results from $source into $target" - - New-Item -ItemType Directory -Path $target -Force | Out-Null - - if (Test-Path $source) { - Get-ChildItem -Path $source -Recurse | Copy-Item -Destination $target -Force - Write-Host "Copied allure-results into $target" - } else { - Write-Host "Source path $source not found, skipping copy" - } - if: ${{ always() }} - - # NOTE - need this to get the results names properly categorized in Allure report at Suite level - problem is that it can hit a max limit in the system if do a bug test matrix - - name: Upload Allure Results to artifacts + # Garnet cluster tests + test-garnet-cluster: + name: Garnet Cluster + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + test: [ 'Garnet.test.cluster', 'Garnet.test.cluster.migrate', 'Garnet.test.cluster.replication', 'Garnet.test.cluster.replication.tls', 'Garnet.test.cluster.replication.asyncreplay', 'Garnet.test.cluster.replication.disklesssync', 'Garnet.test.cluster.vectorsets', 'Garnet.test.cluster.multilog' ] + os: [ ubuntu-latest, windows-latest ] + framework: [ 'net8.0', 'net10.0' ] + configuration: [ 'Debug', 'Release' ] + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Setup .NET + uses: actions/setup-dotnet@v4 + - name: Install dependencies + run: dotnet restore + - name: Build Garnet + run: dotnet build --configuration ${{ matrix.configuration }} --no-restore + - name: Run tests ${{ matrix.test }} + run: dotnet test test/cluster/${{ matrix.test }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}" -- NUnit.DisplayName=FullName + timeout-minutes: 55 + - name: Upload test results uses: actions/upload-artifact@v4 with: - name: AllureResults-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}-${{ matrix.test }} - path: test/Allure/AllResults - if: ${{ always() }} - - - name: Clean up raw allure-results folders after copying to Allure staging area - shell: pwsh - run: | - $base = $env:GITHUB_WORKSPACE - $paths = @( - "libs/storage/Tsavorite/cs/test/bin/${{ matrix.configuration }}/${{ matrix.framework }}/allure-results", - "test/${{ matrix.test }}/bin/${{ matrix.configuration }}/${{ matrix.framework }}/allure-results" - ) - foreach ($p in $paths) { - $full = Join-Path $base $p - if (Test-Path $full) { - Write-Host "Removing $full" - Remove-Item $full -Recurse -Force - } - } + name: ${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} + path: ${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} if: ${{ always() }} - generate-allure-report: - name: Generate Allure Report - runs-on: windows-latest # only used to generate report - makes it easy on how calls made so don't have to worry about windows and linux - needs: [ build-test-all ] - if: ${{ always() }} + # Tsavorite tests + test-tsavorite: + name: Tsavorite + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + test: [ 'Tsavorite.test', 'Tsavorite.test.recordops', 'Tsavorite.test.session', 'Tsavorite.test.session.context', 'Tsavorite.test.hlog', 'Tsavorite.test.recovery', 'Tsavorite.test.stress' ] + os: [ ubuntu-latest, windows-latest ] + framework: [ 'net8.0', 'net10.0' ] + configuration: [ 'Debug', 'Release' ] steps: - # First checkout the main repo into the default workspace root - - name: Checkout main repository - uses: actions/checkout@v4 - - # Then checkout the allure_data_history branch into a subfolder - - name: Checkout allure_data_history branch + - name: Check out code uses: actions/checkout@v4 + - name: Set environment variable for Linux + run: echo "RunAzureTests=yes" >> $GITHUB_ENV + if: ${{ matrix.os == 'ubuntu-latest' }} + - name: Set environment variable for Windows + run: echo ("RunAzureTests=yes") >> $env:GITHUB_ENV + if: ${{ matrix.os == 'windows-latest' }} + - name: Setup .NET + uses: actions/setup-dotnet@v4 + - name: Setup Node.js for Azurite + uses: actions/setup-node@v4 with: - ref: allure_data_history - path: history-branch - persist-credentials: true - - - name: Install Allure CLI - shell: pwsh - run: | - if ($env:RUNNER_OS -eq "Windows") { - Set-ExecutionPolicy RemoteSigned -Scope Process -Force - iwr -useb get.scoop.sh | iex - scoop install allure - $shimPath = "$env:USERPROFILE\scoop\shims" - echo "Adding Scoop shims to PATH: $shimPath" - echo "$shimPath" | Out-File -Append -Encoding utf8 $env:GITHUB_PATH - } elseif ($env:RUNNER_OS -eq "Linux") { - npm install -g allure-commandline --save-dev - } else { - Write-Host "Unsupported OS: $env:RUNNER_OS" - exit 1 - } - - - name: Download allure test results that were staged in artifacts and merge into one folder - uses: actions/download-artifact@v4 + node-version: '22' + - name: Cache Azurite + uses: actions/cache@v4 with: - path: test/Allure/AllResults - pattern: AllureResults-* - - - name: Copy all results into one folder (CombinedResults) - shell: pwsh + path: ${{ runner.os == 'Windows' && '%APPDATA%\npm-cache' || '~/.npm' }} + key: azurite-${{ runner.os }} + - name: Install and Run Azurite + shell: bash run: | - $source = Join-Path $env:GITHUB_WORKSPACE "test/Allure/AllResults" - $target = Join-Path $env:GITHUB_WORKSPACE "test/Allure/CombinedResults" - New-Item -ItemType Directory -Path $target -Force - Get-ChildItem -Path $source -Recurse -File | ForEach-Object { - Copy-Item -Path $_.FullName -Destination $target -Force - } - - - name: Final cleanup of stray allure-results folders + npm install -g azurite + azurite & + - name: Install dependencies + run: dotnet restore libs/storage/Tsavorite/cs/Tsavorite.slnx + - name: Build Tsavorite + run: dotnet build libs/storage/Tsavorite/cs/Tsavorite.slnx --configuration ${{ matrix.configuration }} --no-restore + - name: Resolve Tsavorite test directory shell: pwsh run: | - $base = $env:GITHUB_WORKSPACE - # Remove any allure-results directories anywhere under workspace - Get-ChildItem -Path $base -Recurse -Directory -Filter "allure-results" | - ForEach-Object { - Write-Host "Removing $($_.FullName)" - Remove-Item $_.FullName -Recurse -Force - } - - # Remove the staging folder once CombinedResults is ready - $staging = Join-Path $base "test/Allure/AllResults" - if (Test-Path $staging) { - Write-Host "Removing staging folder $staging" - Remove-Item $staging -Recurse -Force + $tsavoriteDirMap = @{ + 'Tsavorite.test' = 'libs/storage/Tsavorite/cs/test' + 'Tsavorite.test.recordops' = 'libs/storage/Tsavorite/cs/test/test.recordops' + 'Tsavorite.test.session' = 'libs/storage/Tsavorite/cs/test/test.session' + 'Tsavorite.test.session.context' = 'libs/storage/Tsavorite/cs/test/test.session.context' + 'Tsavorite.test.hlog' = 'libs/storage/Tsavorite/cs/test/test.hlog' + 'Tsavorite.test.recovery' = 'libs/storage/Tsavorite/cs/test/test.recovery' + 'Tsavorite.test.stress' = 'libs/storage/Tsavorite/cs/test/test.stress' } - - - name: Copy history from allure_data_history branch into CombinedResults - shell: pwsh - run: | - $source = "${{ github.workspace }}/history-branch/test/Allure/history" - $target = "${{ github.workspace }}/test/Allure/CombinedResults/history" - if (Test-Path $source) { - New-Item -ItemType Directory -Path $target -Force - Copy-Item -Path (Join-Path $source '*') -Destination $target -Recurse -Force - Write-Host "History copied from allure_data_history into CombinedResults" - } else { - Write-Host "No history found in allure_data_history, starting fresh" - } - - - name: Run GenerateAllureReport.ps1 - shell: pwsh - run: | - Set-Location "${{ github.workspace }}/test/Allure" - .\GenerateAllureReport.ps1 - - - name: Upload Allure history to artifacts so can use for debugging if needed - uses: actions/upload-artifact@v4 - with: - name: allure-history - path: test/Allure/allure-report/history - - - name: Commit updated history to allure_data_history - shell: pwsh - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - - # Use an orphan commit (no history) to prevent the allure_data_history - # branch from accumulating large history.json blobs across hundreds of - # commits, which bloats the repository pack file and slows git clone. - git checkout --orphan allure_data_history - git rm -rf . - - # The generated allure-report/history is untracked and survives git rm. - # Copy (not move) so the subsequent artifact upload and website publish - # steps still have the history directory in allure-report/. - Copy-Item -Path "${{ github.workspace }}/test/Allure/allure-report/history" -Destination "${{ github.workspace }}/test/Allure/history" -Recurse -Force - - git add test/Allure/history - git commit -m "Update Allure history [CI]" - git push origin allure_data_history --force - - - name: Upload Allure HTML report + $dir = $tsavoriteDirMap['${{ matrix.test }}'] + echo "TSAVORITE_TEST_DIR=$dir" >> $env:GITHUB_ENV + - name: Run tests ${{ matrix.test }} + run: dotnet test ${{ env.TSAVORITE_TEST_DIR }} -f ${{ matrix.framework }} --configuration ${{ matrix.configuration }} --logger "console;verbosity=detailed" --logger trx --results-directory "${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }}" + timeout-minutes: 55 + - name: Upload test results uses: actions/upload-artifact@v4 with: - name: allure-report - path: test/Allure/allure-report - - - name: Publish Allure report to GH Web site - shell: pwsh - run: | - $source = Join-Path $env:GITHUB_WORKSPACE "test/Allure/allure-report" - $target = Join-Path $env:GITHUB_WORKSPACE "website/static/allure" - Write-Host "Copying Allure report from $source to $target" - New-Item -ItemType Directory -Path $target -Force | Out-Null - Copy-Item -Path (Join-Path $source '*') -Destination $target -Recurse -Force - Write-Host "Allure report staged in website/static/allure" - - - name: Echo completion - run: echo Done! + name: ${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} + path: ${{ matrix.test }}-${{ matrix.os }}-${{ matrix.framework }}-${{ matrix.configuration }} + if: ${{ always() }} diff --git a/.gitignore b/.gitignore index 371d74f583a..a0649e807e2 100644 --- a/.gitignore +++ b/.gitignore @@ -218,5 +218,9 @@ test/tmp/ # BenchmarkDotNet Results BenchmarkDotNet.Artifacts/ +# PerfView Results +*PerfView*.* + # Helm chart artifacts .cr-release-packages/ +libs/native/bftree-garnet/target/ diff --git a/Directory.Packages.props b/Directory.Packages.props index b6059652e5d..e940f57c6bb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,8 +4,7 @@ true - - + @@ -32,7 +31,7 @@ - + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3d308082d1f..66302782bbe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/ COPY libs/common/*.csproj libs/common/ COPY libs/host/*.csproj libs/host/ COPY libs/server/*.csproj libs/server/ +COPY libs/native/bftree-garnet/*.csproj libs/native/bftree-garnet/ COPY libs/resources/*.csproj libs/resources/ COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/ COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/ diff --git a/Dockerfile.alpine b/Dockerfile.alpine index 628130215e0..31656a12975 100644 --- a/Dockerfile.alpine +++ b/Dockerfile.alpine @@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/ COPY libs/common/*.csproj libs/common/ COPY libs/host/*.csproj libs/host/ COPY libs/server/*.csproj libs/server/ +COPY libs/native/bftree-garnet/*.csproj libs/native/bftree-garnet/ COPY libs/resources/*.csproj libs/resources/ COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/ COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/ diff --git a/Dockerfile.azurelinux b/Dockerfile.azurelinux index c3d6ca4176c..bd9826ed192 100644 --- a/Dockerfile.azurelinux +++ b/Dockerfile.azurelinux @@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/ COPY libs/common/*.csproj libs/common/ COPY libs/host/*.csproj libs/host/ COPY libs/server/*.csproj libs/server/ +COPY libs/native/bftree-garnet/*.csproj libs/native/bftree-garnet/ COPY libs/resources/*.csproj libs/resources/ COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/ COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/ diff --git a/Dockerfile.chiseled b/Dockerfile.chiseled index fe394e54525..1f457c59d19 100644 --- a/Dockerfile.chiseled +++ b/Dockerfile.chiseled @@ -12,6 +12,7 @@ COPY libs/cluster/*.csproj libs/cluster/ COPY libs/common/*.csproj libs/common/ COPY libs/host/*.csproj libs/host/ COPY libs/server/*.csproj libs/server/ +COPY libs/native/bftree-garnet/*.csproj libs/native/bftree-garnet/ COPY libs/resources/*.csproj libs/resources/ COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/ COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/ diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index a4374421d23..0db8ad9acdb 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -8,6 +8,7 @@ COPY libs/cluster/*.csproj libs/cluster/ COPY libs/common/*.csproj libs/common/ COPY libs/host/*.csproj libs/host/ COPY libs/server/*.csproj libs/server/ +COPY libs/native/bftree-garnet/*.csproj libs/native/bftree-garnet/ COPY libs/resources/*.csproj libs/resources/ COPY libs/storage/Tsavorite/cs/src/core/*.csproj libs/storage/Tsavorite/cs/src/core/ COPY libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/*.csproj libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/ diff --git a/Garnet.slnx b/Garnet.slnx index 59c0a0b3cb3..4ab7db32f9f 100644 --- a/Garnet.slnx +++ b/Garnet.slnx @@ -1,4 +1,5 @@ + @@ -17,6 +18,7 @@ + @@ -25,16 +27,13 @@ + - - - - - - + + @@ -48,7 +47,6 @@ - @@ -79,7 +77,22 @@ - - + + + + + + + + + + + + + + + + + diff --git a/README.md b/README.md index b2ccf432ec9..c89151e50a2 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,6 @@ [![](https://img.shields.io/nuget/dt/microsoft.garnet.svg?label=nuget%20library&color=007edf&logo=nuget)](https://www.nuget.org/packages/microsoft.garnet) [![](https://img.shields.io/nuget/dt/garnet-server.svg?label=dotnet%20tool&color=007edf&logo=nuget)](https://www.nuget.org/packages/garnet-server) [![BDN Charts](https://img.shields.io/badge/BDN%20Charts-8A2BE2)](https://microsoft.github.io/garnet/charts/) -[![Allure Report](https://img.shields.io/badge/Allure%20Report-orange)](https://microsoft.github.io/garnet/allure/) [![Discord Shield](https://discordapp.com/api/guilds/1213937452272582676/widget.png?style=shield)](https://aka.ms/garnet-discord) Garnet is a new remote cache-store from Microsoft Research, that offers several unique benefits: diff --git a/Version.props b/Version.props index 8a4605fe309..8fe35ab90a5 100644 --- a/Version.props +++ b/Version.props @@ -1,6 +1,6 @@ - 1.1.6 + 2.0.0-beta.7 diff --git a/benchmark/BDN.benchmark/BDN.benchmark.csproj b/benchmark/BDN.benchmark/BDN.benchmark.csproj index 8e6466597a5..19cbe59d2a3 100644 --- a/benchmark/BDN.benchmark/BDN.benchmark.csproj +++ b/benchmark/BDN.benchmark/BDN.benchmark.csproj @@ -1,32 +1,33 @@  - - Exe - enable - true - ../../Garnet.snk - false - + + Exe + enable + true + ../../Garnet.snk + false + - - + + - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - + \ No newline at end of file diff --git a/benchmark/BDN.benchmark/BfTree/BfTreeOperations.cs b/benchmark/BDN.benchmark/BfTree/BfTreeOperations.cs new file mode 100644 index 00000000000..68fe3730944 --- /dev/null +++ b/benchmark/BDN.benchmark/BfTree/BfTreeOperations.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using BenchmarkDotNet.Attributes; +using Garnet.server.BfTreeInterop; +using Tsavorite.core; + +namespace BDN.benchmark.BfTree +{ + /// + /// Benchmarks for BfTree FFI point operations comparing span-based (fixed pinning) + /// vs PinnedSpanByte (zero-overhead) hot paths, across Memory and Disk backends. + /// + [MemoryDiagnoser] + public unsafe class BfTreePointOperations + { + const int ValueSize = 8; + + BfTreeService tree; + string treePath; + + // Pinned arrays allocated via GC.AllocateArray(pinned: true) + byte[] key; + byte[] value; + byte[] readBuffer; + + // Pre-built PinnedSpanByte for zero-overhead benchmarks + PinnedSpanByte pinnedKey; + PinnedSpanByte pinnedValue; + byte* pinnedReadBufPtr; + int pinnedReadBufLen; + + [Params("Memory", "Disk")] + public string Backend { get; set; } + + [GlobalSetup] + public void GlobalSetup() + { + if (Backend == "Disk") + { + treePath = Path.Combine(Path.GetTempPath(), $"bftree_bench_{Guid.NewGuid():N}.bftree"); + tree = new BfTreeService( + storageBackend: StorageBackendType.Disk, + filePath: treePath, + cbMinRecordSize: 8); + } + else + { + tree = new BfTreeService( + storageBackend: StorageBackendType.Memory, + cbMinRecordSize: 8); + } + + // Allocate pinned arrays — no GCHandle needed + key = GC.AllocateArray("key00000"u8.Length, pinned: true); + value = GC.AllocateArray(ValueSize, pinned: true); + readBuffer = GC.AllocateArray(ValueSize, pinned: true); + + pinnedKey = PinnedSpanByte.FromPinnedPointer((byte*)Unsafe.AsPointer(ref key[0]), key.Length); + pinnedValue = PinnedSpanByte.FromPinnedPointer((byte*)Unsafe.AsPointer(ref value[0]), value.Length); + pinnedReadBufPtr = (byte*)Unsafe.AsPointer(ref readBuffer[0]); + pinnedReadBufLen = readBuffer.Length; + + // Pre-populate 1000 key-value entries (key00000..key00999 → val00000..val00999) + // so the total data exceeds the base page size (~4 KB default). This + // ensures reads are served from the cache and disk-backed reads don't + // hit a cold-page corner case. + for (var i = 0; i < 1000; i++) + { + Encoding.UTF8.GetBytes($"key{i:D5}", key); + Encoding.UTF8.GetBytes($"val{i:D5}", value); + tree.Insert(key, value); + } + + // Set the benchmark key-value (key00000) + "key00000"u8.CopyTo(key); + "val00000"u8.CopyTo(value); + + // Validate the read actually returns the correct data + var result = tree.Read(key, readBuffer, out var bytesRead); + Debug.Assert(result == BfTreeReadResult.Found, + $"GlobalSetup validation: expected Found, got {result}"); + Debug.Assert(bytesRead == value.Length, + $"GlobalSetup validation: expected {value.Length} bytes, got {bytesRead}"); + Debug.Assert(readBuffer.AsSpan(0, bytesRead).SequenceEqual(value), + "GlobalSetup validation: read value does not match inserted value"); + } + + [GlobalCleanup] + public void GlobalCleanup() + { + tree?.Dispose(); + if (treePath != null && File.Exists(treePath)) + File.Delete(treePath); + } + + [Benchmark] + public BfTreeReadResult Read_Span() + { + return tree.Read(key, readBuffer, out _); + } + + [Benchmark] + public BfTreeReadResult Read_Pinned() + { + return tree.Read(pinnedKey, pinnedReadBufPtr, pinnedReadBufLen, out _); + } + + [Benchmark] + public int FFI_Noop() + { + return tree.Noop(pinnedKey); + } + } + + /// + /// Benchmarks for BfTree scan operations with callback (zero-alloc). + /// Uses disk-backed mode since cache_only does not support scan. + /// + [MemoryDiagnoser] + public class BfTreeScanOperations + { + private BfTreeService tree; + private string treePath; + private byte[] scanBuffer; + private static readonly byte[] StartKey = [0]; + + [Params(10, 100)] + public int EntryCount { get; set; } + + [GlobalSetup] + public void GlobalSetup() + { + treePath = Path.Combine(Path.GetTempPath(), $"bftree_scanbench_{Guid.NewGuid():N}.bftree"); + tree = new BfTreeService(filePath: treePath, cbMinRecordSize: 8); + scanBuffer = new byte[8192]; + + // Pre-populate 1000 key-value entries (key00000..key00999 → val00000..val00999) + // so the total data exceeds the base page size (~4 KB default). This + // ensures reads are served from the cache and disk-backed reads don't + // hit a cold-page corner case. + for (var i = 0; i < 1000; i++) + { + var key = Encoding.UTF8.GetBytes($"key:{i:D5}"); + var value = Encoding.UTF8.GetBytes($"val:{i:D5}"); + tree.Insert(key, value); + } + } + + [GlobalCleanup] + public void GlobalCleanup() + { + tree?.Dispose(); + if (File.Exists(treePath)) + File.Delete(treePath); + } + + [Benchmark] + public int Scan() + { + return tree.ScanWithCount(StartKey, EntryCount + 1, scanBuffer, + static (key, value) => true); + } + } +} \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs index e428ca60a0f..e0009dd8647 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs @@ -16,7 +16,7 @@ unsafe class ClusterContext EmbeddedRespServer server; RespServerSession session; readonly BenchUtils benchUtils = new(); - readonly int port = 7000; + readonly int port = 7000; // This is not in the shared ClusterTestContext so does not conflict public static ReadOnlySpan keyTag => "{0}"u8; public Request[] singleGetSet; @@ -29,16 +29,28 @@ public void Dispose() server.Dispose(); } - public void SetupSingleInstance(bool disableSlotVerification = false) + public void SetupSingleInstance(ClusterParams clusterParams) { var opt = new GarnetServerOptions { QuietMode = true, - EnableCluster = !disableSlotVerification, + EnableCluster = !clusterParams.disableSlotVerification, EndPoints = [new IPEndPoint(IPAddress.Loopback, port)], CleanClusterConfig = true, - ClusterAnnounceEndpoint = new IPEndPoint(IPAddress.Loopback, port) + ClusterAnnounceEndpoint = new IPEndPoint(IPAddress.Loopback, port), + EnableAOF = clusterParams.enableAof, }; + + if (clusterParams.enableAof) + { + opt.EnableAOF = true; + opt.UseAofNullDevice = true; + opt.FastAofTruncate = true; + opt.CommitFrequencyMs = -1; + opt.AofPageSize = "128m"; + opt.AofMemorySize = "256m"; + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) opt.CheckpointDir = "/tmp"; server = new EmbeddedRespServer(opt); @@ -168,5 +180,4 @@ public void CreateCTXNSET(int keySize = 8, int batchSize = 100) public void Consume(byte* ptr, int length) => session.TryConsumeMessages(ptr, length); } - } \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Cluster/ClusterMigrate.cs b/benchmark/BDN.benchmark/Cluster/ClusterMigrate.cs index 532989937c4..45cd0689d75 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterMigrate.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterMigrate.cs @@ -25,7 +25,7 @@ public unsafe class ClusterMigrate /// public IEnumerable ClusterParamsProvider() { - yield return new(false); + yield return new(false, false); } ClusterContext cc; @@ -34,7 +34,7 @@ public IEnumerable ClusterParamsProvider() public void GlobalSetup() { cc = new ClusterContext(); - cc.SetupSingleInstance(); + cc.SetupSingleInstance(Params); cc.AddSlotRange([(0, 16383)]); cc.CreateGetSet(); cc.CreateMGetMSet(); diff --git a/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs b/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs index 2799885ae92..9da411d87bd 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs @@ -22,8 +22,9 @@ public unsafe class ClusterOperations /// public IEnumerable ClusterParamsProvider() { - yield return new(false); - yield return new(true); + yield return new(false, false); + yield return new(true, false); + yield return new(false, true); } ClusterContext cc; @@ -32,7 +33,7 @@ public IEnumerable ClusterParamsProvider() public virtual void GlobalSetup() { cc = new ClusterContext(); - cc.SetupSingleInstance(Params.disableSlotVerification); + cc.SetupSingleInstance(Params); cc.AddSlotRange([(0, 16383)]); cc.CreateGetSet(); cc.CreateMGetMSet(); diff --git a/benchmark/BDN.benchmark/Cluster/ClusterParams.cs b/benchmark/BDN.benchmark/Cluster/ClusterParams.cs index a47a86568bc..68b3e5abe98 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterParams.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterParams.cs @@ -13,12 +13,18 @@ public struct ClusterParams /// public bool disableSlotVerification; + /// + /// Whether to enable AOF + /// + public bool enableAof; + /// /// Constructor /// - public ClusterParams(bool disableSlotVerification) + public ClusterParams(bool disableSlotVerification, bool enableAof) { this.disableSlotVerification = disableSlotVerification; + this.enableAof = enableAof; } /// @@ -26,12 +32,16 @@ public ClusterParams(bool disableSlotVerification) /// public override string ToString() { - if (!disableSlotVerification) + if (!disableSlotVerification && !enableAof) return "None"; var ret = ""; if (disableSlotVerification) ret += "DSV"; + + if (enableAof) + ret += ret.Length == 0 ? "AOF" : "+AOF"; + return ret; } } diff --git a/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs b/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs index b833e696c52..835e966ba6c 100644 --- a/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs +++ b/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs @@ -22,15 +22,15 @@ sealed class CustomTxnSet : CustomTransactionProcedure /// public const string CommandName = "CTXNSET"; - ArgSlice setA; - ArgSlice setB; - ArgSlice setC; - ArgSlice setD; + PinnedSpanByte setA; + PinnedSpanByte setB; + PinnedSpanByte setC; + PinnedSpanByte setD; - ArgSlice valueA; - ArgSlice valueB; - ArgSlice valueC; - ArgSlice valueD; + PinnedSpanByte valueA; + PinnedSpanByte valueB; + PinnedSpanByte valueC; + PinnedSpanByte valueD; /// /// CTXNSET key1 key2 key3 key4 value1 value2 value3 value4 @@ -52,10 +52,10 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce valueC = GetNextArg(ref procInput, ref offset); valueD = GetNextArg(ref procInput, ref offset); - AddKey(setA, LockType.Exclusive, isObject: false); - AddKey(setB, LockType.Exclusive, isObject: false); - AddKey(setC, LockType.Exclusive, isObject: false); - AddKey(setD, LockType.Exclusive, isObject: false); + AddKey(setA, LockType.Exclusive, StoreType.Main); + AddKey(setB, LockType.Exclusive, StoreType.Main); + AddKey(setC, LockType.Exclusive, StoreType.Main); + AddKey(setD, LockType.Exclusive, StoreType.Main); return true; } diff --git a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs index df9bf71f533..d456a3522e0 100644 --- a/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs +++ b/benchmark/BDN.benchmark/Embedded/EmbeddedRespServer.cs @@ -30,7 +30,6 @@ public EmbeddedRespServer(GarnetServerOptions opts, ILoggerFactory loggerFactory new SubscribeBroker( null, opts.PubSubPageSizeBytes(), - opts.SubscriberRefreshFrequencyMs, pubSubEpoch, true); } @@ -51,6 +50,14 @@ internal RespServerSession GetRespSession() return new RespServerSession(0, new EmbeddedNetworkSender(), storeWrapper, subscribeBroker: subscribeBroker, null, true); } + internal RespServerSession[] GetRespSessions(int count) + { + var sessions = new RespServerSession[count]; + for (var i = 0; i < count; i++) + sessions[i] = new RespServerSession(i, new EmbeddedNetworkSender(), storeWrapper, subscribeBroker: subscribeBroker, null, true); + return sessions; + } + internal EmbeddedNetworkHandler GetNetworkHandler() { return garnetServerEmbedded.CreateNetworkHandler(); diff --git a/benchmark/BDN.benchmark/Filter/FilterExpressionBenchmarks.cs b/benchmark/BDN.benchmark/Filter/FilterExpressionBenchmarks.cs deleted file mode 100644 index 76384526843..00000000000 --- a/benchmark/BDN.benchmark/Filter/FilterExpressionBenchmarks.cs +++ /dev/null @@ -1,551 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Buffers.Binary; -using System.Text; -using BenchmarkDotNet.Attributes; -using Garnet.server; - -namespace BDN.benchmark.Filter -{ - // ════════════════════════════════════════════════════════════════════════ - // 1. COMPILATION (one-time cost per VSIM query) - // ════════════════════════════════════════════════════════════════════════ - - /// Compile filter string → postfix program. - [MemoryDiagnoser] - public class FilterCompileBenchmarks - { - private byte[] _comparison; - private byte[] _logicalAnd; - private byte[] _stringEq; - private byte[] _arithmetic; - private byte[] _containment; - private byte[] _combined; - - [GlobalSetup] - public void Setup() - { - _comparison = ".year > 1950"u8.ToArray(); - _logicalAnd = ".year > 1950 and .rating >= 4.0"u8.ToArray(); - _stringEq = ".genre == \"action\""u8.ToArray(); - _arithmetic = "(.year - 2000) ** 2 < 100"u8.ToArray(); - _containment = "\"classic\" in .tags"u8.ToArray(); - _combined = ".rating * 2 > 8 and (.year >= 1980 or \"modern\" in .tags) and .genre == \"action\""u8.ToArray(); - } - - private static void Compile(byte[] filter) - { - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - ExprCompiler.TryCompile(filter, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out _, out _); - } - - [Benchmark(Description = "Comparison (.year > N)")] - public void Comparison() => Compile(_comparison); - - [Benchmark(Description = "Logical AND (2 clauses)")] - public void LogicalAnd() => Compile(_logicalAnd); - - [Benchmark(Description = "String equality")] - public void StringEq() => Compile(_stringEq); - - [Benchmark(Description = "Arithmetic + power")] - public void Arithmetic() => Compile(_arithmetic); - - [Benchmark(Description = "Containment (in)")] - public void Containment() => Compile(_containment); - - [Benchmark(Description = "Combined (all ops)")] - public void Combined() => Compile(_combined); - } - - // ════════════════════════════════════════════════════════════════════════ - // 2. FIELD EXTRACTION (per candidate, per selector) - // ════════════════════════════════════════════════════════════════════════ - - /// - /// Extract a single field from raw JSON bytes. - /// Parameterized by JSON size: Small (2 fields), Medium (5), Large (12 + nested obj). - /// - [MemoryDiagnoser] - public class FilterExtractBenchmarks - { - // Small: {"year":1980,"rating":4.5} - // Medium: {"year":1980,"rating":4.5,"genre":"action","director":"Spielberg","tags":["classic","popular"]} - // Large: 12 fields including nested object and 3-element array - private byte[] _small; - private byte[] _medium; - private byte[] _large; - - [GlobalSetup] - public void Setup() - { - _small = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5}"); - _medium = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"tags\":[\"classic\",\"popular\"]}"); - _large = Encoding.UTF8.GetBytes("{\"id\":12345,\"title\":\"Test Movie\",\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"studio\":\"Universal\",\"budget\":50000000,\"tags\":[\"classic\",\"popular\",\"award-winning\"],\"metadata\":{\"source\":\"imdb\",\"verified\":true},\"active\":true}"); - } - - // --- Number fields (zero-alloc) --- - [Benchmark(Description = "Number · Small JSON (1st field)")] - public void Num_Small() => AttributeExtractor.ExtractField(_small, "year"u8); - - [Benchmark(Description = "Number · Medium JSON (2nd field)")] - public void Num_Medium() => AttributeExtractor.ExtractField(_medium, "rating"u8); - - [Benchmark(Description = "Number · Large JSON (skip 8 fields)")] - public void Num_Large() => AttributeExtractor.ExtractField(_large, "budget"u8); - - // --- String fields (zero-alloc) --- - [Benchmark(Description = "String · Medium JSON")] - public void Str_Medium() => AttributeExtractor.ExtractField(_medium, "genre"u8); - - [Benchmark(Description = "String · Large JSON (skip 5)")] - public void Str_Large() => AttributeExtractor.ExtractField(_large, "director"u8); - - // --- Array fields (zero-alloc with runtime pool) --- - [Benchmark(Description = "Array[2] · Medium JSON")] - public void Arr_Medium() => AttributeExtractor.ExtractField(_medium, "tags"u8); - - [Benchmark(Description = "Array[3] · Large JSON")] - public void Arr_Large() => AttributeExtractor.ExtractField(_large, "tags"u8); - - // --- Boolean (zero-alloc) --- - [Benchmark(Description = "Boolean · Large JSON (skip nested obj)")] - public void Bool_Large() => AttributeExtractor.ExtractField(_large, "active"u8); - - // --- Missing field (zero-alloc) --- - [Benchmark(Description = "Missing · Small JSON")] - public void Miss_Small() => AttributeExtractor.ExtractField(_small, "missing"u8); - - [Benchmark(Description = "Missing · Medium JSON")] - public void Miss_Medium() => AttributeExtractor.ExtractField(_medium, "missing"u8); - - [Benchmark(Description = "Missing · Large JSON")] - public void Miss_Large() => AttributeExtractor.ExtractField(_large, "missing"u8); - } - - // ════════════════════════════════════════════════════════════════════════ - // 3. EXECUTION BY EXPRESSION TYPE (compile-once, run per candidate) - // Fixed JSON: Medium (5 fields, includes array) - // Ordered: most frequent → least frequent real-world query patterns - // ════════════════════════════════════════════════════════════════════════ - - /// - /// Run pre-compiled filters against medium JSON. - /// Ordered from most common to least common real-world usage patterns. - /// - [MemoryDiagnoser] - public class FilterRunByExprBenchmarks - { - private byte[] _comparisonFilter; - private byte[] _logicalAndFilter; - private byte[] _stringEqFilter; - private byte[] _containsArrayFilter; - private byte[] _logicalOrFilter; - private byte[] _notFilter; - private byte[] _stringNeqFilter; - private byte[] _arithmeticFilter; - private byte[] _powerFilter; - private byte[] _containsStringFilter; - private byte[] _combinedFilter; - private byte[] _json; - - [GlobalSetup] - public void Setup() - { - _comparisonFilter = ".year > 1950"u8.ToArray(); - _logicalAndFilter = ".year > 1950 and .rating >= 4.0"u8.ToArray(); - _stringEqFilter = ".genre == \"action\""u8.ToArray(); - _containsArrayFilter = "\"classic\" in .tags"u8.ToArray(); - _logicalOrFilter = ".year < 1960 or .rating > 4.0"u8.ToArray(); - _notFilter = "not (.genre == \"drama\")"u8.ToArray(); - _stringNeqFilter = ".genre != \"drama\""u8.ToArray(); - _arithmeticFilter = ".rating * 2 > 8"u8.ToArray(); - _powerFilter = "(.year - 2000) ** 2 < 100"u8.ToArray(); - _containsStringFilter = "\"act\" in .genre"u8.ToArray(); - _combinedFilter = ".rating * 2 > 8 and (.year >= 1980 or \"modern\" in .tags) and .genre == \"action\""u8.ToArray(); - _json = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"tags\":[\"classic\",\"popular\"]}"); - } - - /// - /// Compile filter, build ExprProgram, extract fields, and evaluate — all on the stack. - /// - private static bool RunFilter(byte[] filterBytes, byte[] json) - { - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out _); - if (instrCount < 0) return false; - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - Span<(int Start, int Length)> selectorBuf = stackalloc (int, int)[32]; - var selectorCount = VectorManager.GetSelectorRanges(program.Instructions, program.Length, filterBytes, selectorBuf); - var selectorRanges = selectorBuf[..selectorCount]; - - Span extractedFields = stackalloc ExprToken[selectorCount > 0 ? selectorCount : 1]; - program.ResetRuntimePool(); - AttributeExtractor.ExtractFields(json, filterBytes, selectorRanges, extractedFields, ref program); - - Span stackBuf = stackalloc ExprToken[16]; - var stack = new ExprStack(stackBuf); - return ExprRunner.Run(ref program, json, filterBytes, selectorRanges, extractedFields, ref stack); - } - - // ── Common: range / categorical ────────────────────────────────── - - [Benchmark(Description = "1. .year > N (range)")] - public bool Comparison() => RunFilter(_comparisonFilter, _json); - - [Benchmark(Description = "2. .year > N and .rating >= M (multi-range)")] - public bool LogicalAnd() => RunFilter(_logicalAndFilter, _json); - - [Benchmark(Description = "3. .genre == \"action\" (category)")] - public bool StringEq() => RunFilter(_stringEqFilter, _json); - - [Benchmark(Description = "4. \"x\" in .tags (tag search)")] - public bool InArray() => RunFilter(_containsArrayFilter, _json); - - // ── Moderate: logical combinations ─────────────────────────────── - - [Benchmark(Description = "5. A or B (logical OR)")] - public bool LogicalOr() => RunFilter(_logicalOrFilter, _json); - - [Benchmark(Description = "6. not (A) (exclusion)")] - public bool Not() => RunFilter(_notFilter, _json); - - [Benchmark(Description = "7. .genre != \"drama\" (not-equal)")] - public bool StringNeq() => RunFilter(_stringNeqFilter, _json); - - // ── Less common: computed / advanced ───────────────────────────── - - [Benchmark(Description = "8. .rating * 2 > 8 (arithmetic)")] - public bool Arithmetic() => RunFilter(_arithmeticFilter, _json); - - [Benchmark(Description = "9. (.year-2000)**2 < 100 (power)")] - public bool Power() => RunFilter(_powerFilter, _json); - - [Benchmark(Description = "10. \"act\" in .genre (substring)")] - public bool InString() => RunFilter(_containsStringFilter, _json); - - // ── Realistic combined ─────────────────────────────────────────── - - [Benchmark(Description = "11. Combined (all ops)")] - public bool Combined() => RunFilter(_combinedFilter, _json); - } - - // ════════════════════════════════════════════════════════════════════════ - // 4. EXECUTION BY JSON COMPLEXITY (fixed filter, varying JSON) - // ════════════════════════════════════════════════════════════════════════ - - /// - /// Same filter run against small / medium / large JSON. - /// Shows how JSON size affects extraction + evaluation time. - /// - [MemoryDiagnoser] - public class FilterRunByJsonBenchmarks - { - private byte[] _numericFilterBytes; - private byte[] _arrayFilterBytes; - - private byte[] _small; // 2 fields, no array - private byte[] _medium; // 5 fields, 2-element array - private byte[] _large; // 12 fields, 3-element array, nested object - - [GlobalSetup] - public void Setup() - { - _numericFilterBytes = ".year > 1950 and .rating >= 4.0"u8.ToArray(); - _arrayFilterBytes = "\"classic\" in .tags"u8.ToArray(); - - _small = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5}"); - _medium = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"tags\":[\"classic\",\"popular\"]}"); - _large = Encoding.UTF8.GetBytes("{\"id\":12345,\"title\":\"Test Movie\",\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"studio\":\"Universal\",\"budget\":50000000,\"tags\":[\"classic\",\"popular\",\"award-winning\"],\"metadata\":{\"source\":\"imdb\",\"verified\":true},\"active\":true}"); - } - - private static bool RunFilter(byte[] filterBytes, byte[] json) - { - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out _); - if (instrCount < 0) return false; - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - Span<(int Start, int Length)> selectorBuf = stackalloc (int, int)[32]; - var selectorCount = VectorManager.GetSelectorRanges(program.Instructions, program.Length, filterBytes, selectorBuf); - var selectorRanges = selectorBuf[..selectorCount]; - - Span extractedFields = stackalloc ExprToken[selectorCount > 0 ? selectorCount : 1]; - program.ResetRuntimePool(); - AttributeExtractor.ExtractFields(json, filterBytes, selectorRanges, extractedFields, ref program); - - Span stackBuf = stackalloc ExprToken[16]; - var stack = new ExprStack(stackBuf); - return ExprRunner.Run(ref program, json, filterBytes, selectorRanges, extractedFields, ref stack); - } - - // --- Numeric filter --- - [Benchmark(Description = "Numeric AND · Small JSON")] - public bool Numeric_Small() => RunFilter(_numericFilterBytes, _small); - - [Benchmark(Description = "Numeric AND · Medium JSON")] - public bool Numeric_Medium() => RunFilter(_numericFilterBytes, _medium); - - [Benchmark(Description = "Numeric AND · Large JSON")] - public bool Numeric_Large() => RunFilter(_numericFilterBytes, _large); - - // --- Array filter --- - [Benchmark(Description = "in .tags · Small JSON (no tags → false)")] - public bool Array_Small() => RunFilter(_arrayFilterBytes, _small); - - [Benchmark(Description = "in .tags · Medium JSON (2 elem)")] - public bool Array_Medium() => RunFilter(_arrayFilterBytes, _medium); - - [Benchmark(Description = "in .tags · Large JSON (3 elem)")] - public bool Array_Large() => RunFilter(_arrayFilterBytes, _large); - } - - // ════════════════════════════════════════════════════════════════════════ - // 5. BATCH (compile once, run N candidates) - // ════════════════════════════════════════════════════════════════════════ - - /// - /// Simulate real VSIM post-filtering: compile once, evaluate N candidates. - /// Shows total allocation and throughput at scale. - /// - [MemoryDiagnoser] - public class FilterBatchBenchmarks - { - private byte[] _numericAndFilter; - private byte[] _combinedFilter; - private byte[] _small; - private byte[] _medium; - - [GlobalSetup] - public void Setup() - { - _numericAndFilter = ".year > 1950 and .rating >= 4.0"u8.ToArray(); - _combinedFilter = ".rating * 2 > 8 and (.year >= 1980 or \"modern\" in .tags) and .genre == \"action\""u8.ToArray(); - _small = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5}"); - _medium = Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"tags\":[\"classic\",\"popular\"]}"); - } - - private static int RunBatch(byte[] filterBytes, byte[] small, byte[] medium, int N) - { - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out _); - if (instrCount < 0) return 0; - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - Span<(int Start, int Length)> selectorBuf = stackalloc (int, int)[32]; - var selectorCount = VectorManager.GetSelectorRanges(program.Instructions, program.Length, filterBytes, selectorBuf); - var selectorRanges = selectorBuf[..selectorCount]; - - Span extractedFields = stackalloc ExprToken[selectorCount > 0 ? selectorCount : 1]; - Span stackBuf = stackalloc ExprToken[16]; - var stack = new ExprStack(stackBuf); - - var matched = 0; - for (var i = 0; i < N; i++) - { - var json = (i % 3 == 0) ? small : medium; - program.ResetRuntimePool(); - AttributeExtractor.ExtractFields(json, filterBytes, selectorRanges, extractedFields, ref program); - if (ExprRunner.Run(ref program, json, filterBytes, selectorRanges, extractedFields, ref stack)) - matched++; - } - return matched; - } - - [Benchmark(Description = "Numeric AND · N candidates (zero-alloc)")] - [Arguments(10)] - [Arguments(100)] - [Arguments(1000)] - public int NumericAnd(int N) => RunBatch(_numericAndFilter, _small, _medium, N); - - [Benchmark(Description = "Combined + array · N candidates")] - [Arguments(10)] - [Arguments(100)] - [Arguments(1000)] - public int Combined(int N) => RunBatch(_combinedFilter, _small, _medium, N); - } - - // ════════════════════════════════════════════════════════════════════════ - // 6. END-TO-END ApplyPostFilter (compile + extract + evaluate N candidates) - // Exercises the full pipeline including length-prefixed attribute span layout - // ════════════════════════════════════════════════════════════════════════ - - /// - /// End-to-end benchmark of . - /// Builds a realistic length-prefixed attribute span (as produced by VSIM), - /// then calls ApplyPostFilter which compiles the filter, extracts fields, - /// and evaluates each candidate. - /// - [MemoryDiagnoser] - public class FilterApplyPostFilterBenchmarks - { - // Filters of varying complexity - private byte[] _numericFilter; - private byte[] _stringFilter; - private byte[] _arrayFilter; - private byte[] _combinedFilter; - - // Pre-built length-prefixed attribute spans for different candidate counts - private byte[] _attrs10; - private byte[] _attrs100; - private byte[] _attrs1000; - - // Bitmap buffers (ceil(N/8) bytes) - private byte[] _bitmap10; - private byte[] _bitmap100; - private byte[] _bitmap1000; - - // Scratch buffer for ApplyPostFilter - private ScratchBufferBuilder _scratchBufferBuilder; - - [GlobalSetup] - public void Setup() - { - _scratchBufferBuilder = new ScratchBufferBuilder(); - - _numericFilter = ".year > 1950 and .rating >= 4.0"u8.ToArray(); - _stringFilter = ".genre == \"action\""u8.ToArray(); - _arrayFilter = "\"classic\" in .tags"u8.ToArray(); - _combinedFilter = ".rating * 2 > 8 and (.year >= 1980 or \"classic\" in .tags) and .genre == \"action\""u8.ToArray(); - - // Build diverse JSON candidates — mix of matching and non-matching - var candidates = new[] - { - // Matches numeric+string+combined: year>1950, rating>=4.0, genre=action - Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5,\"genre\":\"action\",\"director\":\"Spielberg\",\"tags\":[\"classic\",\"popular\"]}"), - // Matches numeric only: year>1950, rating>=4.0, genre=drama - Encoding.UTF8.GetBytes("{\"year\":2005,\"rating\":4.2,\"genre\":\"drama\",\"director\":\"Nolan\",\"tags\":[\"modern\"]}"), - // Doesn't match numeric: year<1950 - Encoding.UTF8.GetBytes("{\"year\":1940,\"rating\":3.8,\"genre\":\"noir\",\"director\":\"Wilder\"}"), - // Matches all: year>1950, rating>=4.0, genre=action, has classic tag - Encoding.UTF8.GetBytes("{\"year\":1999,\"rating\":4.9,\"genre\":\"action\",\"director\":\"Wachowski\",\"tags\":[\"classic\",\"scifi\",\"popular\"]}"), - // Large JSON (12 fields) — matches numeric - Encoding.UTF8.GetBytes("{\"id\":12345,\"title\":\"Test Movie\",\"year\":2010,\"rating\":4.1,\"genre\":\"comedy\",\"director\":\"Anderson\",\"studio\":\"Fox\",\"budget\":50000000,\"tags\":[\"indie\"],\"metadata\":{\"source\":\"imdb\",\"verified\":true},\"active\":true}"), - // Small JSON — doesn't match (missing fields) - Encoding.UTF8.GetBytes("{\"year\":1980,\"rating\":4.5}"), - // Large JSON — matches combined - Encoding.UTF8.GetBytes("{\"id\":99,\"title\":\"Action Hero\",\"year\":2020,\"rating\":4.8,\"genre\":\"action\",\"director\":\"Bay\",\"studio\":\"Paramount\",\"budget\":200000000,\"tags\":[\"classic\",\"blockbuster\"],\"metadata\":{\"source\":\"rotten\"},\"active\":true}"), - }; - - _attrs10 = BuildAttributeSpan(candidates, 10); - _attrs100 = BuildAttributeSpan(candidates, 100); - _attrs1000 = BuildAttributeSpan(candidates, 1000); - - _bitmap10 = new byte[(10 + 7) / 8]; - _bitmap100 = new byte[(100 + 7) / 8]; - _bitmap1000 = new byte[(1000 + 7) / 8]; - } - - /// - /// Build a length-prefixed attribute span: for each candidate, write [int32 len][json bytes]. - /// Cycles through the candidate array to fill N entries. - /// - private static byte[] BuildAttributeSpan(byte[][] candidates, int count) - { - // Calculate total size - var totalSize = 0; - for (var i = 0; i < count; i++) - totalSize += sizeof(int) + candidates[i % candidates.Length].Length; - - var result = new byte[totalSize]; - var offset = 0; - for (var i = 0; i < count; i++) - { - var json = candidates[i % candidates.Length]; - BinaryPrimitives.WriteInt32LittleEndian(result.AsSpan(offset), json.Length); - offset += sizeof(int); - json.CopyTo(result, offset); - offset += json.Length; - } - return result; - } - - // ── Numeric filter: .year > 1950 and .rating >= 4.0 ──────────── - - [Benchmark(Description = "Numeric AND · 10 candidates")] - public int Numeric_10() => VectorManager.ApplyPostFilter(_numericFilter, 10, _attrs10, _bitmap10, _scratchBufferBuilder); - - [Benchmark(Description = "Numeric AND · 100 candidates")] - public int Numeric_100() => VectorManager.ApplyPostFilter(_numericFilter, 100, _attrs100, _bitmap100, _scratchBufferBuilder); - - [Benchmark(Description = "Numeric AND · 1000 candidates")] - public int Numeric_1000() => VectorManager.ApplyPostFilter(_numericFilter, 1000, _attrs1000, _bitmap1000, _scratchBufferBuilder); - - // ── String filter: .genre == "action" ─────────────────────────── - - [Benchmark(Description = "String EQ · 10 candidates")] - public int String_10() => VectorManager.ApplyPostFilter(_stringFilter, 10, _attrs10, _bitmap10, _scratchBufferBuilder); - - [Benchmark(Description = "String EQ · 100 candidates")] - public int String_100() => VectorManager.ApplyPostFilter(_stringFilter, 100, _attrs100, _bitmap100, _scratchBufferBuilder); - - [Benchmark(Description = "String EQ · 1000 candidates")] - public int String_1000() => VectorManager.ApplyPostFilter(_stringFilter, 1000, _attrs1000, _bitmap1000, _scratchBufferBuilder); - - // ── Array filter: "classic" in .tags ──────────────────────────── - - [Benchmark(Description = "Array IN · 10 candidates")] - public int Array_10() => VectorManager.ApplyPostFilter(_arrayFilter, 10, _attrs10, _bitmap10, _scratchBufferBuilder); - - [Benchmark(Description = "Array IN · 100 candidates")] - public int Array_100() => VectorManager.ApplyPostFilter(_arrayFilter, 100, _attrs100, _bitmap100, _scratchBufferBuilder); - - [Benchmark(Description = "Array IN · 1000 candidates")] - public int Array_1000() => VectorManager.ApplyPostFilter(_arrayFilter, 1000, _attrs1000, _bitmap1000, _scratchBufferBuilder); - - // ── Combined filter: rating*2>8 and (year>=1980 or "classic" in .tags) and genre=="action" - - [Benchmark(Description = "Combined · 10 candidates")] - public int Combined_10() => VectorManager.ApplyPostFilter(_combinedFilter, 10, _attrs10, _bitmap10, _scratchBufferBuilder); - - [Benchmark(Description = "Combined · 100 candidates")] - public int Combined_100() => VectorManager.ApplyPostFilter(_combinedFilter, 100, _attrs100, _bitmap100, _scratchBufferBuilder); - - [Benchmark(Description = "Combined · 1000 candidates")] - public int Combined_1000() => VectorManager.ApplyPostFilter(_combinedFilter, 1000, _attrs1000, _bitmap1000, _scratchBufferBuilder); - } -} \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Operations/OperationsBase.cs b/benchmark/BDN.benchmark/Operations/OperationsBase.cs index f57caaa032b..1e519e6ed72 100644 --- a/benchmark/BDN.benchmark/Operations/OperationsBase.cs +++ b/benchmark/BDN.benchmark/Operations/OperationsBase.cs @@ -27,10 +27,17 @@ public abstract unsafe class OperationsBase public IEnumerable OperationParamsProvider() { yield return new(false, false); + if (ParamsNoneOnly) + yield break; yield return new(true, false); yield return new(false, true); } + /// + /// Set by environment variable BDNRUN_OP_PARAM - determines if running with only "None" parameters (no ACLs, no AOF) or with all combinations of parameters + /// + internal static bool ParamsNoneOnly; + /// /// Batch size per method invocation /// With a batchSize of 100, we have a convenient conversion of latency to throughput: diff --git a/benchmark/BDN.benchmark/Operations/RangeIndexOperations.cs b/benchmark/BDN.benchmark/Operations/RangeIndexOperations.cs new file mode 100644 index 00000000000..cf25a090b13 --- /dev/null +++ b/benchmark/BDN.benchmark/Operations/RangeIndexOperations.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using BenchmarkDotNet.Attributes; +using Embedded.server; + +namespace BDN.benchmark.Operations +{ + /// + /// Benchmark for RangeIndex (BfTree) operations. + /// Pre-creates a memory-only range index with 100 key-value entries, + /// then benchmarks individual RI.SET, RI.GET, RI.DEL, RI.SCAN, and RI.RANGE operations. + /// + [MemoryDiagnoser] + public class RangeIndexOperations : OperationsBase + { + // RI.SET myindex key00000 val00000 + static ReadOnlySpan RISET => "*4\r\n$6\r\nRI.SET\r\n$7\r\nmyindex\r\n$8\r\nkey00000\r\n$8\r\nval00000\r\n"u8; + Request riSet; + + // RI.GET myindex key00000 + static ReadOnlySpan RIGET => "*3\r\n$6\r\nRI.GET\r\n$7\r\nmyindex\r\n$8\r\nkey00000\r\n"u8; + Request riGet; + + // RI.DEL myindex key99999 (non-existent field — bf-tree delete is void, so this is a no-op) + static ReadOnlySpan RIDEL => "*3\r\n$6\r\nRI.DEL\r\n$7\r\nmyindex\r\n$8\r\nkey99999\r\n"u8; + Request riDel; + + // RI.SCAN myindex key00048 COUNT 5 + static ReadOnlySpan RISCAN => "*5\r\n$7\r\nRI.SCAN\r\n$7\r\nmyindex\r\n$8\r\nkey00048\r\n$5\r\nCOUNT\r\n$1\r\n5\r\n"u8; + Request riScan; + + // RI.RANGE myindex key00048 key00052 + static ReadOnlySpan RIRANGE => "*4\r\n$8\r\nRI.RANGE\r\n$7\r\nmyindex\r\n$8\r\nkey00048\r\n$8\r\nkey00052\r\n"u8; + Request riRange; + + // RI.SCAN myindex key00048 COUNT 5 FIELDS KEY + static ReadOnlySpan RISCAN_KEYS => "*7\r\n$7\r\nRI.SCAN\r\n$7\r\nmyindex\r\n$8\r\nkey00048\r\n$5\r\nCOUNT\r\n$1\r\n5\r\n$6\r\nFIELDS\r\n$3\r\nKEY\r\n"u8; + Request riScanKeys; + + /// + /// Skip ACL variants — RI commands are not in the default ACL whitelist. + /// + public new IEnumerable OperationParamsProvider() + { + yield return new(false, false); + yield return new(false, true); + } + + public override void GlobalSetup() + { + base.GlobalSetup(); + + SetupOperation(ref riSet, RISET); + SetupOperation(ref riGet, RIGET); + SetupOperation(ref riDel, RIDEL); + SetupOperation(ref riScan, RISCAN); + SetupOperation(ref riRange, RIRANGE); + SetupOperation(ref riScanKeys, RISCAN_KEYS); + + // Create a disk-backed range index (memory-only mode does not support scan) + var createCmd = "*5\r\n$9\r\nRI.CREATE\r\n$7\r\nmyindex\r\n$4\r\nDISK\r\n$9\r\nMINRECORD\r\n$1\r\n8\r\n"; + SlowConsumeMessage(System.Text.Encoding.UTF8.GetBytes(createCmd)); + + // Pre-populate 1000 key-value entries (key00000..key00999 → val00000..val00999) + // so the total data exceeds the base page size (~4 KB default). This + // ensures reads are served from the cache and disk-backed reads don't + // hit a cold-page corner case. + for (var i = 0; i < 1000; i++) + { + var key = $"key{i:D5}"; + var val = $"val{i:D5}"; + var cmd = $"*4\r\n$6\r\nRI.SET\r\n$7\r\nmyindex\r\n${key.Length}\r\n{key}\r\n${val.Length}\r\n{val}\r\n"; + SlowConsumeMessage(System.Text.Encoding.UTF8.GetBytes(cmd)); + } + } + + [Benchmark] + public void RISet() + { + Send(riSet); + } + + [Benchmark] + public void RIGet() + { + Send(riGet); + } + + [Benchmark] + public void RIDel() + { + Send(riDel); + } + + [Benchmark] + public void RIScan() + { + Send(riScan); + } + + [Benchmark] + public void RIScanKeysOnly() + { + Send(riScanKeys); + } + + [Benchmark] + public void RIRange() + { + Send(riRange); + } + } +} \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Operations/RawStringOperations.cs b/benchmark/BDN.benchmark/Operations/RawStringOperations.cs index ac4cf31db53..a0a1e355b48 100644 --- a/benchmark/BDN.benchmark/Operations/RawStringOperations.cs +++ b/benchmark/BDN.benchmark/Operations/RawStringOperations.cs @@ -18,10 +18,10 @@ public unsafe class RawStringOperations : OperationsBase static ReadOnlySpan SETEX => "*4\r\n$5\r\nSETEX\r\n$1\r\nd\r\n$1\r\n9\r\n$1\r\nd\r\n"u8; Request setex; - static ReadOnlySpan SETNX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nNX\r\n"u8; + static ReadOnlySpan SETNX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nNX\r\n"u8; // Becomes SETEXNX rather than SETNX Request setnx; - static ReadOnlySpan SETXX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nXX\r\n"u8; + static ReadOnlySpan SETXX => "*4\r\n$3\r\nSET\r\n$1\r\na\r\n$1\r\na\r\n$2\r\nXX\r\n"u8; // Becomes SETEXXX rather than SETXX Request setxx; static ReadOnlySpan GETNF => "*2\r\n$3\r\nGET\r\n$1\r\nb\r\n"u8; diff --git a/benchmark/BDN.benchmark/Program.cs b/benchmark/BDN.benchmark/Program.cs index 92e77b5c643..80103fa1730 100644 --- a/benchmark/BDN.benchmark/Program.cs +++ b/benchmark/BDN.benchmark/Program.cs @@ -40,7 +40,7 @@ public BaseConfig() .WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredPGO", "0")); // Get value of environment variable BDNRUNPARAM - determines if running net8.0, net10.0 or both (if env var is not set or invalid) - var bdnRunParam = Environment.GetEnvironmentVariable("BDNRUNPARAM"); + var bdnRunParam = (Environment.GetEnvironmentVariable("BDNRUNPARAM") ?? string.Empty).ToLower(); switch (bdnRunParam) { @@ -57,5 +57,16 @@ public BaseConfig() ); break; } + + // Get value of environment variable BDN_OP_PARAM - determines if running ACL and AOF as well as 'none' + var bdnOpParam = (Environment.GetEnvironmentVariable("BDN_OP_PARAM") ?? string.Empty).ToLower(); + switch (bdnOpParam) + { + case "none": + BDN.benchmark.Operations.OperationsBase.ParamsNoneOnly = true; + break; + default: + break; + } } } \ No newline at end of file diff --git a/benchmark/Device.benchmark/Device.benchmark.csproj b/benchmark/Device.benchmark/Device.benchmark.csproj index 3aa56eac9e2..b8d159b8ac6 100644 --- a/benchmark/Device.benchmark/Device.benchmark.csproj +++ b/benchmark/Device.benchmark/Device.benchmark.csproj @@ -6,10 +6,6 @@ true - - - - PreserveNewest diff --git a/benchmark/Resp.benchmark/BenchUtils.cs b/benchmark/Resp.benchmark/Common/BenchUtils.cs similarity index 99% rename from benchmark/Resp.benchmark/BenchUtils.cs rename to benchmark/Resp.benchmark/Common/BenchUtils.cs index e95a210bf03..3ae6c0ccff9 100644 --- a/benchmark/Resp.benchmark/BenchUtils.cs +++ b/benchmark/Resp.benchmark/Common/BenchUtils.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Collections.Generic; using System.Diagnostics; using System.Net.Security; using System.Text; diff --git a/benchmark/Resp.benchmark/BenchmarkLoggerProvider.cs b/benchmark/Resp.benchmark/Common/BenchmarkLoggerProvider.cs similarity index 99% rename from benchmark/Resp.benchmark/BenchmarkLoggerProvider.cs rename to benchmark/Resp.benchmark/Common/BenchmarkLoggerProvider.cs index 06ad62bd4aa..d6bfbe953d2 100644 --- a/benchmark/Resp.benchmark/BenchmarkLoggerProvider.cs +++ b/benchmark/Resp.benchmark/Common/BenchmarkLoggerProvider.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.IO; using System.Runtime.CompilerServices; using Microsoft.Extensions.Logging; diff --git a/benchmark/Resp.benchmark/ClientTypes.cs b/benchmark/Resp.benchmark/Common/ClientTypes.cs similarity index 85% rename from benchmark/Resp.benchmark/ClientTypes.cs rename to benchmark/Resp.benchmark/Common/ClientTypes.cs index 827e55f59c5..1dd268707ba 100644 --- a/benchmark/Resp.benchmark/ClientTypes.cs +++ b/benchmark/Resp.benchmark/Common/ClientTypes.cs @@ -8,6 +8,7 @@ public enum ClientType : byte LightClient, SERedis, GarnetClientSession, - GarnetClient + GarnetClient, + InProc } } \ No newline at end of file diff --git a/benchmark/Resp.benchmark/GeoUtils.cs b/benchmark/Resp.benchmark/Common/GeoUtils.cs similarity index 99% rename from benchmark/Resp.benchmark/GeoUtils.cs rename to benchmark/Resp.benchmark/Common/GeoUtils.cs index 4821663d498..32826ad3d17 100644 --- a/benchmark/Resp.benchmark/GeoUtils.cs +++ b/benchmark/Resp.benchmark/Common/GeoUtils.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; - namespace Resp.benchmark { internal static class GeoUtils diff --git a/benchmark/Resp.benchmark/Common/HashUtils.cs b/benchmark/Resp.benchmark/Common/HashUtils.cs new file mode 100644 index 00000000000..c8d6e6168e0 --- /dev/null +++ b/benchmark/Resp.benchmark/Common/HashUtils.cs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +namespace Resp.benchmark +{ + /// + /// Utility class for generating stable hashes + /// Implementations are copied from Trill: https://github.com/microsoft/Trill/blob/master/Sources/Core/Microsoft.StreamProcessing/Utilities/Utility.cs + /// + internal static class HashUtils + { + /// + /// Generate a stable hashcode for input string. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int StableHash(this string stringToHash) + { + unsafe + { + fixed (char* str = stringToHash) + { + return StableHashUnsafe(str, stringToHash.Length); + } + } + } + + /// + /// Stable hash implementations. + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe int StableHashUnsafe(char* stringToHash, int length) + { + const long magicno = 40343L; + ulong hashState = (ulong)length; + var stringChars = stringToHash; + for (int i = 0; i < length; i++, stringChars++) + hashState = magicno * hashState + *stringChars; + + var rotate = magicno * hashState; + var rotated = (rotate >> 4) | (rotate << 60); + return (int)(rotated ^ (rotated >> 32)); + } + + public static unsafe ulong MurmurHash2x64A(Span bString, uint seed = 0) + { + fixed (byte* p = bString) + { + return MurmurHash2x64A(p, bString.Length, seed); + } + } + + /// + /// MurmurHash2 Get 64-bit hash code for a byte array + /// + /// + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe ulong MurmurHash2x64A(byte* bString, int len, uint seed = 0) + { + ulong m = (ulong)0xc6a4a7935bd1e995; + int r = 47; + ulong h = seed ^ ((ulong)len * m); + byte* data = bString; + byte* end = data + (len - (len & 7)); + + while (data != end) + { + ulong k; + k = (ulong)data[0]; + k |= (ulong)data[1] << 8; + k |= (ulong)data[2] << 16; + k |= (ulong)data[3] << 24; + k |= (ulong)data[4] << 32; + k |= (ulong)data[5] << 40; + k |= (ulong)data[6] << 48; + k |= (ulong)data[7] << 56; + + k *= m; + k ^= k >> r; + k *= m; + h ^= k; + h *= m; + + data += 8; + } + + int cs = len & 7; + + if (cs >= 7) + h ^= ((ulong)data[6] << 48); + + if (cs >= 6) + h ^= ((ulong)data[5] << 40); + + if (cs >= 5) + h ^= ((ulong)data[4] << 32); + + if (cs >= 4) + h ^= ((ulong)data[3] << 24); + + if (cs >= 3) + h ^= ((ulong)data[2] << 16); + + if (cs >= 2) h ^= ((ulong)data[1] << 8); + if (cs >= 1) + { + h ^= (ulong)data[0]; + h *= m; + } + + h ^= h >> r; + h *= m; + h ^= h >> r; + return h; + } + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/NumUtils.cs b/benchmark/Resp.benchmark/Common/NumUtils.cs similarity index 100% rename from benchmark/Resp.benchmark/NumUtils.cs rename to benchmark/Resp.benchmark/Common/NumUtils.cs diff --git a/benchmark/Resp.benchmark/OpType.cs b/benchmark/Resp.benchmark/Common/OpType.cs similarity index 100% rename from benchmark/Resp.benchmark/OpType.cs rename to benchmark/Resp.benchmark/Common/OpType.cs diff --git a/benchmark/Resp.benchmark/PeriodicCheckpointer.cs b/benchmark/Resp.benchmark/Common/PeriodicCheckpointer.cs similarity index 93% rename from benchmark/Resp.benchmark/PeriodicCheckpointer.cs rename to benchmark/Resp.benchmark/Common/PeriodicCheckpointer.cs index a97c7340a22..01b2687d68d 100644 --- a/benchmark/Resp.benchmark/PeriodicCheckpointer.cs +++ b/benchmark/Resp.benchmark/Common/PeriodicCheckpointer.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.Threading; using Microsoft.Extensions.Logging; using StackExchange.Redis; @@ -26,7 +24,8 @@ public void Start(string address, int port) var server = redis.GetServer(address, port); while (true) { - if (cts.IsCancellationRequested) break; + if (cts.IsCancellationRequested) + break; Thread.Sleep(periodMs); try { diff --git a/benchmark/Resp.benchmark/RandomGenerator.cs b/benchmark/Resp.benchmark/Common/RandomGenerator.cs similarity index 100% rename from benchmark/Resp.benchmark/RandomGenerator.cs rename to benchmark/Resp.benchmark/Common/RandomGenerator.cs diff --git a/benchmark/Resp.benchmark/ZipfGenerator.cs b/benchmark/Resp.benchmark/Common/ZipfGenerator.cs similarity index 95% rename from benchmark/Resp.benchmark/ZipfGenerator.cs rename to benchmark/Resp.benchmark/Common/ZipfGenerator.cs index 5e8e1019918..f0201325a1a 100644 --- a/benchmark/Resp.benchmark/ZipfGenerator.cs +++ b/benchmark/Resp.benchmark/Common/ZipfGenerator.cs @@ -1,8 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; - namespace Resp.benchmark { public class ZipfGenerator @@ -29,7 +27,7 @@ public ZipfGenerator(RandomGenerator rng, int size, double theta = 0.99) private static double Zeta(int count, double theta) { double zetaN = 0.0; - for (var ii = 1; ii <= count; ++ii) + for (var ii = 1; ii <= count; ii++) zetaN += 1.0 / Math.Pow(ii, theta); return zetaN; } diff --git a/benchmark/Resp.benchmark/HashUtils.cs b/benchmark/Resp.benchmark/HashUtils.cs deleted file mode 100644 index 4c7bf4f5761..00000000000 --- a/benchmark/Resp.benchmark/HashUtils.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.CompilerServices; - -namespace Resp.benchmark -{ - /// - /// Utility class for generating stable hashes - /// Implementations are copied from Trill: https://github.com/microsoft/Trill/blob/master/Sources/Core/Microsoft.StreamProcessing/Utilities/Utility.cs - /// - internal static class HashUtils - { - /// - /// Generate a stable hashcode for input string. - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int StableHash(this string stringToHash) - { - unsafe - { - fixed (char* str = stringToHash) - { - return StableHashUnsafe(str, stringToHash.Length); - } - } - } - - /// - /// Stable hash implementations. - /// - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe int StableHashUnsafe(char* stringToHash, int length) - { - const long magicno = 40343L; - ulong hashState = (ulong)length; - var stringChars = stringToHash; - for (int i = 0; i < length; i++, stringChars++) - hashState = magicno * hashState + *stringChars; - - var rotate = magicno * hashState; - var rotated = (rotate >> 4) | (rotate << 60); - return (int)(rotated ^ (rotated >> 32)); - } - } -} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBench.cs b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBench.cs new file mode 100644 index 00000000000..eb3ff820215 --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBench.cs @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Net; +using System.Text; +using Garnet.server; +using Tsavorite.core; + +namespace Resp.benchmark +{ + public class AofBench + { + public static GarnetServerOptions GetServerOptions(Options options) + { + var serverOptions = new GarnetServerOptions + { + ClusterAnnounceEndpoint = new IPEndPoint(IPAddress.Loopback, 6379), + QuietMode = true, + IndexMemorySize = options.IndexMemorySize, + EnableAOF = options.EnableAOF || options.AofBench, + EnableCluster = options.EnableCluster, + ClusterConfigFlushFrequencyMs = -1, + FastAofTruncate = options.EnableCluster && options.UseAofNullDevice, + UseAofNullDevice = options.UseAofNullDevice, + AofMemorySize = options.AofMemorySize, + AofPageSize = options.AofPageSize, + CommitFrequencyMs = options.CommitFrequencyMs, + AofPhysicalSublogCount = options.AofPhysicalSublogCount, + AofReplayTaskCount = options.AofReplayTaskCount, + ReplicationOffsetMaxLag = 0, + CheckpointDir = OperatingSystem.IsLinux() ? "/tmp" : null, + }; + return serverOptions; + } + + readonly ManualResetEventSlim waiter = new(); + readonly Options options; + readonly AofGen aofGen; + readonly AofReplayStream[] aofReplayStream; + StringBuilder stats = new(); + long total_bytes_processed = 0; + long total_pages_processed = 0; + long total_records_replayed = 0; + long total_records_enqueued = 0; + + volatile bool done = false; + + AofAddress aofTailAddress; + readonly LightEpoch epoch; + + public AofBench(Options options) + { + this.options = options; + + var replayEnabled = options.AofBenchType is AofBenchType.Replay or AofBenchType.ReplayNoResp; + if (!options.EnableCluster && options.AofBenchType == AofBenchType.Replay) + throw new Exception("InProc/AofBench with AofBenchType.Replay requires --cluster!"); + + var serverOptions = GetServerOptions(options); + aofGen = new AofGen(options); + + if (options.IsReplayEnabled) + { + options.EnableCluster = true; + var instance = new GarnetServerInstance(options); + aofGen.primaryId = instance.primaryId; + aofReplayStream = [.. Enumerable.Range(0, options.AofPhysicalSublogCount).Select( + x => new AofReplayStream(instance, threadId: x, startAddress: 64, options))]; + } + else + { + epoch = new LightEpoch(); + } + } + + public void GenerateData() => aofGen.GenerateData(); + + public void Run(int threads) + { + var workers = new Thread[threads]; + + Console.WriteLine($"Epoch instance count:{LightEpoch.ActiveInstanceCount()}"); + + try + { + var msg = options.AofBenchType switch + { + AofBenchType.Replay or AofBenchType.ReplayNoResp or AofBenchType.ReplayDirect => $">>> Running {options.AofBenchType} using {threads}x{options.AofReplayTaskCount} worker(s) >>>", + AofBenchType.EnqueueSharded or AofBenchType.EnqueueRandom => $">>> Running {options.AofBenchType} using {threads} worker(s) >>>", + _ => throw new Exception($"AofBenchType {options.AofBenchType} not supported"), + }; + Console.WriteLine(msg); + + if (options.IsReplayEnabled) + aofTailAddress = aofGen.appendOnlyFile.Log.TailAddress; + + // Run the experiment. + for (var idx = 0; idx < threads; ++idx) + { + var x = idx; + workers[idx] = options.AofBenchType switch + { + AofBenchType.Replay => new Thread(() => RunAofReplayBench(x)), + AofBenchType.ReplayNoResp => new Thread(() => RunAofReplayBenchNoResp(x)), + AofBenchType.ReplayDirect => new Thread(() => RunAofReplayBenchDirect(x)), + AofBenchType.EnqueueSharded or AofBenchType.EnqueueRandom => new Thread(() => RunAofEnqueueBench(x)), + _ => throw new Exception($"AofBenchType {options.AofBenchType} not supported"), + }; + } + + // Start threads. + foreach (var worker in workers) + worker.Start(); + + waiter.Set(); + + Stopwatch swatch = new(); + swatch.Start(); + // Let workers operate for a specific RunTime + Thread.Sleep(TimeSpan.FromSeconds(options.RunTime)); + done = true; + + // Wait for AOF load to complete + foreach (var worker in workers) + worker.Join(); + + swatch.Stop(); + + var seconds = swatch.ElapsedMilliseconds / 1000.0; + if (options.IsReplayEnabled) + { + var bytesPerSecond = (total_bytes_processed / seconds) / (double)1_000_000_000; + var recordsReplayedPerSecond = total_records_replayed / seconds; + Console.WriteLine($"[Total time]: {swatch.ElapsedMilliseconds:N2} ms for {total_bytes_processed:N0} AOF bytes"); + Console.WriteLine($"[Bandwidth]: {bytesPerSecond:N2} GiB/sec"); + Console.WriteLine($"[Total pages send]: {total_pages_processed:N0}"); + Console.WriteLine($"[Total records replayed]: {total_records_replayed:N0}"); + Console.WriteLine($"[Throughput]: {recordsReplayedPerSecond:N2} records/sec"); + } + else + { + var bytesPerSecond = (total_bytes_processed / seconds) / (double)1_000_000_000; + var recordsEnqueuedPerSecond = total_records_enqueued / seconds; + Console.WriteLine($"[Total time]: {swatch.ElapsedMilliseconds:N2} ms for {total_bytes_processed:N0} AOF bytes"); + Console.WriteLine($"[Bandwidth]: {bytesPerSecond:N2} GiB/sec"); + Console.WriteLine($"[Total records enqueued]: {total_records_enqueued:N0}"); + Console.WriteLine($"[Throughput]: {recordsEnqueuedPerSecond:N2} records/sec"); + } + } + finally + { + done = false; + total_records_replayed = 0; + total_records_enqueued = 0; + total_bytes_processed = 0; + waiter.Reset(); + Console.WriteLine("------------------------------"); + } + } + + unsafe void RunAofEnqueueBench(int threadId) + { + waiter.Wait(); + var kvPairs = aofGen.GetKVPairBuffer(threadId); + var recordsEnqueued = 0L; + var bytesEnqueued = 0L; + while (!done) + { + for (var i = 0; i < kvPairs.Count; i++) + { + if (done) break; + var kvPair = kvPairs[i]; + var kb = kvPair.Item1; + var vb = kvPair.Item2; + fixed (byte* keyPtr = kb) + fixed (byte* valPtr = vb) + { + var key = SpanByte.FromPinnedPointer(keyPtr, kb.Length); + var value = SpanByte.FromPinnedPointer(valPtr, vb.Length); + StringInput input = default; + aofGen.appendOnlyFile.Log.Enqueue( + AofEntryType.StoreUpsert, + 1, + threadId, + key, + value, + ref input, + epoch, + out _); + bytesEnqueued += sizeof(AofShardedHeader) + key.TotalSize() + value.TotalSize() + input.SerializedLength; + } + recordsEnqueued++; + } + + if (done) break; + } + //Console.WriteLine($"[{threadId}] - Enqueued: {recordsEnqueued:N0} records"); + _ = Interlocked.Add(ref total_records_enqueued, recordsEnqueued); + _ = Interlocked.Add(ref total_bytes_processed, bytesEnqueued); + } + + unsafe void RunAofReplayBench(int threadId) + { + var messages = aofGen.GetRespReplayMessages(threadId); + var offset = 0; + var pagesSend = 0L; + var totalBytes = 0L; + var recordsReplayedCount = 0L; + var pageSize = 1 << options.AofPageSizeBits(); + + // Track monotonically increasing addresses for circular replay + var previousAddress = 64L; + var currentAddress = 64L; + + waiter.Wait(); + + // Initialize stream for replay + aofReplayStream[threadId].InitializeReplayStream(); + + while (!done) + { + var pos = offset++ % messages.Length; + var msg = messages[pos]; + var nextAddress = currentAddress + msg.payloadLength; + + fixed (byte* ptr = msg.buffer) + { + // Update fixed-width address fields in-place for the current replay position. + // Addresses are zero-padded to max digits during generation (see WriterClusterAppendLogFixedWidth) + // so overwriting here does not change message length. + AofGen.PatchAddress(ptr, msg.previousAddressDigitOffset, previousAddress); + AofGen.PatchAddress(ptr, msg.currentAddressDigitOffset, currentAddress); + AofGen.PatchAddress(ptr, msg.nextAddressDigitOffset, nextAddress); + + aofReplayStream[threadId].ConsumeResp(ptr + msg.messageOffset, msg.messageLength); + + pagesSend++; + totalBytes += msg.payloadLength; + recordsReplayedCount += msg.recordCount; + } + + previousAddress = nextAddress; + currentAddress = currentAddress == 64 ? pageSize : currentAddress + pageSize; + } + + //Console.WriteLine($"[{threadId}] - Pages send: {pagesSend:N0}, Total AOF bytes send: {totalBytes:N0}, Total records replayed:{recordsReplayedCount:N0}"); + _ = Interlocked.Add(ref total_pages_processed, pagesSend); + _ = Interlocked.Add(ref total_bytes_processed, totalBytes); + _ = Interlocked.Add(ref total_records_replayed, recordsReplayedCount); + } + + unsafe void RunAofReplayBenchNoResp(int threadId) + { + var buffers = aofGen.GetPageBuffers(threadId); + var offset = 0; + var currentAddress = 64L; + var nextAddress = 64L; + var pagesSend = 0L; + var totalBytes = 0L; + var recordsReplayedCount = 0L; + + waiter.Wait(); + + // Initialize stream for replay + aofReplayStream[threadId].InitializeReplayStream(); + + while (!done) + { + var pos = offset++ % buffers.Length; + var currPage = buffers[pos]; + fixed (byte* payloadPtr = currPage.payload) + { + nextAddress = currentAddress + currPage.payloadLength; + aofReplayStream[threadId].ConsumeNoResp(payloadPtr, currPage.payloadLength, currentAddress, nextAddress, isProtected: false); + + // First page has a valid address from 64. + // After that currentAddress starts from beginning of bage (i.e. multiple of page size) + currentAddress = currentAddress == 64 ? currPage.Length : currentAddress + currPage.Length; + pagesSend++; + totalBytes += currPage.payloadLength; + recordsReplayedCount += currPage.recordCount; + } + } + + //Console.WriteLine($"[{threadId}] - Pages send: {pagesSend:N0}, Total AOF bytes send: {totalBytes:N0}, Total records replayed:{recordsReplayedCount:N0}"); + _ = Interlocked.Add(ref total_pages_processed, pagesSend); + _ = Interlocked.Add(ref total_bytes_processed, totalBytes); + _ = Interlocked.Add(ref total_records_replayed, recordsReplayedCount); + } + + unsafe void RunAofReplayBenchDirect(int threadId) + { + var buffers = aofGen.GetPageBuffers(threadId); + var offset = 0; + var currentAddress = 64L; + var nextAddress = 64L; + var pagesSend = 0L; + var totalBytes = 0L; + var recordsReplayedCount = 0L; + + waiter.Wait(); + + // Initialize stream for replay + aofReplayStream[threadId].InitializeReplayStream(); + + while (!done) + { + var pos = offset++ % buffers.Length; + var currPage = buffers[pos]; + fixed (byte* payloadPtr = currPage.payload) + { + nextAddress = currentAddress + currPage.payloadLength; + aofReplayStream[threadId].ConsumeDirect(payloadPtr, currPage.payloadLength, currentAddress, nextAddress, isProtected: false); + + // First page has a valid address from 64. + // After that currentAddress starts from beginning of bage (i.e. multiple of page size) + currentAddress = currentAddress == 64 ? currPage.Length : currentAddress + currPage.Length; + pagesSend++; + totalBytes += currPage.payloadLength; + recordsReplayedCount += currPage.recordCount; + } + } + + //Console.WriteLine($"[{threadId}] - Pages send: {pagesSend:N0}, Total AOF bytes send: {totalBytes:N0}, Total records replayed:{recordsReplayedCount:N0}"); + _ = Interlocked.Add(ref total_pages_processed, pagesSend); + _ = Interlocked.Add(ref total_bytes_processed, totalBytes); + _ = Interlocked.Add(ref total_records_replayed, recordsReplayedCount); + } + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBenchType.cs b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBenchType.cs new file mode 100644 index 00000000000..b5b9c1fed9e --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofBenchType.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Resp.benchmark +{ + public enum AofBenchType + { + /// + /// Enqueue to sublog randomly. + /// + EnqueueRandom, + /// + /// Enqueue to sublog in a sharded manner. + /// + EnqueueSharded, + /// + /// Simulate AOF replay. + /// + Replay, + /// + /// Simulate AOF replay (skipping resp parsing) + /// + ReplayNoResp, + /// + /// Simulate AOF replay (skip enqueue) + /// + ReplayDirect, + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofGen.cs b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofGen.cs new file mode 100644 index 00000000000..47bd0c44a22 --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofGen.cs @@ -0,0 +1,489 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Text; +using Garnet.common; +using Garnet.server; +using Tsavorite.core; + +namespace Resp.benchmark +{ + public class Page(int size) + { + public int Length => payload.Length; + public byte[] payload = GC.AllocateArray(size, pinned: true); + public int payloadLength = 0; + public int recordCount = 0; + } + + public class RespReplayMessage(int size) + { + public byte[] buffer = GC.AllocateArray(size, pinned: true); + public int messageOffset = 0; + public int messageLength = 0; + public int payloadOffset = 0; + public int payloadLength = 0; + public int recordCount = 0; + + // Byte offsets into buffer where fixed-width address digit fields begin. + // Used for in-place patching of addresses during circular replay. + public int previousAddressDigitOffset; + public int currentAddressDigitOffset; + public int nextAddressDigitOffset; + } + + public sealed class AofGen + { + /// + /// Maximum size of the RESP header for CLUSTER APPENDLOG (args 1-7 + bulk string header). + /// Generous upper bound to accommodate fixed-width address fields and nodeId. + /// + const int MaxRespHeaderSize = 512; + + /// + /// Maximum digits for a non-negative long address value (long.MaxValue = 19 digits). + /// Addresses are always zero-padded to this width to allow in-place patching during circular replay. + /// + const int MaxAddressDigits = 19; + readonly GarnetLog garnetLog; + + /// + /// Writes a complete RESP-formatted CLUSTER APPENDLOG message including the payload. + /// + internal static unsafe int WriterClusterAppendLog( + byte* bufferPtr, + int bufferLength, + string nodeId, + int physicalSublogIdx, + long previousAddress, + long currentAddress, + long nextAddress, + long payloadPtr, + int payloadLength) + { + var CLUSTER = "$7\r\nCLUSTER\r\n"u8; + var appendLog = "APPENDLOG"u8; + + var curr = bufferPtr; + var end = bufferPtr + bufferLength; + + var arraySize = 8; + + // + if (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 1 + if (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 2 + if (!RespWriteUtils.TryWriteBulkString(appendLog, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 3 + if (!RespWriteUtils.TryWriteAsciiBulkString(nodeId, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 4 + if (!RespWriteUtils.TryWriteArrayItem(physicalSublogIdx, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 5 + if (!RespWriteUtils.TryWriteArrayItem(previousAddress, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 6 + if (!RespWriteUtils.TryWriteArrayItem(currentAddress, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 7 + if (!RespWriteUtils.TryWriteArrayItem(nextAddress, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + // 8 + if (!RespWriteUtils.TryWriteBulkString(new Span((void*)payloadPtr, payloadLength), ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + + return (int)(curr - bufferPtr); + } + + /// + /// Like but allocates maximum space for address values + /// (zero-padded to digits) so they can be patched in-place + /// during circular replay. Records the digit offsets in . + /// + internal static unsafe int WriterClusterAppendLogFixedWidth( + byte* bufferPtr, + int bufferLength, + string nodeId, + int physicalSublogIdx, + long previousAddress, + long currentAddress, + long nextAddress, + long payloadPtr, + int payloadLength, + RespReplayMessage msg) + { + var CLUSTER = "$7\r\nCLUSTER\r\n"u8; + var appendLog = "APPENDLOG"u8; + + var curr = bufferPtr; + var end = bufferPtr + bufferLength; + + if (!RespWriteUtils.TryWriteArrayLength(8, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + if (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + if (!RespWriteUtils.TryWriteBulkString(appendLog, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + if (!RespWriteUtils.TryWriteAsciiBulkString(nodeId, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + if (!RespWriteUtils.TryWriteArrayItem(physicalSublogIdx, ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + + // Write address fields with fixed-width zero-padded digits to allow in-place patching + msg.previousAddressDigitOffset = WriteFixedWidthAddress(previousAddress, ref curr, end, bufferPtr); + msg.currentAddressDigitOffset = WriteFixedWidthAddress(currentAddress, ref curr, end, bufferPtr); + msg.nextAddressDigitOffset = WriteFixedWidthAddress(nextAddress, ref curr, end, bufferPtr); + + if (!RespWriteUtils.TryWriteBulkString(new Span((void*)payloadPtr, payloadLength), ref curr, end)) + throw new GarnetException("Not enough space in buffer"); + + return (int)(curr - bufferPtr); + } + + /// + /// Writes a long as a fixed-width RESP bulk string: $19\r\n{zero-padded digits}\r\n. + /// Returns the byte offset from where the digit field begins. + /// + static unsafe int WriteFixedWidthAddress(long value, ref byte* curr, byte* end, byte* bufferStart) + { + // "$19\r\n" (5 bytes) + 19 digits + "\r\n" (2 bytes) = 26 bytes + const int totalLen = 5 + MaxAddressDigits + 2; + if (totalLen > (int)(end - curr)) + throw new GarnetException("Not enough space in buffer"); + + *curr++ = (byte)'$'; + *curr++ = (byte)'1'; + *curr++ = (byte)'9'; + *curr++ = (byte)'\r'; + *curr++ = (byte)'\n'; + + var digitOffset = (int)(curr - bufferStart); + WriteZeroPaddedInt64(value, curr, MaxAddressDigits); + curr += MaxAddressDigits; + + *curr++ = (byte)'\r'; + *curr++ = (byte)'\n'; + + return digitOffset; + } + + /// + /// Overwrites a fixed-width address digit field in-place during circular replay. + /// + internal static unsafe void PatchAddress(byte* buffer, int digitOffset, long value) + { + WriteZeroPaddedInt64(value, buffer + digitOffset, MaxAddressDigits); + } + + static unsafe void WriteZeroPaddedInt64(long value, byte* dest, int width) + { + for (var i = width - 1; i >= 0; i--) + { + dest[i] = (byte)('0' + (value % 10)); + value /= 10; + } + } + + public readonly GarnetAppendOnlyFile appendOnlyFile; + + readonly Options options; + readonly GarnetServerOptions aofServerOptions; + + /// + /// threads x pageNum + /// + Page[][] pageBuffers; + + /// + /// threads x pageNum (RESP-formatted replay messages, only for AofBenchType.Replay) + /// + RespReplayMessage[][] respReplayMessageBuffers; + + /// + /// Primary node ID used for generating RESP replay messages + /// + internal string primaryId; + + /// + /// DBSize kv pairs + /// + List<(byte[], byte[])>[] kvPairBuffers; + + long total_number_of_aof_records = 0L; + long total_number_of_aof_bytes = 0L; + + public Page[] GetPageBuffers(int threadIdx) => pageBuffers[threadIdx]; + public RespReplayMessage[] GetRespReplayMessages(int threadIdx) => respReplayMessageBuffers[threadIdx]; + public List<(byte[], byte[])> GetKVPairBuffer(int threadIdx) => kvPairBuffers[threadIdx]; + + public AofGen(Options options) + { + this.options = options; + this.aofServerOptions = new GarnetServerOptions() + { + EnableAOF = true, + AofMemorySize = options.AofMemorySize, + AofPageSize = options.AofPageSize, + UseAofNullDevice = true, + CommitFrequencyMs = -1, + FastAofTruncate = true, + EnableCluster = true, + ReplicationOffsetMaxLag = 0, + AofPhysicalSublogCount = options.AofPhysicalSublogCount + }; + aofServerOptions.GetAofSettings(0, out var logSettings); + appendOnlyFile = new GarnetAppendOnlyFile(aofServerOptions, logSettings, Program.loggerFactory.CreateLogger("AofGen - AOF instance")); + garnetLog = appendOnlyFile.Log; + + if (options.IsReplayEnabled) + { + if (options.AofBenchType == AofBenchType.Replay) + { + respReplayMessageBuffers = new RespReplayMessage[options.AofPhysicalSublogCount][]; + } + else + { + pageBuffers = new Page[options.AofPhysicalSublogCount][]; + } + } + else + { + kvPairBuffers = new List<(byte[], byte[])>[options.NumThreads.Max()]; + } + + if (options.AofPhysicalSublogCount != options.NumThreads.Max() && options.AofBenchType == AofBenchType.EnqueueSharded) + throw new Exception("Use --threads(MAX)== --aof-sublog-count to generated perfectly sharded data!"); + } + + byte[] GetKey() => Encoding.ASCII.GetBytes(Generator.CreateHexId(size: Math.Max(options.KeyLength, 8))); + + byte[] GetKey(int threadId) + { + while (true) + { + var keyData = Encoding.ASCII.GetBytes(Generator.CreateHexId(size: Math.Max(options.KeyLength, 8))); + var physicalSublogIdx = garnetLog.GetPhysicalSublogIdx(keyData); + if (physicalSublogIdx == threadId) return keyData; + } + } + + byte[] GetValue() => Encoding.ASCII.GetBytes(Generator.CreateHexId(size: Math.Max(options.ValueLength, 8))); + + List<(byte[], byte[])> GenerateKVPairs(int threadId, bool random) + { + var kvPairs = new List<(byte[], byte[])>(); + + for (var i = 0; i < options.DbSize; i++) + { + var key = random ? GetKey() : GetKey(threadId); + var value = GetValue(); + kvPairs.Add((key, value)); + } + return kvPairs; + } + + public void GenerateData() + { + Console.WriteLine($"Generating AofBench Data!"); + var threads = options.IsReplayEnabled ? options.AofPhysicalSublogCount : options.NumThreads.Max(); + var workers = new Thread[threads]; + + // Run the experiment. + for (var idx = 0; idx < threads; ++idx) + { + var x = idx; + workers[idx] = options.AofBenchType switch + { + AofBenchType.Replay or AofBenchType.ReplayNoResp or AofBenchType.ReplayDirect => new Thread(() => GeneratePages(x)), + AofBenchType.EnqueueSharded or AofBenchType.EnqueueRandom => new Thread(() => GenerateKeys(x)), + _ => throw new Exception($"AofBenchType {options.AofBenchType} not supported"), + }; + } + + Stopwatch swatch = new(); + swatch.Start(); + + // Start threads. + foreach (var worker in workers) + worker.Start(); + + // Wait for workers to complete + foreach (var worker in workers) + worker.Join(); + + swatch.Stop(); + + var seconds = swatch.ElapsedMilliseconds / 1000.0; + if (options.IsReplayEnabled) + { + Console.WriteLine($"Generated {threads}x{options.AofGenPages} pages of size {aofServerOptions.AofPageSize} in {seconds:N2} secs"); + Console.WriteLine($"Generated number of AOF records: {total_number_of_aof_records:N0}"); + Console.WriteLine($"Generated number of AOF bytes: {total_number_of_aof_bytes:N0}"); + } + else + { + Console.WriteLine($"Generated {threads}x{options.DbSize} KV pairs in {seconds:N2} secs"); + } + } + + unsafe void GeneratePages(int threadId) + { + var seqNumGen = new SequenceNumberGenerator(0); + var number_of_aof_records = 0L; + var number_of_aof_bytes = 0L; + var kvPairs = GenerateKVPairs(threadId, options.AofPhysicalSublogCount == 1); + var pages = options.AofGenPages; + var pageSize = 1 << aofServerOptions.AofPageSizeBits(); + var generateResp = options.AofBenchType == AofBenchType.Replay; + + if (generateResp) + { + respReplayMessageBuffers[threadId] = new RespReplayMessage[pages]; + + // Simulate address progression matching RunAofReplayBench + var previousAddress = 64L; + var currentAddress = 64L; + + // Temp page used to fill data, then copied into the final RESP message + var tempPage = new Page(pageSize); + + for (var i = 0; i < pages; i++) + { + // Fill page data into temp buffer to determine payload length + tempPage.payloadLength = 0; + tempPage.recordCount = 0; + FillPage(threadId, kvPairs, i, tempPage); + + var nextAddress = currentAddress + tempPage.payloadLength; + + // Allocate RESP message buffer: header overhead + payload + trailing \r\n + var respMessage = new RespReplayMessage(MaxRespHeaderSize + tempPage.payloadLength + 2); + + fixed (byte* bufferPtr = respMessage.buffer) + fixed (byte* payloadPtr = tempPage.payload) + { + // Write RESP message with fixed-width zero-padded address fields + // to enable in-place patching during circular replay + var messageLen = WriterClusterAppendLogFixedWidth( + bufferPtr, + respMessage.buffer.Length, + nodeId: primaryId, + physicalSublogIdx: threadId, + previousAddress, + currentAddress, + nextAddress, + (long)payloadPtr, + tempPage.payloadLength, + respMessage); + + respMessage.messageOffset = 0; + respMessage.messageLength = messageLen; + respMessage.payloadLength = tempPage.payloadLength; + respMessage.recordCount = tempPage.recordCount; + } + + respReplayMessageBuffers[threadId][i] = respMessage; + + previousAddress = nextAddress; + currentAddress = currentAddress == 64 ? pageSize : currentAddress + pageSize; + } + } + else + { + pageBuffers[threadId] = new Page[pages]; + for (var i = 0; i < pages; i++) + { + pageBuffers[threadId][i] = new Page(pageSize); + FillPage(threadId, kvPairs, i, pageBuffers[threadId][i]); + } + } + + _ = Interlocked.Add(ref total_number_of_aof_records, number_of_aof_records); + _ = Interlocked.Add(ref total_number_of_aof_bytes, number_of_aof_bytes); + + void FillPage(int threadId, List<(byte[], byte[])> kvPairs, int pageCount, Page page) + { + fixed (byte* pagePtr = page.payload) + { + var pageOffset = pagePtr; + // First page starts from 64 address, so the payload space must be smaller + var pageEnd = pageOffset + page.Length - (pageCount == 0 ? 64 : 0); + var kvOffset = 0; + while (true) + { + var kvPair = kvPairs[kvOffset++ % kvPairs.Count]; + var keyData = kvPair.Item1; + var valueData = kvPair.Item2; + StringInput input = default; + fixed (byte* keyPtr = keyData) + fixed (byte* valuePtr = valueData) + { + var key = SpanByte.FromPinnedPointer(keyPtr, keyData.Length); + var value = SpanByte.FromPinnedPointer(valuePtr, valueData.Length); + var aofHeader = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = AofEntryType.StoreUpsert, + storeVersion = 1, + sessionID = 0 + }; + var useShardedHeader = options.AofPhysicalSublogCount > 1 || options.AofReplayTaskCount > 1; + if (!useShardedHeader) + { + if (!garnetLog.GetSubLog(threadId).DummyEnqueue( + ref pageOffset, + pageEnd, + aofHeader, + key, + value, + ref input)) + break; + } + else + { + var extendedAofHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedHeader, + opType = aofHeader.opType, + storeVersion = aofHeader.storeVersion, + sessionID = aofHeader.sessionID + }, + sequenceNumber = seqNumGen.GetSequenceNumber() + }; + + if (!garnetLog.GetSubLog(threadId).DummyEnqueue( + ref pageOffset, + pageEnd, + extendedAofHeader, + key, + value, + ref input)) + break; + } + page.recordCount++; + } + } + + var payloadLength = (int)(pageOffset - pagePtr); + page.payloadLength = payloadLength; + number_of_aof_records += page.recordCount; + number_of_aof_bytes += payloadLength; + } + } + } + + void GenerateKeys(int threadId) + { + kvPairBuffers[threadId] = GenerateKVPairs(threadId, options.AofBenchType == AofBenchType.EnqueueRandom); + //Console.WriteLine($"[{threadId}] - Generated {kvPairBuffers[threadId].Count} KV pairs for {options.AofBenchType}"); + } + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofReplayStream.cs b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofReplayStream.cs new file mode 100644 index 00000000000..760fb42b444 --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/AOFBench/AofReplayStream.cs @@ -0,0 +1,201 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Net; +using Embedded.server; +using Garnet.cluster; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace Resp.benchmark +{ + internal sealed class AofReplayStream : IDisposable + { + const int maxChunkSize = 1 << 20; + readonly Options options; + readonly int threadId; + readonly CancellationTokenSource cts = new(); + readonly long startAddress; + long previousAddress; + readonly ILogger logger = null; + readonly string primaryId; + + public long Size => previousAddress - startAddress; + + readonly GarnetServerInstance instance; + + // Direct replay mode (InProc): bypass RESP, call ReplicaReplayDriver directly + ReplicaReplayDriver replayDriver; + + public byte[] buffer; + + public AofReplayStream( + GarnetServerInstance instance, + int threadId, + long startAddress, + Options options) + { + this.options = options; + this.instance = instance; + this.threadId = threadId; + this.primaryId = instance.primaryId; + this.startAddress = startAddress; + previousAddress = startAddress; + buffer = GC.AllocateArray(2 << options.AofPageSizeBits(), pinned: true); + + instance.GetClusterSession(0).UnsafeSetConfig(replicaOf: primaryId); + } + + public void Dispose() + { + cts.Cancel(); + cts.Dispose(); + } + + ClusterNode GetClusterNodes(Options opts) + { + var redis = ConnectionMultiplexer.Connect( + BenchUtils.GetConfig( + opts.Address, + opts.Port, + useTLS: opts.EnableTLS, + tlsHost: opts.TlsHost, + allowAdmin: true)); + + var servers = redis.GetServers(); + if (servers.Length < 2) + throw new Exception("Too few nodes for AOF bench to run"); + + var endpoint = new IPEndPoint(IPAddress.Parse(opts.Address), opts.Port); + var primaryServer = redis.GetServer(endpoint); + var nodes = primaryServer.ClusterNodes(); + var primaryNodeId = (string)primaryServer.Execute("cluster", "myid"); + + ClusterNode replicaNode = null; + foreach (var node in nodes.Nodes) + { + if (node.ParentNodeId != null && node.ParentNodeId.Equals(primaryNodeId)) + replicaNode = node; + } + + if (replicaNode == null) + throw new Exception($"No replica found for [{endpoint}] to run AOF bench!"); + return replicaNode; + } + + public unsafe void InitializeReplayStream() + { + if (options.AofBenchType == AofBenchType.ReplayDirect) + { + // Direct mode: initialize ReplicaReplayDriver without going through RESP + var clusterProvider = (ClusterProvider)instance.server.StoreWrapper.clusterProvider; + var networkSender = new EmbeddedNetworkSender(); + clusterProvider.replicationManager.InitializeReplicaReplayDriver(threadId, networkSender); + replayDriver = clusterProvider.replicationManager.ReplicaReplayDriverStore.GetReplayDriver(threadId); + replayDriver.ResumeReplay(); + } + else + { + fixed (byte* ptr = buffer) + { + var respMessageSize = AofGen.WriterClusterAppendLog( + ptr, + buffer.Length, + nodeId: primaryId, + physicalSublogIdx: threadId, + previousAddress: -1, + currentAddress: -1, + nextAddress: -1, + payloadPtr: -1, + payloadLength: 0); + _ = instance.sessions[threadId].TryConsumeMessages(ptr, respMessageSize); + } + } + } + + public unsafe void Consume(byte* payloadPtr, int payloadLength, long currentAddress, long nextAddress, bool isProtected) + { + try + { + fixed (byte* ptr = buffer) + { + var respMessageSize = AofGen.WriterClusterAppendLog( + ptr, + buffer.Length, + nodeId: primaryId, + physicalSublogIdx: threadId, + previousAddress, + currentAddress, + nextAddress, + (long)payloadPtr, + payloadLength); + _ = instance.sessions[threadId].TryConsumeMessages(ptr, respMessageSize); + } + + previousAddress = nextAddress; + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.AofSyncTaskInfo.Consume"); + throw; + } + } + + public unsafe void ConsumeResp(byte* respMessagePtr, int respMessageLength) + { + try + { + _ = instance.sessions[threadId].TryConsumeMessages(respMessagePtr, respMessageLength); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.AofSyncTaskInfo.ConsumeResp"); + throw; + } + } + + public unsafe void ConsumeNoResp(byte* payloadPtr, int payloadLength, long currentAddress, long nextAddress, bool isProtected) + { + try + { + instance.sessions[threadId].clusterSession.ProcessPrimaryStream( + physicalSublogIdx: threadId, + payloadPtr, + payloadLength, + previousAddress, + currentAddress, + nextAddress); + + previousAddress = nextAddress; + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.AofSyncTaskInfo.ConsumeNoResp"); + throw; + } + } + + public unsafe void ConsumeDirect(byte* payloadPtr, int payloadLength, long currentAddress, long nextAddress, bool isProtected) + { + try + { + replayDriver.Consume( + payloadPtr, + payloadLength, + currentAddress, + nextAddress, + isProtected: false); + + previousAddress = nextAddress; + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.AofSyncTaskInfo.ConsumeNoResp"); + throw; + } + } + + public void Throttle() + { } + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/AOFBench/TsavoriteExtension.cs b/benchmark/Resp.benchmark/OfflineBench/AOFBench/TsavoriteExtension.cs new file mode 100644 index 00000000000..d905e24bba0 --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/AOFBench/TsavoriteExtension.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Reflection; +using Tsavorite.core; + +namespace Resp.benchmark +{ + internal static class TsavoriteExtension + { + private static readonly MethodInfo AlignMethod; + private static readonly FieldInfo HeaderSizeField; + private static readonly MethodInfo SetHeaderMethod; + + static TsavoriteExtension() + { + var type = typeof(TsavoriteLog); + var flags = BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Static; + + AlignMethod = type.GetMethod("Align", BindingFlags.NonPublic | BindingFlags.Static); + HeaderSizeField = type.GetField("headerSize", flags); + SetHeaderMethod = type.GetMethod("SetHeader", flags); + } + + /// + /// DummyEnqueue to provided buffer if there is enough space. + /// Used to simulate AOF layout + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public static unsafe bool DummyEnqueue( + this TsavoriteLog log, + ref byte* beginPageAddress, + byte* endPageAddress, + THeader userHeader, + ReadOnlySpan item1, + ReadOnlySpan item2, + ref TInput input) + where THeader : unmanaged where TInput : IStoreInput + { + var headerSize = (int)HeaderSizeField.GetValue(log); + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize() + input.SerializedLength; + var allocatedLength = headerSize + (int)AlignMethod.Invoke(null, [length]); + + if (beginPageAddress + allocatedLength > endPageAddress) + return false; + + var physicalAddress = beginPageAddress; + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item2.TotalSize(); + input.CopyTo(physicalAddress + offset, input.SerializedLength); + + SetHeaderMethod.Invoke(log, [length, (IntPtr)beginPageAddress]); + beginPageAddress += allocatedLength; + return true; + } + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/OfflineBench/GarnetServerInstance.cs b/benchmark/Resp.benchmark/OfflineBench/GarnetServerInstance.cs new file mode 100644 index 00000000000..ae1dcb7bcb3 --- /dev/null +++ b/benchmark/Resp.benchmark/OfflineBench/GarnetServerInstance.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Net; +using Embedded.server; +using Garnet.common; +using Garnet.server; + +namespace Resp.benchmark +{ + public class GarnetServerInstance + { + public static GarnetServerOptions GetServerOptions(Options options) + { + var serverOptions = new GarnetServerOptions + { + ClusterAnnounceEndpoint = new IPEndPoint(IPAddress.Loopback, 6379), + QuietMode = true, + IndexMemorySize = options.IndexMemorySize, + EnableAOF = options.EnableAOF || options.AofBench, + EnableCluster = options.EnableCluster, + ClusterConfigFlushFrequencyMs = -1, + FastAofTruncate = options.EnableCluster && options.UseAofNullDevice, + UseAofNullDevice = options.UseAofNullDevice, + AofMemorySize = options.AofMemorySize, + AofPageSize = options.AofPageSize, + CommitFrequencyMs = options.CommitFrequencyMs, + AofPhysicalSublogCount = options.AofPhysicalSublogCount, + AofReplayTaskCount = options.AofReplayTaskCount, + ReplicationOffsetMaxLag = 0, + CheckpointDir = OperatingSystem.IsLinux() ? "/tmp" : null, + }; + return serverOptions; + } + + internal EmbeddedRespServer server; + internal RespServerSession[] sessions; + internal readonly string primaryId; + + public GarnetServerInstance(Options options) + { + var serverOptions = AofBench.GetServerOptions(options); + primaryId = Generator.CreateHexId(); + server = new EmbeddedRespServer(serverOptions, Program.loggerFactory, new GarnetServerEmbedded()); + sessions = server.GetRespSessions(options.AofPhysicalSublogCount); + sessions[0].clusterSession.UnsafeSetConfig(replicaOf: primaryId); + } + + public IClusterSession GetClusterSession(int idx) + => sessions[idx].clusterSession; + + internal RespServerSession GetRespServerSession(int idx) + => sessions[idx]; + } +} \ No newline at end of file diff --git a/benchmark/Resp.benchmark/ReqGen.cs b/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs similarity index 99% rename from benchmark/Resp.benchmark/ReqGen.cs rename to benchmark/Resp.benchmark/OfflineBench/ReqGen.cs index c90f79305d4..5dccc90990b 100644 --- a/benchmark/Resp.benchmark/ReqGen.cs +++ b/benchmark/Resp.benchmark/OfflineBench/ReqGen.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.Collections.Generic; -using System.Threading; using Garnet.common; namespace Resp.benchmark @@ -306,6 +303,7 @@ private void RandomString() { for (int i = 0; i < valueBuffer.Length; i++) { + // The first digit of the output string should not be zero. if (i == 0) valueBuffer[i] = ascii_digits[1 + valueRandomGen.Next(ascii_digits.Length - 1)]; else diff --git a/benchmark/Resp.benchmark/ReqGenLoadBuffers.cs b/benchmark/Resp.benchmark/OfflineBench/ReqGenLoadBuffers.cs similarity index 99% rename from benchmark/Resp.benchmark/ReqGenLoadBuffers.cs rename to benchmark/Resp.benchmark/OfflineBench/ReqGenLoadBuffers.cs index fd452599df1..c7ddbc41d9f 100644 --- a/benchmark/Resp.benchmark/ReqGenLoadBuffers.cs +++ b/benchmark/Resp.benchmark/OfflineBench/ReqGenLoadBuffers.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; namespace Resp.benchmark diff --git a/benchmark/Resp.benchmark/ReqGenUtils.cs b/benchmark/Resp.benchmark/OfflineBench/ReqGenUtils.cs similarity index 100% rename from benchmark/Resp.benchmark/ReqGenUtils.cs rename to benchmark/Resp.benchmark/OfflineBench/ReqGenUtils.cs diff --git a/benchmark/Resp.benchmark/RespPerfBench.cs b/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs similarity index 76% rename from benchmark/Resp.benchmark/RespPerfBench.cs rename to benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs index e691091775e..01be17ad037 100644 --- a/benchmark/Resp.benchmark/RespPerfBench.cs +++ b/benchmark/Resp.benchmark/OfflineBench/RespPerfBench.cs @@ -1,16 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Net; using System.Text; -using System.Threading; -using System.Threading.Tasks; +using Embedded.server; using Garnet.client; using Garnet.common; +using Garnet.server; using StackExchange.Redis; namespace Resp.benchmark @@ -25,6 +22,8 @@ public partial class RespPerfBench readonly ManualResetEventSlim waiter = new(); readonly Options opts; readonly IConnectionMultiplexer redis; + internal EmbeddedRespServer server; + internal RespServerSession[] sessions; KeyValuePair[] database; @@ -33,7 +32,7 @@ public partial class RespPerfBench volatile bool done = false; long total_ops_done = 0; - + long total_bytes_consumed = 0; public RespPerfBench(Options opts, int Start, IConnectionMultiplexer redis) { @@ -41,6 +40,32 @@ public RespPerfBench(Options opts, int Start, IConnectionMultiplexer redis) this.Start = Start; if (opts.Client == ClientType.SERedis) this.redis = redis; + + if (opts.Client == ClientType.InProc) + { + if (opts.EnableCluster && !opts.SkipLoad && !opts.LSet) + throw new Exception("Use --lset when running InProc and with cluster enabled to load data!"); + + var serverOptions = AofBench.GetServerOptions(opts); + server = new EmbeddedRespServer(serverOptions, Program.loggerFactory, new GarnetServerEmbedded()); + sessions = server.GetRespSessions(opts.NumThreads.Max()); + + if (opts.EnableCluster) + { + AddSlotRange([(0, 16383)]); + unsafe void AddSlotRange(List<(int, int)> slotRanges) + { + foreach (var slotRange in slotRanges) + { + var clusterAddSlotsRange = Encoding.ASCII.GetBytes($"*4\r\n$7\r\nCLUSTER\r\n$13\r\nADDSLOTSRANGE\r\n" + + $"${Garnet.common.NumUtils.CountDigits(slotRange.Item1)}\r\n{slotRange.Item1}\r\n" + + $"${Garnet.common.NumUtils.CountDigits(slotRange.Item2)}\r\n{slotRange.Item2}\r\n"); + fixed (byte* req = clusterAddSlotsRange) + _ = sessions[0].TryConsumeMessages(req, clusterAddSlotsRange.Length); + } + } + } + } } /// @@ -59,6 +84,13 @@ public void LoadData( int valueLen = default, bool numericValue = false) { + if (opts.Client == ClientType.InProc && loadDbThreads > sessions.Length) + { + foreach (var session in sessions) + session.Dispose(); + sessions = server.GetRespSessions(loadDbThreads); + } + if (load_rg != null) opts.DbSize = load_rg.DbSize; @@ -76,7 +108,8 @@ public void LoadData( LightOperate(OpType.SET, opts.DbSize, loadBatchSize, loadDbThreads, opts.DbSize / loadDbThreads, default, load_rg, false, false, keyLen, valueLen, numericValue: numericValue); load_rg = null; - GetDBSIZE(loadDbThreads); + if (opts.Client != ClientType.InProc) + GetDBSIZE(loadDbThreads); } private unsafe void GetDBSIZE(int loadDbThreads) @@ -209,7 +242,7 @@ public void PerformMGET(int NumOps, int NumThreads, int BatchSize = 1 << 12) Thread[] workers = new Thread[NumThreads]; // Run the experiment. - for (int idx = 0; idx < NumThreads; ++idx) + for (int idx = 0; idx < NumThreads; idx++) { int x = idx; workers[idx] = new Thread(() => MGetThreadRunner(x, NumOps, BatchSize)); @@ -320,24 +353,29 @@ public ReqGen LightOperate( } // Query database - Thread[] workers = new Thread[NumThreads]; + var workers = new Thread[NumThreads]; // Run the experiment. - for (int idx = 0; idx < NumThreads; ++idx) + for (var idx = 0; idx < NumThreads; idx++) { - int x = idx; + var x = idx; workers[idx] = opts.Client switch { - ClientType.LightClient => new Thread(() => LightOperateThreadRunner(OpsPerThread, opType, rg)), + ClientType.LightClient => new Thread(() => LightOperateThreadRunner(x, OpsPerThread, opType, rg)), ClientType.GarnetClientSession => new Thread(() => GarnetClientSessionOperateThreadRunner(OpsPerThread, opType, rg)), ClientType.SERedis => new Thread(() => SERedisOperateThreadRunner(OpsPerThread, opType, rg)), + ClientType.InProc => new Thread(() => InProcOperateThreadRunner(x, OpsPerThread, opType, rg)), _ => throw new Exception($"ClientType {opts.Client} not supported"), }; } + AofAddress beginAddress = default; + if (opts.Client == ClientType.InProc && server.StoreWrapper.appendOnlyFile != null) + beginAddress = server.StoreWrapper.appendOnlyFile.Log.TailAddress; + // Start threads. - foreach (Thread worker in workers) + foreach (var worker in workers) worker.Start(); waiter.Set(); @@ -350,28 +388,43 @@ public ReqGen LightOperate( Thread.Sleep(runTime); done = true; } - foreach (Thread worker in workers) + foreach (var worker in workers) worker.Join(); swatch.Stop(); - double seconds = swatch.ElapsedMilliseconds / 1000.0; - double opsPerSecond = total_ops_done / seconds; + var seconds = swatch.ElapsedMilliseconds / 1000.0; + var opsPerSecond = total_ops_done / seconds; + var byteConsumerPerSecond = (total_bytes_consumed / seconds) / (double)1_000_000_000; if (verbose) { - Console.WriteLine($"Total time: {swatch.ElapsedMilliseconds:N2}ms for {total_ops_done:N2} ops"); - Console.WriteLine($"Throughput: {opsPerSecond:N2} ops/sec"); + Console.WriteLine($"[Total time]: {swatch.ElapsedMilliseconds:N2}ms for {total_ops_done:N2} ops"); + Console.WriteLine($"[Throughput]: {opsPerSecond:N2} ops/sec"); + if (ClientType.InProc == opts.Client) + { + Console.WriteLine($"[BytesConsumed]: {total_bytes_consumed:N0} bytes"); + Console.WriteLine($"[BytesConsumedPerSecond]: {byteConsumerPerSecond:N2} GiB/sec"); + if (server.StoreWrapper.appendOnlyFile != null) + { + var tailAddress = sessions[0].StoreWrapper.TailAddress; + var aofSize = tailAddress.AggregateDiff(beginAddress); + var tpt = (aofSize / seconds) / (double)1_000_000_000; + Console.WriteLine($"[AOF Total Size]: {aofSize:N2} bytes"); + Console.WriteLine($"[AOF Append Tpt]: {tpt:N2} GiB/sec"); + } + } } done = false; total_ops_done = 0; + total_bytes_consumed = 0; waiter.Reset(); return rg; } - private unsafe void LightOperateThreadRunner(int NumOps, OpType opType, ReqGen rg) + private unsafe void LightOperateThreadRunner(int threadId, int NumOps, OpType opType, ReqGen rg) { var lighClientOnResponseDelegate = new LightClient.OnResponseDelegateUnsafe(ReqGen.OnResponse); using ClientBase client = new LightClient(new IPEndPoint(IPAddress.Parse(opts.Address), opts.Port), (int)opType, lighClientOnResponseDelegate, rg.GetBufferSize(), opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null); @@ -379,8 +432,8 @@ private unsafe void LightOperateThreadRunner(int NumOps, OpType opType, ReqGen r client.Connect(); client.Authenticate(opts.Auth); - int maxReqs = (NumOps / rg.BatchCount); - int numReqs = 0; + var maxReqs = (NumOps / rg.BatchCount); + var numReqs = 0; waiter.Wait(); @@ -388,7 +441,7 @@ private unsafe void LightOperateThreadRunner(int NumOps, OpType opType, ReqGen r sw.Start(); while (!done) { - byte[] buf = rg.GetRequest(out int len); + byte[] buf = rg.GetRequest(out var len); client.Send(buf, len, (opType == OpType.MSET || opType == OpType.MPFADD) ? 1 : rg.BatchCount); client.CompletePendingRequests(); numReqs++; @@ -416,8 +469,8 @@ private void GarnetClientSessionOperateThreadRunner(int NumOps, OpType opType, R c.CompletePending(); } - int maxReqs = NumOps / rg.BatchCount; - int numReqs = 0; + var maxReqs = NumOps / rg.BatchCount; + var numReqs = 0; waiter.Wait(); @@ -448,8 +501,8 @@ private void SERedisOperateThreadRunner(int NumOps, OpType opType, ReqGen rg) } var db = redis.GetDatabase(0); - int maxReqs = NumOps / rg.BatchCount; - int numReqs = 0; + var maxReqs = NumOps / rg.BatchCount; + var numReqs = 0; waiter.Wait(); @@ -458,7 +511,7 @@ private void SERedisOperateThreadRunner(int NumOps, OpType opType, ReqGen rg) while (!done) { var reqArgs = rg.GetRequestArgs(); - for (int i = 0; i < reqArgs.Count; i += 2) + for (var i = 0; i < reqArgs.Count; i += 2) db.StringSet(reqArgs[i], reqArgs[i + 1]); numReqs++; if (numReqs == maxReqs) break; @@ -468,22 +521,48 @@ private void SERedisOperateThreadRunner(int NumOps, OpType opType, ReqGen rg) Interlocked.Add(ref total_ops_done, numReqs * rg.BatchCount); } + private unsafe void InProcOperateThreadRunner(int threadId, int NumOps, OpType opType, ReqGen rg) + { + var maxReqs = NumOps / rg.BatchCount; + var numReqs = 0; + + waiter.Wait(); + + Stopwatch sw = new(); + var bytesConsumed = 0L; + sw.Start(); + while (!done) + { + var buf = rg.GetRequest(out var len); + fixed (byte* ptr = buf) + _ = sessions[threadId].TryConsumeMessages(ptr, len); + + bytesConsumed += len; + numReqs++; + if (numReqs == maxReqs) break; + } + sw.Stop(); + + Interlocked.Add(ref total_ops_done, numReqs * rg.BatchCount); + Interlocked.Add(ref total_bytes_consumed, bytesConsumed); + } + private void MGetThreadRunner(int threadid, int NumOps, int BatchSize = 1 << 12) { - bool checkResults = false; - int DbSize = database.Length; + var checkResults = false; + var DbSize = database.Length; using var redis = ConnectionMultiplexer.Connect($"{opts.Address}:{opts.Port},connectTimeout=999999,syncTimeout=999999"); - IDatabase db = redis.GetDatabase(0); + var db = redis.GetDatabase(0); Random r = new(threadid); Random r2 = new(threadid); Stopwatch sw = new(); sw.Start(); - int idx = 0; + var idx = 0; var getBatch = new RedisKey[BatchSize]; - for (int b = 0; b < NumOps; b++) + for (var b = 0; b < NumOps; b++) { getBatch[idx++] = database[r.Next(DbSize)].Key; if (idx == BatchSize) @@ -491,7 +570,7 @@ private void MGetThreadRunner(int threadid, int NumOps, int BatchSize = 1 << 12) var result = db.StringGet(getBatch); if (checkResults) { - for (int k = 0; k < idx; k++) + for (var k = 0; k < idx; k++) { if (database[r2.Next(DbSize)].Value != result[k]) Console.WriteLine("OperateThreadRunner: Error"); @@ -505,7 +584,7 @@ private void MGetThreadRunner(int threadid, int NumOps, int BatchSize = 1 << 12) var result = db.StringGet([.. getBatch.Take(idx)]); if (checkResults) { - for (int k = 0; k < idx; k++) + for (var k = 0; k < idx; k++) { if (database[r2.Next(DbSize)].Value != result[k]) Console.WriteLine("OperateThreadRunner: Error"); @@ -521,7 +600,7 @@ private void CreateLocalDB() { Console.WriteLine($"Creating database of size {opts.DbSize}"); database = new KeyValuePair[opts.DbSize]; - for (int k = 0; k < opts.DbSize; k++) + for (var k = 0; k < opts.DbSize; k++) { database[k] = new KeyValuePair(new RedisKey(k.ToString()), new RedisValue(k.ToString())); } @@ -533,16 +612,16 @@ private void LoadDatabaseStringSet(int BatchSize = 1 << 12) using var redis = ConnectionMultiplexer.Connect($"{opts.Address}:{opts.Port},connectTimeout=999999,syncTimeout=999999"); var db = redis.GetDatabase(0); - int DbSize = database.Length; + var DbSize = database.Length; Console.WriteLine($"Loading database of size {database.Length}"); Stopwatch sw = new(); sw.Start(); - bool MSet = true; + var MSet = true; if (MSet) { - for (int b = 0; b < DbSize; b += BatchSize) + for (var b = 0; b < DbSize; b += BatchSize) { db.StringSet([.. database.Skip(b).Take(BatchSize)]); if (b > 0 && b % 1000000 == 0) @@ -552,8 +631,8 @@ private void LoadDatabaseStringSet(int BatchSize = 1 << 12) else { var tasks = new Task[BatchSize]; - int idx = 0; - for (int b = 0; b < DbSize; b++) + var idx = 0; + for (var b = 0; b < DbSize; b++) { tasks[idx] = db.StringSetAsync(database[b].Key, database[b].Value); idx++; diff --git a/benchmark/Resp.benchmark/OnlineReqGen.cs b/benchmark/Resp.benchmark/OnlineBench/OnlineReqGen.cs similarity index 93% rename from benchmark/Resp.benchmark/OnlineReqGen.cs rename to benchmark/Resp.benchmark/OnlineBench/OnlineReqGen.cs index c00a916a27f..bdffbef8cf3 100644 --- a/benchmark/Resp.benchmark/OnlineReqGen.cs +++ b/benchmark/Resp.benchmark/OnlineBench/OnlineReqGen.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Runtime.CompilerServices; using System.Text; @@ -20,7 +19,7 @@ public unsafe partial class OnlineReqGen /// /// DbSize, ObjectDbSize, NumBuffs /// - public readonly int DbSize, ObjectDbSize, NumBuffs; + public readonly int DbSize, NumBuffs; readonly byte[] ascii_chars = Encoding.ASCII.GetBytes("abcdefghijklmnopqrstvuwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"); readonly byte[] number_chars = Encoding.ASCII.GetBytes("0123456789"); @@ -35,19 +34,11 @@ public unsafe partial class OnlineReqGen public readonly int keyLen, valueLen; - public OnlineReqGen(int thread_id, int DbSize, bool randomGen = true, bool zipf = false, int keyLen = default, int valueLen = default, int objectDbSize = -1) + public OnlineReqGen(int thread_id, int DbSize, bool randomGen = true, bool zipf = false, int keyLen = default, int valueLen = default) { this.randomGen = randomGen; this.DbSize = DbSize; this.zipf = zipf; - if (objectDbSize == -1) - { - this.ObjectDbSize = DbSize; - } - else - { - this.ObjectDbSize = objectDbSize; - } this.keyLen = Math.Max(NumUtils.NumDigits(DbSize), keyLen); this.valueLen = valueLen == default ? 8 : valueLen; @@ -113,7 +104,8 @@ public Memory GenerateKeyBytesRandom() { uint key = (uint)(randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(DbSize)) : (keyIndex++ % DbSize)); key *= 20323; - for (int i = 0; i < keyLen; i++) + keyBuffer[0] = (byte)'S'; // Uniquifier to avoid collisions with object keys. + for (int i = 1; i < keyLen; i++) { keyBuffer[i] = ascii_chars[key % ascii_chars.Length]; key *= 3; @@ -188,8 +180,9 @@ public Memory GenerateObjectEntryScoreBytes() /// public Memory GenerateObjectKeyBytesRandom() { - uint key = (uint)(randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(ObjectDbSize)) : (keyIndex++ % ObjectDbSize)); - for (int i = 0; i < keyLen; i++) + uint key = (uint)(randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(DbSize)) : (keyIndex++ % DbSize)); + keyBuffer[0] = (byte)'O'; // Uniquifier to avoid collisions with string keys. + for (int i = 1; i < keyLen; i++) { keyBuffer[i] = ascii_chars[key % ascii_chars.Length]; key *= 3; diff --git a/benchmark/Resp.benchmark/RespOnlineBench.cs b/benchmark/Resp.benchmark/OnlineBench/RespOnlineBench.cs similarity index 98% rename from benchmark/Resp.benchmark/RespOnlineBench.cs rename to benchmark/Resp.benchmark/OnlineBench/RespOnlineBench.cs index 24ebef22fc5..ca7f215f384 100644 --- a/benchmark/Resp.benchmark/RespOnlineBench.cs +++ b/benchmark/Resp.benchmark/OnlineBench/RespOnlineBench.cs @@ -1,13 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; -using System.Linq; using System.Net; using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; using Garnet.client; using Garnet.common; using HdrHistogram; @@ -65,7 +61,6 @@ static bool IsValidRange(long value) readonly CancellationTokenSource cts = new(); volatile int workerCount = 0; - public RespOnlineBench(Options opts, int resetInterval = 30, int runDuration = int.MaxValue, ILoggerFactory loggerFactory = null) { this.runDuration = runDuration; @@ -209,7 +204,7 @@ private void InitializeClients() private Thread[] InitializeThreadWorkers() { Thread[] workers = new Thread[NumThreads]; - for (int idx = 0; idx < NumThreads; ++idx) + for (int idx = 0; idx < NumThreads; idx++) { int x = idx; @@ -564,7 +559,7 @@ public async void OpRunnerGarnetClientSession(int thread_id) _ = Interlocked.Increment(ref workerCount); if (opts.BatchSize.First() != 1) throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, opts.ObjectDbSize); + var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength); GarnetClientSession client = null; if (!opts.Pool) @@ -608,8 +603,12 @@ public async void OpRunnerGarnetClientSession(int thread_id) OpType.ZREM => await ZREM(), OpType.ZCARD => await ZCARD(), OpType.READWRITETX => await c.ExecuteAsync("READWRITETX", req.GenerateKey(), req.GenerateKey(), req.GenerateKey(), "1000"), - OpType.SAMPLEUPDATETX => await c.ExecuteAsync("SAMPLEUPDATETX", req.GenerateKeyRandom(), req.GenerateValue(), req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), req.GenerateObjectEntryScore(), req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), req.GenerateObjectEntryScore()), - OpType.SAMPLEDELETETX => await c.ExecuteAsync("SAMPLEDELETETX", req.GenerateKeyRandom(), req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), req.GenerateObjectKeyRandom(), req.GenerateObjectEntry()), + OpType.SAMPLEUPDATETX => await c.ExecuteAsync("SAMPLEUPDATETX", req.GenerateKeyRandom(), req.GenerateValue(), // stringKey + req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), req.GenerateObjectEntryScore(), // sortedSetKey1 + req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), req.GenerateObjectEntryScore()), // sortedSetKey2 + OpType.SAMPLEDELETETX => await c.ExecuteAsync("SAMPLEDELETETX", req.GenerateKeyRandom(), // stringKey + req.GenerateObjectKeyRandom(), req.GenerateObjectEntry(), // sortedSetKey1 + req.GenerateObjectKeyRandom(), req.GenerateObjectEntry()), // sortedSetKey2 _ => throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!") }; diff --git a/benchmark/Resp.benchmark/Options.cs b/benchmark/Resp.benchmark/Options.cs index 605e988e465..2979900f51a 100644 --- a/benchmark/Resp.benchmark/Options.cs +++ b/benchmark/Resp.benchmark/Options.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Collections.Generic; using CommandLine; using Microsoft.Extensions.Logging; @@ -102,9 +101,6 @@ public partial class Options [Option("op-workload", Separator = ',', Default = new[] { OpType.GET, OpType.SET, OpType.DEL }, HelpText = "Workload of commands for online bench.")] public IEnumerable OpWorkload { get; set; } - [Option("object-dbsize", Required = false, Default = -1, HelpText = "Object DB size")] - public int ObjectDbSize { get; set; } - [Option("save-freq", Required = false, Default = 0, HelpText = "Save (checkpoint) frequency in seconds")] public int SaveFreqSecs { get; set; } @@ -116,5 +112,113 @@ public partial class Options [Option("file-logger", Required = false, Default = null, HelpText = "Enable file logger and write to the specified path.")] public string FileLogger { get; set; } + + [Option("aof-bench", Required = false, Default = false, HelpText = "Run AOF bench at replica.")] + public bool AofBench { get; set; } + + [Option("aof-bench-type", Required = false, Default = AofBenchType.Replay, HelpText = "Run AOF bench at replica.")] + public AofBenchType AofBenchType { get; set; } + + [Option("aof-gen-pages", Required = false, Default = 64, HelpText = "DB size")] + public int AofGenPages { get; set; } + + /* + * InProc/AofBench server options + */ + [Option("aof", Required = false, Default = false, HelpText = "Enable AOF")] + public bool EnableAOF { get; set; } + + [Option("cluster", Required = false, Default = false, HelpText = "Enable Cluster")] + public bool EnableCluster { get; set; } + + [Option('i', "index", Required = false, Default = "1g", HelpText = "Start size of hash index in bytes (rounds down to power of 2)")] + public string IndexMemorySize { get; set; } + + [Option("aof-null-device", Required = false, HelpText = "With main-memory replication, use null device for AOF. Ensures no disk IO, but can cause data loss during replication.")] + public bool UseAofNullDevice { get; set; } + + [Option("aof-commit-freq", Required = false, Default = 0, HelpText = "Write ahead logging (append-only file) commit issue frequency in milliseconds. 0 = issue an immediate commit per operation, -1 = manually issue commits using COMMITAOF command")] + public int CommitFrequencyMs { get; set; } + + [Option("aof-physical-sublog-count", Required = false, Default = 1, HelpText = "Number of sublogs used for AOF.")] + public int AofPhysicalSublogCount { get; set; } + + [Option("aof-replay-task-count", Required = false, Default = 1, HelpText = "Number of replay tasks per physical sublog at the replica.")] + public int AofReplayTaskCount { get; set; } + + [Option("aof-memory-size", Required = false, Default = "64m", HelpText = "Total AOF memory buffer used in bytes (rounds down to power of 2) - spills to disk after this limit.")] + public string AofMemorySize { get; set; } + + [Option("aof-page-size", Required = false, Default = "4m", HelpText = "Size of each AOF page in bytes(rounds down to power of 2)")] + public string AofPageSize { get; set; } + + /// + /// Parse size from string specification + /// + /// + /// + /// + public static long ParseSize(string value, out int bytesRead) + { + ReadOnlySpan suffix = ['k', 'm', 'g', 't', 'p']; + long result = 0; + bytesRead = 0; + for (var i = 0; i < value.Length; i++) + { + var c = value[i]; + if (char.IsDigit(c)) + { + result = (result * 10) + (byte)c - '0'; + bytesRead++; + } + else + { + for (var s = 0; s < suffix.Length; s++) + { + if (char.ToLower(c) == suffix[s]) + { + result *= (long)Math.Pow(1024, s + 1); + bytesRead++; + + if (i + 1 < value.Length && char.ToLower(value[i + 1]) == 'b') + bytesRead++; + + return result; + } + } + } + } + return result; + } + + /// + /// Get AOF Page size in bits + /// + /// + public int AofPageSizeBits() + { + var size = ParseSize(AofPageSize, out _); + var adjustedSize = PreviousPowerOf2(size); + return (int)Math.Log(adjustedSize, 2); + } + + /// + /// Previous power of 2 + /// + /// + /// + internal static long PreviousPowerOf2(long v) + { + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + return v - (v >> 1); + } + + public bool IsReplayEnabled + => AofBenchType is AofBenchType.Replay or AofBenchType.ReplayNoResp or AofBenchType.ReplayDirect; } } \ No newline at end of file diff --git a/benchmark/Resp.benchmark/Program.cs b/benchmark/Resp.benchmark/Program.cs index 6da133408bf..7f8e6691cf0 100644 --- a/benchmark/Resp.benchmark/Program.cs +++ b/benchmark/Resp.benchmark/Program.cs @@ -1,11 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using System.Collections.Generic; -using System.Linq; using System.Net; -using System.Threading; using CommandLine; using Garnet.client; using Garnet.common; @@ -94,6 +90,20 @@ static void PrintBenchMarkSummary(Options opts) Console.WriteLine($"minWorkerThreads: {minWorkerThreads}"); Console.WriteLine($"minCompletionPortThreads: {minCompletionPortThreads}"); Console.WriteLine("----------------------------------"); + + if (opts.Client == ClientType.InProc || opts.AofBench) + { + Console.WriteLine("------EMBEDDED-SERVER-CONFIG------"); + Console.WriteLine($"aof:{opts.EnableAOF || opts.AofBench}"); + Console.WriteLine($"aof-null-device:{opts.UseAofNullDevice}"); + Console.WriteLine($"aof-commit-freq:{opts.CommitFrequencyMs}"); + Console.WriteLine($"aof-memory-size:{opts.AofMemorySize}"); + Console.WriteLine($"aof-page-size:{opts.AofPageSize}"); + Console.WriteLine($"cluster:{opts.EnableCluster}"); + Console.WriteLine($"index:{opts.IndexMemorySize}"); + Console.WriteLine($"aof-sublog-count:{opts.AofPhysicalSublogCount}"); + Console.WriteLine("----------------------------------"); + } } static bool DisabledFeatures(Options opts) @@ -173,7 +183,8 @@ static void Main(string[] args) loggerFactory = CreateLoggerFactory(opts); - WaitForServer(opts); + if (!(opts.Client == ClientType.InProc || opts.AofBench)) + WaitForServer(opts); if (opts.SaveFreqSecs > 0) { @@ -254,6 +265,23 @@ static void RunBasicCommandsBenchmark(Options opts) bench.Run(); return; } + else if (opts.AofBench) + { + if (opts.IsReplayEnabled) + { + var bench = new AofBench(opts); + bench.GenerateData(); + bench.Run(opts.AofPhysicalSublogCount); + } + else + { + var bench = new AofBench(opts); + bench.GenerateData(); + + foreach (var threadCount in opts.NumThreads) + bench.Run(threadCount); + } + } else { var bench = new RespPerfBench(opts, 0, redis); @@ -261,7 +289,7 @@ static void RunBasicCommandsBenchmark(Options opts) if (!opts.SkipLoad) bench.LoadData(keyLen: keyLen, valueLen: valueLen, numericValue: opts.Op == OpType.INCR); - foreach (int BatchSize in opts.BatchSize) + foreach (var BatchSize in opts.BatchSize) bench.Run( opts.Op, opts.TotalOps, diff --git a/benchmark/Resp.benchmark/Resp.benchmark.csproj b/benchmark/Resp.benchmark/Resp.benchmark.csproj index b8d159b8ac6..e2ce7bc7009 100644 --- a/benchmark/Resp.benchmark/Resp.benchmark.csproj +++ b/benchmark/Resp.benchmark/Resp.benchmark.csproj @@ -4,6 +4,10 @@ Exe bin\$(Configuration)\$(TargetFramework)\$(AssemblyName).xml true + true + ../../Garnet.snk + false + enable @@ -12,17 +16,28 @@ - - + + + + + + + + + - - + + + + + + - + \ No newline at end of file diff --git a/benchmark/Resp.benchmark/TxnPerfBench.cs b/benchmark/Resp.benchmark/TxnPerfBench.cs index 6312ae7b26e..5d918fdf9dc 100644 --- a/benchmark/Resp.benchmark/TxnPerfBench.cs +++ b/benchmark/Resp.benchmark/TxnPerfBench.cs @@ -1,11 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; -using System.Linq; using System.Net; -using System.Threading; using Garnet.client; using Garnet.common; using HdrHistogram; @@ -132,7 +129,7 @@ public void Run() redis = Program.redis; } } - for (int idx = 0; idx < NumThreads; ++idx) + for (int idx = 0; idx < NumThreads; idx++) { int x = idx; diff --git a/libs/GlobalUsings.cs b/libs/GlobalUsings.cs new file mode 100644 index 00000000000..6fe0a6296a3 --- /dev/null +++ b/libs/GlobalUsings.cs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#if SERVER_PROJECT || TEST_PROJECT || CLUSTER_PROJECT +global using BasicGarnetApi = Garnet.server.GarnetApi< + Tsavorite.core.BasicContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.BasicContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.BasicContext, + Tsavorite.core.ObjectAllocator>>>; +#endif + +#if SERVER_PROJECT +global using ConsistentReadGarnetApi = Garnet.server.GarnetApi< + Tsavorite.core.ConsistentReadContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.ConsistentReadContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.ConsistentReadContext, + Tsavorite.core.ObjectAllocator>>>; +global using TransactionalConsistentReadGarnetApi = Garnet.server.GarnetApi< + Tsavorite.core.TransactionalConsistentReadContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.TransactionalConsistentReadContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.TransactionalConsistentReadContext, + Tsavorite.core.ObjectAllocator>>>; +#endif + +#if SERVER_PROJECT || TEST_PROJECT || HOST_PROJECT +global using StoreAllocator = Tsavorite.core.ObjectAllocator>; +global using StoreFunctions = Tsavorite.core.StoreFunctions; +#endif + +#if SERVER_PROJECT || CLUSTER_PROJECT +global using StringBasicContext = Tsavorite.core.BasicContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.StringInput, + Garnet.server.StringOutput, + long, Garnet.server.MainSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using VectorBasicContext = Tsavorite.core.BasicContext< + Garnet.common.VectorElementKey, + Garnet.server.VectorInput, + Garnet.server.VectorOutput, + long, Garnet.server.VectorSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; +#endif + +#if SERVER_PROJECT +global using TransactionalGarnetApi = Garnet.server.GarnetApi< + Tsavorite.core.TransactionalContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.TransactionalContext, + Tsavorite.core.ObjectAllocator>>, + Tsavorite.core.TransactionalContext, + Tsavorite.core.ObjectAllocator>>>; + +global using StringTransactionalContext = Tsavorite.core.TransactionalContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.StringInput, + Garnet.server.StringOutput, + long, Garnet.server.MainSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using StringTransactionalUnsafeContext = Tsavorite.core.TransactionalUnsafeContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.StringInput, + Garnet.server.StringOutput, + long, + Garnet.server.MainSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadStringBasicContext = Tsavorite.core.ConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.StringInput, + Garnet.server.StringOutput, + long, + Garnet.server.MainSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadStringTransactionalContext = Tsavorite.core.TransactionalConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.StringInput, + Garnet.server.StringOutput, + long, + Garnet.server.MainSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ObjectBasicContext = Tsavorite.core.BasicContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.ObjectInput, + Garnet.server.ObjectOutput, + long, Garnet.server.ObjectSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ObjectTransactionalContext = Tsavorite.core.TransactionalContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.ObjectInput, + Garnet.server.ObjectOutput, + long, + Garnet.server.ObjectSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadObjectBasicContext = Tsavorite.core.ConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.ObjectInput, + Garnet.server.ObjectOutput, + long, Garnet.server.ObjectSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadObjectTransactionalContext = Tsavorite.core.TransactionalConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.ObjectInput, + Garnet.server.ObjectOutput, + long, + Garnet.server.ObjectSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using UnifiedBasicContext = Tsavorite.core.BasicContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.UnifiedInput, + Garnet.server.UnifiedOutput, + long, + Garnet.server.UnifiedSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using UnifiedTransactionalContext = Tsavorite.core.TransactionalContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.UnifiedInput, + Garnet.server.UnifiedOutput, + long, + Garnet.server.UnifiedSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadUnifiedBasicContext = Tsavorite.core.ConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.UnifiedInput, + Garnet.server.UnifiedOutput, + long, + Garnet.server.UnifiedSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using ConsistentReadUnifiedTransactionalContext = Tsavorite.core.TransactionalConsistentReadContext< + Garnet.common.FixedSpanByteKey, + Garnet.server.UnifiedInput, + Garnet.server.UnifiedOutput, + long, + Garnet.server.UnifiedSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +global using VectorTransactionalContext = Tsavorite.core.TransactionalContext< + Garnet.common.VectorElementKey, + Garnet.server.VectorInput, + Garnet.server.VectorOutput, + long, Garnet.server.VectorSessionFunctions, + Tsavorite.core.StoreFunctions, + Tsavorite.core.ObjectAllocator>>; + +#endif \ No newline at end of file diff --git a/libs/client/ClientSession/GarnetClientSession.cs b/libs/client/ClientSession/GarnetClientSession.cs index 0d90bff8f3a..7a1e988da0a 100644 --- a/libs/client/ClientSession/GarnetClientSession.cs +++ b/libs/client/ClientSession/GarnetClientSession.cs @@ -78,6 +78,11 @@ public sealed partial class GarnetClientSession : IServerHook, IMessageConsumer /// readonly string authPassword = null; + /// + /// Set client name on the server for easier identification in monitoring and debugging. + /// + readonly string clientName = null; + /// /// Indicating whether this instance is using its own network pool or one that was provided /// @@ -100,9 +105,11 @@ public sealed partial class GarnetClientSession : IServerHook, IMessageConsumer /// TLS options /// Username to authenticate with /// Password to authenticate with + /// Client name to be used with CLIENT SETNAME command /// Settings for send and receive network buffers /// Buffer pool to use for allocating send and receive buffers /// Max outstanding network sends allowed + /// Flag if raw result from response will be processed /// Logger public GarnetClientSession( EndPoint endpoint, @@ -111,6 +118,7 @@ public GarnetClientSession( SslClientAuthenticationOptions tlsOptions = null, string authUsername = null, string authPassword = null, + string clientName = null, int networkSendThrottleMax = 8, bool rawResult = false, ILogger logger = null) @@ -119,7 +127,7 @@ public GarnetClientSession( this.usingManagedNetworkPool = networkPool != null; this.networkBufferSettings = networkBufferSettings; - this.networkPool = networkPool ?? networkBufferSettings.CreateBufferPool(); + this.networkPool = networkPool ?? networkBufferSettings.CreateBufferPool(ownerType: PoolOwnerType.GarnetClientSession, logger: logger); this.bufferSizeDigits = NumUtils.CountDigits(this.networkBufferSettings.sendBufferSize); this.logger = logger; @@ -128,6 +136,7 @@ public GarnetClientSession( this.disposed = 0; this.authUsername = authUsername; this.authPassword = authPassword; + this.clientName = clientName; this.RawResult = rawResult; } @@ -170,6 +179,20 @@ public unsafe void Connect(CancellationToken token = default) logger?.LogError(e, "AUTH returned error"); throw; } + + try + { + if (clientName != null) + { + Execute("CLIENT", "SETINFO", "LIB-NAME", "GarnetClientSession"); + Execute("CLIENT", "SETNAME", clientName); + } + } + catch (Exception e) + { + logger?.LogError(e, "Client set info returned error!"); + throw; + } } /// @@ -211,6 +234,20 @@ public async Task ConnectAsync(int timeoutMs = 0, CancellationToken token = defa logger?.LogError(e, "AUTH returned error"); throw; } + + try + { + if (clientName != null) + { + _ = await ExecuteAsync("CLIENT", "SETINFO", "LIB-NAME", "GarnetClientSession").ConfigureAwait(false); + _ = await ExecuteAsync("CLIENT", "SETNAME", clientName).ConfigureAwait(false); + } + } + catch (Exception e) + { + logger?.LogError(e, "Client set info returned error!"); + throw; + } } /// @@ -264,7 +301,7 @@ private async Task ConnectSendSocketAsync(int millisecondsTimeout = 0, C NoDelay = true }; - if (await TryConnectSocketAsync(socket, endpoint, millisecondsTimeout, cancellationToken)) + if (await TryConnectSocketAsync(socket, endpoint, millisecondsTimeout, cancellationToken).ConfigureAwait(false)) return socket; } } @@ -274,7 +311,7 @@ private async Task ConnectSendSocketAsync(int millisecondsTimeout = 0, C if (EndPoint is not UnixDomainSocketEndPoint) socket.NoDelay = true; - if (await TryConnectSocketAsync(socket, EndPoint, millisecondsTimeout, cancellationToken)) + if (await TryConnectSocketAsync(socket, EndPoint, millisecondsTimeout, cancellationToken).ConfigureAwait(false)) return socket; } @@ -321,12 +358,12 @@ private async Task TryConnectSocketAsync(Socket socket, EndPoint endpoint, using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var connectTask = socket.ConnectAsync(endpoint, timeoutCts.Token).AsTask(); - if (await Task.WhenAny(connectTask, Task.Delay(millisecondsTimeout, timeoutCts.Token)) == connectTask) + if (await Task.WhenAny(connectTask, Task.Delay(millisecondsTimeout, timeoutCts.Token)).ConfigureAwait(false) == connectTask) { // Task completed within timeout. // Consider that the task may have faulted or been canceled. // We re-await the task so that any exceptions/cancellation is rethrown. - await connectTask; + await connectTask.ConfigureAwait(false); } else { @@ -424,12 +461,21 @@ public void ExecuteForArray(params string[] command) /// /// ClusterAppendLog /// - public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, long currentAddress, long nextAddress, long payloadPtr, int payloadLength) + /// + /// + /// + /// + /// + /// + /// + /// + /// + public unsafe void ExecuteClusterAppendLog(string nodeId, int physicalSublogIdx, long previousAddress, long currentAddress, long nextAddress, long payloadPtr, int payloadLength) { Debug.Assert(nodeId != null); var curr = offset; - var arraySize = 7; + var arraySize = 8; while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) { @@ -438,6 +484,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 1 while (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) { Flush(); @@ -445,6 +492,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 2 while (!RespWriteUtils.TryWriteBulkString(appendLog, ref curr, end)) { Flush(); @@ -452,6 +500,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 3 while (!RespWriteUtils.TryWriteAsciiBulkString(nodeId, ref curr, end)) { Flush(); @@ -459,6 +508,15 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 4 + while (!RespWriteUtils.TryWriteArrayItem(physicalSublogIdx, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 5 while (!RespWriteUtils.TryWriteArrayItem(previousAddress, ref curr, end)) { Flush(); @@ -466,6 +524,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 6 while (!RespWriteUtils.TryWriteArrayItem(currentAddress, ref curr, end)) { Flush(); @@ -473,6 +532,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, } offset = curr; + // 7 while (!RespWriteUtils.TryWriteArrayItem(nextAddress, ref curr, end)) { Flush(); @@ -483,6 +543,7 @@ public unsafe void ExecuteClusterAppendLog(string nodeId, long previousAddress, if (payloadLength > networkBufferSettings.sendBufferSize) throw new Exception($"Payload length {payloadLength} is larger than bufferSize {networkBufferSettings.sendBufferSize} bytes"); + // 8 while (!RespWriteUtils.TryWriteBulkString(new Span((void*)payloadPtr, payloadLength), ref curr, end)) { Flush(); @@ -652,12 +713,17 @@ private unsafe void Flush() Dispose(); throw; } - networkSender.GetResponseObject(); - offset = networkSender.GetResponseObjectHead(); - end = networkSender.GetResponseObjectTail(); + ResetOffset(); } } + private unsafe void ResetOffset() + { + networkSender.GetResponseObject(); + offset = networkSender.GetResponseObjectHead(); + end = networkSender.GetResponseObjectTail(); + } + /// public bool TryCreateMessageConsumer(Span bytesReceived, INetworkSender networkSender, out IMessageConsumer session) => throw new NotSupportedException(); diff --git a/libs/client/ClientSession/GarnetClientSessionClusterExtensions.cs b/libs/client/ClientSession/GarnetClientSessionClusterExtensions.cs deleted file mode 100644 index 0b64944783c..00000000000 --- a/libs/client/ClientSession/GarnetClientSessionClusterExtensions.cs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Garnet.common; -using Garnet.networking; - -namespace Garnet.client -{ - /// - /// Mono-threaded remote client session for Garnet (a session makes a single network connection, and - /// expects mono-threaded client access, i.e., no concurrent invocations of API by client) - /// - public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageConsumer - { - static ReadOnlySpan GOSSIP => "GOSSIP"u8; - - /// - /// Send gossip message to corresponding node - /// - /// - /// - public Task ExecuteGossip(Memory byteArray) - { - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - tcsQueue.Enqueue(tcs); - byte* curr = offset; - byte* next = offset; - int arraySize = 3; - - while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) - { - Flush(); - curr = offset; - } - offset = curr; - - //1 - while (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) - { - Flush(); - curr = offset; - } - offset = curr; - - //2 - while (!RespWriteUtils.TryWriteBulkString(GOSSIP, ref curr, end)) - { - Flush(); - curr = offset; - } - offset = curr; - - //3 - while (!RespWriteUtils.TryWriteBulkString(byteArray.Span, ref curr, end)) - { - Flush(); - curr = offset; - } - offset = curr; - - Flush(); - Interlocked.Increment(ref numCommands); - return tcs.Task; - } - } -} \ No newline at end of file diff --git a/libs/client/ClientSession/GarnetClientSessionIncremental.cs b/libs/client/ClientSession/GarnetClientSessionIncremental.cs index 088aa919cc0..6b23ed4d0fd 100644 --- a/libs/client/ClientSession/GarnetClientSessionIncremental.cs +++ b/libs/client/ClientSession/GarnetClientSessionIncremental.cs @@ -18,12 +18,37 @@ enum IncrementalSendType : byte SYNC } + /// + /// When writing a RecordSpan, the format of the associated data. + /// + public enum MigrationRecordSpanType : byte + { + /// + /// Invalid + /// + Invalid = 0, + + /// + /// Serialized . + /// + LogRecord = 1, + + /// + /// Bespoke encoding for Vector Set elements. + /// + VectorSetElement = 2, + + /// + /// Bespoke encoding for Vector Set indexes. + /// + VectorSetIndex = 3, + } + public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageConsumer { IncrementalSendType ist; - bool isMainStore; byte* curr, head; - int keyValuePairCount; + int recordCount; TaskCompletionSource currTcsIterationTask = null; /// @@ -42,15 +67,21 @@ public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageCo public bool NeedsInitialization => curr == null; /// - /// Flush and initialize buffers/parameters used for migrate command + /// Return a of all remaining available space in the network buffer. + /// + public PinnedSpanByte GetAvailableNetworkBufferSpan() => PinnedSpanByte.FromPinnedPointer(curr, (int)(end - curr)); + + /// + /// Flush and initialize buffers/parameters used for Migrate and Replica commands /// /// public void InitializeIterationBuffer(TimeSpan iterationProgressFreq) { + EnsureTcsIsEnqueued(); Flush(); currTcsIterationTask = null; curr = head = null; - keyValuePairCount = 0; + recordCount = 0; this.iterationProgressFreq = default ? TimeSpan.FromSeconds(5) : iterationProgressFreq; } @@ -59,7 +90,14 @@ public void InitializeIterationBuffer(TimeSpan iterationProgressFreq) /// public Task SendAndResetIterationBuffer() { - if (keyValuePairCount == 0) return null; + Task task = null; + if (recordCount == 0) + { + // No records to Flush(), but we need to reset buffer offsets as we may have written a header due to the need to initialize the buffer + // before passing it to Tsavorite as the output SpanByteAndMemory.SpanByte for Read(). + ResetOffset(); + goto done; + } Debug.Assert(end - curr >= 2); *curr++ = (byte)'\r'; @@ -67,114 +105,59 @@ public Task SendAndResetIterationBuffer() // Payload format = [$length\r\n][number of keys (4 bytes)][raw key value pairs]\r\n var size = (int)(curr - 2 - head - (ExtraSpace - 4)); - TrackIterationProgress(keyValuePairCount, size); + TrackIterationProgress(recordCount, size); var success = RespWriteUtils.TryWritePaddedBulkStringLength(size, ExtraSpace - 4, ref head, end); Debug.Assert(success); // Number of key value pairs in payload - *(int*)head = keyValuePairCount; + *(int*)head = recordCount; // Reset offset and flush buffer offset = curr; + EnsureTcsIsEnqueued(); Flush(); Interlocked.Increment(ref numCommands); // Return outstanding task and reset current tcs - var task = currTcsIterationTask.Task; + task = currTcsIterationTask.Task; currTcsIterationTask = null; + recordCount = 0; + + done: curr = head = null; - keyValuePairCount = 0; return task; } /// - /// Try write key value pair for main store directly to the client buffer + /// Try to write the span for the entire record directly to the client buffer /// - /// - /// - /// - /// - public bool TryWriteKeyValueSpanByte(ref SpanByte key, ref SpanByte value, out Task task) + public bool TryWriteRecordSpan(ReadOnlySpan recordSpan, MigrationRecordSpanType type, out Task task) { - task = null; - // Try write key value pair directly to client buffer - if (!WriteSerializedSpanByte(ref key, ref value)) + // We include space for newline at the end, to be added before sending + var recordSpanSize = recordSpan.TotalSize(); + + var totalLen = recordSpanSize + 2 + 1; // +2 for \r\n, +1 for type + if (totalLen > (int)(end - curr)) { - // If failed to write because no space left send outstanding data and retrieve task - // Caller is responsible for retrying + // If there is no space left, send outstanding data and return the send-completion task. + // Caller is responsible for waiting for task completion and retrying. task = SendAndResetIterationBuffer(); return false; } - keyValuePairCount++; - return true; - - bool WriteSerializedSpanByte(ref SpanByte key, ref SpanByte value) - { - var totalLen = key.TotalSize + value.TotalSize + 2 + 2; - if (totalLen > (int)(end - curr)) - return false; - - key.CopyTo(curr); - curr += key.TotalSize; - value.CopyTo(curr); - curr += value.TotalSize; - return true; - } - } + *curr = (byte)type; + curr++; - /// - /// Try write key value pair for object store directly to the client buffer - /// - /// - /// - /// - /// - /// - public bool TryWriteKeyValueByteArray(byte[] key, byte[] value, long expiration, out Task task) - { + recordSpan.SerializeTo(curr, recordSpanSize); + curr += recordSpanSize; + ++recordCount; task = null; - // Try write key value pair directly to client buffer - if (!WriteSerializedKeyValueByteArray(key, value, expiration)) - { - // If failed to write because no space left send outstanding data and retrieve task - // Caller is responsible for retrying - task = SendAndResetIterationBuffer(); - return false; - } - - keyValuePairCount++; return true; - - bool WriteSerializedKeyValueByteArray(byte[] key, byte[] value, long expiration) - { - // We include space for newline at the end, to be added before sending - int totalLen = 4 + key.Length + 4 + value.Length + 8 + 2; - if (totalLen > (int)(end - curr)) - return false; - - *(int*)curr = key.Length; - curr += 4; - fixed (byte* keyPtr = key) - Buffer.MemoryCopy(keyPtr, curr, key.Length, key.Length); - curr += key.Length; - - *(int*)curr = value.Length; - curr += 4; - fixed (byte* valPtr = value) - Buffer.MemoryCopy(valPtr, curr, value.Length, value.Length); - curr += value.Length; - - *(long*)curr = expiration; - curr += 8; - - return true; - } } - long lastLog = 0; - long totalKeyCount = 0; - long totalPayloadSize = 0; + long lastLog; + long totalKeyCount; + long totalPayloadSize; TimeSpan iterationProgressFreq; /// @@ -190,13 +173,22 @@ private void TrackIterationProgress(int keyCount, int size, bool completed = fal var duration = TimeSpan.FromTicks(Stopwatch.GetTimestamp() - lastLog); if (completed || lastLog == 0 || duration >= iterationProgressFreq) { - logger?.LogTrace("[{op}]: store:({storeType}) totalKeyCount:({totalKeyCount}), totalPayloadSize:({totalPayloadSize} KB)", + logger?.LogTrace("[{op}]: totalKeyCount:({totalKeyCount}), totalPayloadSize:({totalPayloadSize} KB)", completed ? "COMPLETED" : ist, - isMainStore ? "MAIN STORE" : "OBJECT STORE", totalKeyCount.ToString("N0"), ((long)((double)totalPayloadSize / 1024)).ToString("N0")); lastLog = Stopwatch.GetTimestamp(); } } + + private void EnsureTcsIsEnqueued() + { + // See comments in SetClusterMigrateHeader() as to why this is decoupled from the header initialization. + if (recordCount > 0 && currTcsIterationTask == null) + { + currTcsIterationTask = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + tcsQueue.Enqueue(currTcsIterationTask); + } + } } } \ No newline at end of file diff --git a/libs/client/ClientSession/GarnetClientSessionMigrationExtensions.cs b/libs/client/ClientSession/GarnetClientSessionMigrationExtensions.cs index 9ac7428ef40..e785eaf7aa0 100644 --- a/libs/client/ClientSession/GarnetClientSessionMigrationExtensions.cs +++ b/libs/client/ClientSession/GarnetClientSessionMigrationExtensions.cs @@ -23,9 +23,6 @@ public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageCo static ReadOnlySpan DELKEY => "DELKEY"u8; static ReadOnlySpan GETKVPAIRINSLOT => "GETKVPAIRINSLOT"u8; - static ReadOnlySpan MAIN_STORE => "SSTORE"u8; - static ReadOnlySpan OBJECT_STORE => "OSTORE"u8; - static ReadOnlySpan VECTOR_STORE => "VSTORE"u8; static ReadOnlySpan T => "T"u8; static ReadOnlySpan F => "F"u8; @@ -170,32 +167,17 @@ public Task SetSlotRange(Memory state, string nodeid, List<(int, i /// /// /// - /// - public void SetClusterMigrateHeader(string sourceNodeId, bool replace, bool isMainStore, bool isVectorSets) + /// + public void SetClusterMigrateHeader(string sourceNodeId, bool replace, bool isVectorSets) { - currTcsIterationTask = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - tcsQueue.Enqueue(currTcsIterationTask); + // For Migration we send the (curr - end) buffer as the SpanByteAndMemory.SpanByte output to Tsavorite. Thus we must + // initialize the header first, so we have curr properly positioned, but we cannot yet enqueue currTcsIterationTask. + // Therefore we defer this until the actual Flush(), when we know we have records to send. This is not a concern for + // Replication, because it uses an iterator and thus knows it has a record before it initializes the header. curr = offset; - this.isMainStore = isMainStore; this.ist = IncrementalSendType.MIGRATE; - ReadOnlySpan storeType; - if (isMainStore) - { - if (isVectorSets) - { - storeType = VECTOR_STORE; - } - else - { - storeType = MAIN_STORE; - } - } - else - { - storeType = OBJECT_STORE; - } - var replaceOption = replace ? T : F; + var vectorSetOption = isVectorSets ? T : F; var arraySize = 6; @@ -239,7 +221,7 @@ public void SetClusterMigrateHeader(string sourceNodeId, bool replace, bool isMa offset = curr; // 5 - while (!RespWriteUtils.TryWriteBulkString(storeType, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(vectorSetOption, ref curr, end)) { Flush(); curr = offset; @@ -262,11 +244,10 @@ public void SetClusterMigrateHeader(string sourceNodeId, bool replace, bool isMa /// /// /// - /// /// - public Task CompleteMigrate(string sourceNodeId, bool replace, bool isMainStore) + public Task CompleteMigrate(string sourceNodeId, bool replace) { - SetClusterMigrateHeader(sourceNodeId, replace, isMainStore, isVectorSets: false); + SetClusterMigrateHeader(sourceNodeId, replace, isVectorSets: false); Debug.Assert(end - curr >= 2); *curr++ = (byte)'\r'; @@ -274,15 +255,16 @@ public Task CompleteMigrate(string sourceNodeId, bool replace, bool isMa // Payload format = [$length\r\n][number of keys (4 bytes)][raw key value pairs]\r\n var size = (int)(curr - 2 - head - (ExtraSpace - 4)); - TrackIterationProgress(keyValuePairCount, size, completed: true); + TrackIterationProgress(recordCount, size, completed: true); var success = RespWriteUtils.TryWritePaddedBulkStringLength(size, ExtraSpace - 4, ref head, end); Debug.Assert(success); // Number of key value pairs in payload - *(int*)head = keyValuePairCount; + *(int*)head = recordCount; // Reset offset and flush buffer offset = curr; + EnsureTcsIsEnqueued(); Flush(); Interlocked.Increment(ref numCommands); @@ -290,7 +272,7 @@ public Task CompleteMigrate(string sourceNodeId, bool replace, bool isMa var task = currTcsIterationTask.Task; currTcsIterationTask = null; curr = head = null; - keyValuePairCount = 0; + recordCount = 0; return task; } } diff --git a/libs/client/ClientSession/GarnetClientSessionReplicationExtensions.cs b/libs/client/ClientSession/GarnetClientSessionReplicationExtensions.cs index 408471734bf..bdd9db5ea68 100644 --- a/libs/client/ClientSession/GarnetClientSessionReplicationExtensions.cs +++ b/libs/client/ClientSession/GarnetClientSessionReplicationExtensions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Garnet.common; @@ -15,12 +16,14 @@ namespace Garnet.client /// public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageConsumer { - static ReadOnlySpan initiate_replica_sync => "INITIATE_REPLICA_SYNC"u8; - static ReadOnlySpan send_ckpt_metadata => "SEND_CKPT_METADATA"u8; - static ReadOnlySpan send_ckpt_file_segment => "SEND_CKPT_FILE_SEGMENT"u8; - static ReadOnlySpan begin_replica_recover => "BEGIN_REPLICA_RECOVER"u8; - static ReadOnlySpan attach_sync => "ATTACH_SYNC"u8; - static ReadOnlySpan sync => "SYNC"u8; + private static ReadOnlySpan initiate_replica_sync => "INITIATE_REPLICA_SYNC"u8; + private static ReadOnlySpan send_ckpt_metadata => "SEND_CKPT_METADATA"u8; + private static ReadOnlySpan send_ckpt_file_segment => "SEND_CKPT_FILE_SEGMENT"u8; + private static ReadOnlySpan begin_replica_recover => "BEGIN_REPLICA_RECOVER"u8; + private static ReadOnlySpan attach_sync => "ATTACH_SYNC"u8; + private static ReadOnlySpan sync => "SYNC"u8; + private static ReadOnlySpan advance_time => "ADVANCE_TIME"u8; + private static ReadOnlySpan snapshot_data => "SNAPSHOT_DATA"u8; /// /// Initiate checkpoint retrieval from replica by sending replica checkpoint information and AOF address range @@ -31,7 +34,8 @@ public sealed unsafe partial class GarnetClientSession : IServerHook, IMessageCo /// /// /// - public Task ExecuteReplicaSync(string nodeId, string primary_replid, byte[] checkpointEntryData, long aofBeginAddress, long aofTailAddress) + /// + public Task ExecuteClusterInitiateReplicaSync(string nodeId, string primary_replid, byte[] checkpointEntryData, Span aofBeginAddress, Span aofTailAddress) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(tcs); @@ -86,7 +90,7 @@ public Task ExecuteReplicaSync(string nodeId, string primary_replid, byt offset = curr; //6 - while (!RespWriteUtils.TryWriteArrayItem(aofBeginAddress, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(aofBeginAddress, ref curr, end)) { Flush(); curr = offset; @@ -94,7 +98,7 @@ public Task ExecuteReplicaSync(string nodeId, string primary_replid, byt offset = curr; //7 - while (!RespWriteUtils.TryWriteArrayItem(aofTailAddress, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(aofTailAddress, ref curr, end)) { Flush(); curr = offset; @@ -112,7 +116,8 @@ public Task ExecuteReplicaSync(string nodeId, string primary_replid, byt /// /// /// - public Task ExecuteSendCkptMetadata(Memory fileTokenBytes, int fileType, Memory data) + /// + public Task ExecuteClusterSendCheckpointMetadata(Memory fileTokenBytes, int fileType, Memory data) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(tcs); @@ -178,7 +183,8 @@ public Task ExecuteSendCkptMetadata(Memory fileTokenBytes, int fil /// /// /// - public Task ExecuteSendFileSegments(Memory fileTokenBytes, int fileType, long startAddress, Span data, int segmentId = -1) + /// + public Task ExecuteClusterSendCheckpointFileSegment(Memory fileTokenBytes, int fileType, long startAddress, Span data, int segmentId = -1) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(tcs); @@ -253,23 +259,110 @@ public Task ExecuteSendFileSegments(Memory fileTokenBytes, int fil return tcs.Task; } + /// + /// Send snapshot data (file segments or metadata) via unified CLUSTER SNAPSHOT_DATA command. + /// + /// The checkpoint token bytes. + /// The checkpoint file type (including metadata variants). + /// The start address for this chunk (-1 for metadata). + /// The data to send. + /// + public Task ExecuteClusterSnapshotData(Memory fileTokenBytes, int fileType, long startAddress, Span data) + { + // The data payload must fit in the send buffer (as a RESP bulk string) after a flush, + // otherwise the TryWriteBulkString/Flush loop below will spin forever. + if (data.Length > networkBufferSettings.sendBufferSize) + ExceptionUtils.ThrowException(new InvalidOperationException( + $"Snapshot data chunk ({data.Length} bytes) exceeds send buffer size ({networkBufferSettings.sendBufferSize} bytes)")); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + tcsQueue.Enqueue(tcs); + byte* curr = offset; + int arraySize = 6; + + while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //1 + while (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //2 + while (!RespWriteUtils.TryWriteBulkString(snapshot_data, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //3 + while (!RespWriteUtils.TryWriteBulkString(fileTokenBytes.Span, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //4 + while (!RespWriteUtils.TryWriteArrayItem(fileType, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //5 + while (!RespWriteUtils.TryWriteArrayItem(startAddress, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //6 + while (!RespWriteUtils.TryWriteBulkString(data, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + Flush(); + Interlocked.Increment(ref numCommands); + return tcs.Task; + } + /// /// Signal replica to recover /// /// - /// /// /// /// /// /// /// - public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool sendObjectStoreCheckpoint, bool replayAOF, string primary_replid, byte[] checkpointEntryData, long beginAddress, long tailAddress) + /// + public Task ExecuteClusterBeginReplicaRecover( + bool sendStoreCheckpoint, + ulong replayAOF, + string primary_replid, + Span checkpointEntryData, + Span beginAddress, + Span tailAddress) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(tcs); - byte* curr = offset; - int arraySize = 9; + var curr = offset; + var arraySize = 8; while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) { @@ -303,7 +396,7 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se offset = curr; //4 - while (!RespWriteUtils.TryWriteBulkString(sendObjectStoreCheckpoint ? "1"u8 : "0"u8, ref curr, end)) + while (!RespWriteUtils.TryWriteArrayItem((long)replayAOF, ref curr, end)) { Flush(); curr = offset; @@ -311,7 +404,7 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se offset = curr; //5 - while (!RespWriteUtils.TryWriteBulkString(replayAOF ? "1"u8 : "0"u8, ref curr, end)) + while (!RespWriteUtils.TryWriteAsciiBulkString(primary_replid, ref curr, end)) { Flush(); curr = offset; @@ -319,7 +412,7 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se offset = curr; //6 - while (!RespWriteUtils.TryWriteAsciiBulkString(primary_replid, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(checkpointEntryData, ref curr, end)) { Flush(); curr = offset; @@ -327,7 +420,7 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se offset = curr; //7 - while (!RespWriteUtils.TryWriteBulkString(checkpointEntryData, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(beginAddress, ref curr, end)) { Flush(); curr = offset; @@ -335,15 +428,7 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se offset = curr; //8 - while (!RespWriteUtils.TryWriteArrayItem(beginAddress, ref curr, end)) - { - Flush(); - curr = offset; - } - offset = curr; - - //9 - while (!RespWriteUtils.TryWriteArrayItem(tailAddress, ref curr, end)) + while (!RespWriteUtils.TryWriteBulkString(tailAddress, ref curr, end)) { Flush(); curr = offset; @@ -360,7 +445,8 @@ public Task ExecuteBeginReplicaRecover(bool sendStoreCheckpoint, bool se /// /// /// - public Task ExecuteAttachSync(byte[] syncMetadata) + /// + public Task ExecuteClusterAttachSync(byte[] syncMetadata) { var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(tcs); @@ -404,20 +490,21 @@ public Task ExecuteAttachSync(byte[] syncMetadata) } /// - /// Set CLUSTER SYNC header info + /// Set CLUSTER ATTACH_SYNC header info /// /// - /// - public void SetClusterSyncHeader(string sourceNodeId, bool isMainStore) + /// + /// + public void SetClusterSyncHeader(string sourceNodeId) { + // Unlike Migration, where we don't know at the time of header initialization if we have a record or not, in Replication + // we know we have a record at the time this is called, so we can initialize it directly. currTcsIterationTask = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); tcsQueue.Enqueue(currTcsIterationTask); curr = offset; - this.isMainStore = isMainStore; this.ist = IncrementalSendType.SYNC; - var storeType = isMainStore ? MAIN_STORE : OBJECT_STORE; - var arraySize = 5; + var arraySize = 4; while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) { Flush(); @@ -450,7 +537,127 @@ public void SetClusterSyncHeader(string sourceNodeId, bool isMainStore) offset = curr; // 4 - while (!RespWriteUtils.TryWriteBulkString(storeType, ref curr, end)) + // Reserve space for the bulk string header + final newline + while (ExtraSpace + 2 > (int)(end - curr)) + { + Flush(); + curr = offset; + } + head = curr; + curr += ExtraSpace; + } + + /// + /// Issue CLUSTER ADVANCE_TIME + /// + /// + /// + /// + /// + public Task ExecuteClusterAdvanceTime(long sequenceNumber, Span aofAddress) + { + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + tcsQueue.Enqueue(tcs); + var curr = offset; + var argCount = 2; + var arraySize = 2 + argCount; + + while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //1 + while (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //2 + while (!RespWriteUtils.TryWriteBulkString(advance_time, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //3 + while (!RespWriteUtils.TryWriteArrayItem(sequenceNumber, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + //4 + while (!RespWriteUtils.TryWriteBulkString(aofAddress, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + Flush(); + Interlocked.Increment(ref numCommands); + return tcs.Task; + } + + /// + /// Issue CLUSTER APPEND_LOG with initialization parameters + /// + /// + /// + /// + /// + /// + /// + /// + public Task ExecuteClusterAppendLogInit(string nodeId, int physicalSublogIdx, long previousAddress, long currentAddress, long nextAddress) + { + Debug.Assert(nodeId != null); + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + tcsQueue.Enqueue(tcs); + var curr = offset; + var arraySize = 7; + + while (!RespWriteUtils.TryWriteArrayLength(arraySize, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 1 + while (!RespWriteUtils.TryWriteDirect(CLUSTER, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 2 + while (!RespWriteUtils.TryWriteBulkString(appendLog, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 3 + while (!RespWriteUtils.TryWriteAsciiBulkString(nodeId, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 4 + while (!RespWriteUtils.TryWriteArrayItem(physicalSublogIdx, ref curr, end)) { Flush(); curr = offset; @@ -458,14 +665,32 @@ public void SetClusterSyncHeader(string sourceNodeId, bool isMainStore) offset = curr; // 5 - // Reserve space for the bulk string header + final newline - while (ExtraSpace + 2 > (int)(end - curr)) + while (!RespWriteUtils.TryWriteArrayItem(previousAddress, ref curr, end)) { Flush(); curr = offset; } - head = curr; - curr += ExtraSpace; + offset = curr; + + // 6 + while (!RespWriteUtils.TryWriteArrayItem(currentAddress, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + // 7 + while (!RespWriteUtils.TryWriteArrayItem(nextAddress, ref curr, end)) + { + Flush(); + curr = offset; + } + offset = curr; + + Flush(); + Interlocked.Increment(ref numCommands); + return tcs.Task; } } } \ No newline at end of file diff --git a/libs/client/CompletionEvent.cs b/libs/client/CompletionEvent.cs index f8801c11e25..a6506fd5775 100644 --- a/libs/client/CompletionEvent.cs +++ b/libs/client/CompletionEvent.cs @@ -7,11 +7,13 @@ namespace Garnet.client { - // This structure uses a SemaphoreSlim as if it were a ManualResetEventSlim, because MRES does not support async waiting. + // This structure uses a SemaphoreSlim as if it were an AutoResetEvent, because ARE does not support async waiting. internal struct CompletionEvent : IDisposable { private SemaphoreSlim semaphore; + public override string ToString() => semaphore?.ToString(); + internal void Initialize() => this.semaphore = new SemaphoreSlim(0); internal void Set() diff --git a/libs/client/GarnetClient.cs b/libs/client/GarnetClient.cs index a1119e1d19a..13c9e33e39a 100644 --- a/libs/client/GarnetClient.cs +++ b/libs/client/GarnetClient.cs @@ -42,6 +42,8 @@ public sealed partial class GarnetClient : IServerHook, IMessageConsumer, IDispo static readonly Memory DECRBY = "$6\r\nDECRBY\r\n"u8.ToArray(); static readonly Memory QUIT = "$4\r\nQUIT\r\n"u8.ToArray(); static readonly Memory AUTH = "$4\r\nAUTH\r\n"u8.ToArray(); + static readonly Memory CLIENT = "$6\r\nCLIENT\r\n"u8.ToArray(); + static readonly Memory[] SETINFO = ["SETINFO"u8.ToArray(), "LIB-NAME"u8.ToArray(), "GarnetClient"u8.ToArray()]; static readonly MemoryResult RESP_OK = new(default(OK_MEM)); readonly int sendPageSize; @@ -90,6 +92,11 @@ public sealed partial class GarnetClient : IServerHook, IMessageConsumer, IDispo /// readonly string authPassword = null; + /// + /// Client name to send to server for identification. + /// + readonly Memory[] clientName = null; + /// /// Exception to throw to ongoing tasks when disposed /// @@ -122,19 +129,23 @@ public sealed partial class GarnetClient : IServerHook, IMessageConsumer, IDispo /// TLS options /// Username to authenticate with /// Password to authenticate with + /// Client name to be used with CLIENT SETNAME command /// Size of pages where requests are written to be sent, determines max request size (rounds down to previous power of 2) + /// Network writer buffer size /// Maximum outstanding tasks before client throttles new requests (rounds down to previous power of 2), default 32K /// Timeout (in milliseconds) after which client disposes itself and throws exception on all active tasks /// Pool for Memory based response buffers /// Record latency using client internal histogram /// /// Max outstanding network sends allowed + /// Shared epoch instance for thread protection; if null, a new instance is created and owned by this client /// Logger instance public GarnetClient( EndPoint endpoint, SslClientAuthenticationOptions tlsOptions = null, string authUsername = null, string authPassword = null, + string clientName = null, int sendPageSize = 1 << 21, int bufferSize = 1 << 17, int maxOutstandingTasks = 1 << 19, @@ -151,6 +162,7 @@ public GarnetClient( this.bufferSize = bufferSize; this.authUsername = authUsername; this.authPassword = authPassword; + this.clientName = clientName != null ? ["SETNAME"u8.ToArray(), Encoding.ASCII.GetBytes(clientName)] : null; if (maxOutstandingTasks > PageOffset.kTaskMask + 1) { @@ -198,7 +210,7 @@ public GarnetClient( public void Connect(CancellationToken token = default) { socket = ConnectSendSocket(); - networkWriter = new NetworkWriter(this, socket, bufferSize, sslOptions, out networkHandler, sendPageSize, networkSendThrottleMax, epoch, logger); + networkWriter = new NetworkWriter(this, socket, bufferSize, sslOptions, out networkHandler, sendPageSize, networkSendThrottleMax, epoch, PoolOwnerType.GarnetClient, logger); networkHandler.Start(sslOptions, EndPoint.ToString(), token); if (timeoutMilliseconds > 0) @@ -228,6 +240,20 @@ public void Connect(CancellationToken token = default) logger?.LogError(e, "AUTH returned error"); throw; } + + try + { + if (clientName != null) + { + _ = ExecuteForStringResultAsync(CLIENT, SETINFO).ConfigureAwait(false).GetAwaiter().GetResult(); + _ = ExecuteForStringResultAsync(CLIENT, clientName).ConfigureAwait(false).GetAwaiter().GetResult(); + } + } + catch (Exception e) + { + logger?.LogError(e, "Client set info returned error!"); + throw; + } } /// @@ -236,7 +262,7 @@ public void Connect(CancellationToken token = default) public async Task ConnectAsync(CancellationToken token = default) { socket = await ConnectSendSocketAsync(timeoutMilliseconds, token).ConfigureAwait(false); - networkWriter = new NetworkWriter(this, socket, bufferSize, sslOptions, out networkHandler, sendPageSize, networkSendThrottleMax, epoch, logger); + networkWriter = new NetworkWriter(this, socket, bufferSize, sslOptions, out networkHandler, sendPageSize, networkSendThrottleMax, epoch, PoolOwnerType.GarnetClient, logger); await networkHandler.StartAsync(sslOptions, EndPoint.ToString(), token).ConfigureAwait(false); if (timeoutMilliseconds > 0) @@ -257,7 +283,35 @@ public async Task ConnectAsync(CancellationToken token = default) } catch (Exception e) { - logger?.LogError(e, "AUTH returned error"); + logger?.LogError(e, "AUTH returned error!"); + throw; + } + + try + { + if (clientName != null) + { + _ = await ExecuteForStringResultAsync(CLIENT, SETINFO).ConfigureAwait(false); + _ = await ExecuteForStringResultAsync(CLIENT, clientName).ConfigureAwait(false); + } + } + catch (Exception e) + { + logger?.LogError(e, "Client set info returned error"); + throw; + } + + try + { + if (clientName != null) + { + _ = await ExecuteForStringResultAsync(CLIENT, SETINFO).ConfigureAwait(false); + _ = await ExecuteForStringResultAsync(CLIENT, clientName).ConfigureAwait(false); + } + } + catch (Exception e) + { + logger?.LogError(e, "Client set info returned error!"); throw; } } @@ -314,7 +368,7 @@ private async Task ConnectSendSocketAsync(int millisecondsTimeout = 0, C NoDelay = true }; - if (await TryConnectSocketAsync(socket, endpoint, millisecondsTimeout, cancellationToken)) + if (await TryConnectSocketAsync(socket, endpoint, millisecondsTimeout, cancellationToken).ConfigureAwait(false)) return socket; } } @@ -324,7 +378,7 @@ private async Task ConnectSendSocketAsync(int millisecondsTimeout = 0, C if (EndPoint is not UnixDomainSocketEndPoint) socket.NoDelay = true; - if (await TryConnectSocketAsync(socket, EndPoint, millisecondsTimeout, cancellationToken)) + if (await TryConnectSocketAsync(socket, EndPoint, millisecondsTimeout, cancellationToken).ConfigureAwait(false)) return socket; } @@ -370,12 +424,12 @@ private async Task TryConnectSocketAsync(Socket socket, EndPoint endpoint, using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); var connectTask = socket.ConnectAsync(endpoint, timeoutCts.Token).AsTask(); - if (await Task.WhenAny(connectTask, Task.Delay(millisecondsTimeout, timeoutCts.Token)) == connectTask) + if (await Task.WhenAny(connectTask, Task.Delay(millisecondsTimeout, timeoutCts.Token)).ConfigureAwait(false) == connectTask) { // Task completed within timeout. // Consider that the task may have faulted or been canceled. // We re-await the task so that any exceptions/cancellation is rethrown. - await connectTask; + await connectTask.ConfigureAwait(false); } else { @@ -413,7 +467,7 @@ async Task TimeoutChecker() var _tcsOffset = tcsOffset; var _tailAddress = networkWriter.GetTailAddress(); - await Task.Delay(timeoutMilliseconds, token); + await Task.Delay(timeoutMilliseconds, token).ConfigureAwait(false); // Check if no new tasks added + no new results processed var _newTcsOffset = tcsOffset; var _newNextTaskId = networkWriter.GetNextTaskId(); @@ -451,7 +505,7 @@ public async Task ReconnectAsync(CancellationToken token = default) networkWriter?.Dispose(); } catch { } - await ConnectAsync(token); + await ConnectAsync(token).ConfigureAwait(false); } /// @@ -479,7 +533,7 @@ void Dispose(bool disposing) void CheckLength(int totalLen, TcsWrapper tcs) { - if (totalLen > networkWriter.PageSize) + if (networkWriter is not null && totalLen > networkWriter.PageSize) { var e = new Exception($"Entry of size {totalLen} does not fit on page of size {networkWriter.PageSize}. Try increasing sendPageSize parameter to GarnetClient constructor."); switch (tcs.taskType) @@ -529,7 +583,7 @@ async ValueTask InputGateAsync(CancellationToken token = default) { if (PipelineLength() < maxOutstandingTasks) break; - await Task.Delay(delayMs, token); + await Task.Delay(delayMs, token).ConfigureAwait(false); if (delayMs == 0) delayMs = 1; else delayMs *= 2; if (delayMs > 4096) delayMs = 4096; @@ -602,7 +656,7 @@ async ValueTask InternalExecuteAsync(TcsWrapper tcs, Memory op, string par totalLen += 1 + NumUtils.CountDigits(arraySize) + 2; CheckLength(totalLen, tcs); - await InputGateAsync(token); + await InputGateAsync(token).ConfigureAwait(false); try { @@ -664,7 +718,7 @@ async ValueTask InternalExecuteAsync(TcsWrapper tcs, Memory op, string par try { networkWriter.epoch.Suspend(); - await AwaitPreviousTaskAsync(taskId); // does not take token, as task is not cancelable at this point + await AwaitPreviousTaskAsync(taskId).ConfigureAwait(false); // does not take token, as task is not cancelable at this point } finally { @@ -715,7 +769,7 @@ async ValueTask InternalExecuteAsync(TcsWrapper tcs, Memory op, Memory op, Memory respOp, IColle } CheckLength(totalLen, tcs); - await InputGateAsync(token); + await InputGateAsync(token).ConfigureAwait(false); try { @@ -1096,7 +1150,7 @@ async ValueTask InternalExecuteAsync(TcsWrapper tcs, Memory respOp, IColle try { networkWriter.epoch.Suspend(); - await AwaitPreviousTaskAsync(taskId); // does not take token, as task is not cancelable at this point + await AwaitPreviousTaskAsync(taskId).ConfigureAwait(false); // does not take token, as task is not cancelable at this point } finally { diff --git a/libs/client/GarnetClientAPI/GarnetClientBasicRespCommands.cs b/libs/client/GarnetClientAPI/GarnetClientBasicRespCommands.cs index abb58d6a334..4bf9983a45c 100644 --- a/libs/client/GarnetClientAPI/GarnetClientBasicRespCommands.cs +++ b/libs/client/GarnetClientAPI/GarnetClientBasicRespCommands.cs @@ -572,7 +572,7 @@ public async Task StringDecrement(Memory key, long value) /// Value /// public async Task StringDecrement(Memory key, long value, CancellationToken token) - => long.Parse(await ExecuteForStringResultWithCancellationAsync(DECRBY, key, Encoding.ASCII.GetBytes(value.ToString()), token)); + => long.Parse(await ExecuteForStringResultWithCancellationAsync(DECRBY, key, Encoding.ASCII.GetBytes(value.ToString()), token).ConfigureAwait(false)); /// /// Decrement number stored at key by value. diff --git a/libs/client/GarnetClientAPI/GarnetClientExecuteAPI.cs b/libs/client/GarnetClientAPI/GarnetClientExecuteAPI.cs index c9b03e2d564..02c3101abf4 100644 --- a/libs/client/GarnetClientAPI/GarnetClientExecuteAPI.cs +++ b/libs/client/GarnetClientAPI/GarnetClientExecuteAPI.cs @@ -149,13 +149,13 @@ public async Task ExecuteForStringResultWithCancellationAsync(Memory ExecuteForStringResultWithCancellationAsync(Memory ExecuteForStringResultWithCancellationAsync(string op, using (token.Register(TokenRegistrationStringCallback, tcs.stringTcs)) { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.stringTcs.Task; + return await tcs.stringTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.stringTcs.Task; + return await tcs.stringTcs.Task.ConfigureAwait(false); } } @@ -377,13 +377,13 @@ public async Task> ExecuteForMemoryResultWithCancellationAsyn using (token.Register(TokenRegistrationMemoryResultCallback, tcs.memoryByteTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } @@ -403,13 +403,13 @@ public async Task> ExecuteForMemoryResultWithCancellationAsyn using (token.Register(TokenRegistrationMemoryResultCallback, tcs.memoryByteTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } @@ -428,13 +428,13 @@ public async Task> ExecuteForMemoryResultWithCancellationAsyn using (token.Register(TokenRegistrationMemoryResultCallback, tcs.memoryByteTcs)) { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } @@ -452,13 +452,13 @@ public async Task> ExecuteForMemoryResultWithCancellationAsyn using (token.Register(TokenRegistrationMemoryResultCallback, tcs.memoryByteTcs)) { var _ = InternalExecuteAsync(tcs, respOp, args, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, args, token); - return await tcs.memoryByteTcs.Task; + return await tcs.memoryByteTcs.Task.ConfigureAwait(false); } } @@ -601,13 +601,13 @@ public async Task ExecuteForStringArrayResultWithCancellationAsync(Mem using (token.Register(TokenRegistrationStringArrayCallback, tcs.stringArrayTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } @@ -628,13 +628,13 @@ public async Task ExecuteForStringArrayResultWithCancellationAsync(Mem using (token.Register(TokenRegistrationStringArrayCallback, tcs.stringArrayTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } @@ -654,13 +654,13 @@ public async Task ExecuteForStringArrayResultWithCancellationAsync(str using (token.Register(TokenRegistrationStringArrayCallback, tcs.stringArrayTcs)) { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } @@ -679,13 +679,13 @@ public async Task ExecuteForStringArrayResultWithCancellationAsync(Mem using (token.Register(TokenRegistrationStringArrayCallback, tcs.stringArrayTcs)) { var _ = InternalExecuteAsync(tcs, respOp, args, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, args, token); - return await tcs.stringArrayTcs.Task; + return await tcs.stringArrayTcs.Task.ConfigureAwait(false); } } @@ -829,13 +829,13 @@ public async Task[]> ExecuteForMemoryResultArrayWithCancellat using (token.Register(TokenRegistrationMemoryResultArrayCallback, tcs.memoryByteArrayTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } @@ -856,13 +856,13 @@ public async Task[]> ExecuteForMemoryResultArrayWithCancellat using (token.Register(TokenRegistrationMemoryResultArrayCallback, tcs.memoryByteArrayTcs)) { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, respOp, param1, param2, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } @@ -883,13 +883,13 @@ public async Task[]> ExecuteForMemoryResultArrayWithCancellat using (token.Register(TokenRegistrationMemoryResultArrayCallback, tcs.memoryByteArrayTcs)) { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.memoryByteArrayTcs.Task; + return await tcs.memoryByteArrayTcs.Task.ConfigureAwait(false); } } @@ -1034,13 +1034,13 @@ public async Task ExecuteForLongResultWithCancellationAsync(string op, ICo using (token.Register(TokenRegistrationLongCallback, tcs.longTcs)) { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.longTcs.Task; + return await tcs.longTcs.Task.ConfigureAwait(false); } } else { var _ = InternalExecuteAsync(tcs, op, args, token); - return await tcs.longTcs.Task; + return await tcs.longTcs.Task.ConfigureAwait(false); } } diff --git a/libs/client/LightEpoch.cs b/libs/client/LightEpoch.cs index a900a6ad6bc..0a1a13e15b7 100644 --- a/libs/client/LightEpoch.cs +++ b/libs/client/LightEpoch.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -19,28 +20,28 @@ public sealed unsafe class LightEpoch /// (1) in AssignInstance, to assign a unique instanceId to each LightEpoch instance, and /// (2) in Metadata, to track per-thread epoch table entries for each LightEpoch instance. /// - [StructLayout(LayoutKind.Explicit, Size = MaxInstances * sizeof(int))] + [InlineArray(MaxInstances)] private struct InstanceIndexBuffer { /// /// Maximum number of concurrent instances of LightEpoch supported. /// - internal const int MaxInstances = 16; + internal const int MaxInstances = 1024; /// /// Anchor field for the buffer. /// - [FieldOffset(0)] int field0; /// /// Reference to the entry for the given instance ID. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] + [UnscopedRef] internal ref int GetRef(int instanceId) { Debug.Assert(instanceId >= 0 && instanceId < MaxInstances); - return ref Unsafe.AsRef((int*)Unsafe.AsPointer(ref field0) + instanceId); + return ref Unsafe.Add(ref field0, instanceId); } } @@ -193,7 +194,7 @@ int SelectInstance() if (kInvalidIndex == Interlocked.CompareExchange(ref entry, 1, kInvalidIndex)) return i; } - throw new InvalidOperationException("Exceeded maximum number of active LightEpoch instances"); + throw new InvalidOperationException($"Exceeded maximum number of active LightEpoch instances {ActiveInstanceCount()} {InstanceIndexBuffer.MaxInstances}"); } /// @@ -422,7 +423,7 @@ long ComputeNewSafeToReclaimEpoch(long currentEpoch) { long oldestOngoingCall = currentEpoch; - for (int index = 1; index <= kTableSize; ++index) + for (int index = 1; index <= kTableSize; index++) { long entry_epoch = (*(tableAligned + index)).localCurrentEpoch; if (0 != entry_epoch) @@ -450,7 +451,7 @@ void SuspendDrain() // Barrier ensures we see the latest epoch table entries. Ensures // that the last suspended thread drains all pending actions. Thread.MemoryBarrier(); - for (int index = 1; index <= kTableSize; ++index) + for (int index = 1; index <= kTableSize; index++) { long entry_epoch = (*(tableAligned + index)).localCurrentEpoch; if (0 != entry_epoch) diff --git a/libs/client/NetworkWriter.cs b/libs/client/NetworkWriter.cs index dd2f5ff6c0e..9c5ac69e795 100644 --- a/libs/client/NetworkWriter.cs +++ b/libs/client/NetworkWriter.cs @@ -79,10 +79,10 @@ internal sealed class NetworkWriter : IDisposable /// /// Constructor /// - public NetworkWriter(GarnetClient serverHook, Socket socket, int messageBufferSize, SslClientAuthenticationOptions sslOptions, out GarnetClientTcpNetworkHandler networkHandler, int sendPageSize, int networkSendThrottleMax, LightEpoch epoch, ILogger logger = null) + public NetworkWriter(GarnetClient serverHook, Socket socket, int messageBufferSize, SslClientAuthenticationOptions sslOptions, out GarnetClientTcpNetworkHandler networkHandler, int sendPageSize, int networkSendThrottleMax, LightEpoch epoch, PoolOwnerType ownerType, ILogger logger = null) { this.networkBufferSettings = new NetworkBufferSettings(messageBufferSize, messageBufferSize); - this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); + this.networkPool = networkBufferSettings.CreateBufferPool(ownerType: ownerType, logger: logger); if (BufferSize > PageOffset.kPageMask) throw new Exception(); this.networkHandler = networkHandler = new GarnetClientTcpNetworkHandler(serverHook, AsyncFlushPageCallback, socket, networkBufferSettings, networkPool, sslOptions != null, serverHook, networkSendThrottleMax: networkSendThrottleMax, logger: logger); diff --git a/libs/client/Utility.cs b/libs/client/Utility.cs index 3e008e66d40..45965a86a7e 100644 --- a/libs/client/Utility.cs +++ b/libs/client/Utility.cs @@ -192,14 +192,14 @@ private static async Task SlowWithCancellationAsync(Task task, Cancella var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using (token.Register(s => ((TaskCompletionSource)s).TrySetResult(true), tcs, useSynchronizationContext)) { - if (task != await Task.WhenAny(task, tcs.Task)) + if (task != await Task.WhenAny(task, tcs.Task).ConfigureAwait(false)) { token.ThrowIfCancellationRequested(); } } // make sure any exceptions in the task get unwrapped and exposed to the caller. - return await task; + return await task.ConfigureAwait(false); } } } \ No newline at end of file diff --git a/libs/cluster/AssemblyInfo.cs b/libs/cluster/AssemblyInfo.cs index 56f4de4819d..6695ff9752c 100644 --- a/libs/cluster/AssemblyInfo.cs +++ b/libs/cluster/AssemblyInfo.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("BDN.benchmark" + ClusterAssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Resp.benchmark" + ClusterAssemblyRef.GarnetPublicKey)] [assembly: InternalsVisibleTo("Garnet.test.cluster" + ClusterAssemblyRef.GarnetPublicKey)] /// diff --git a/libs/cluster/ClusterFactory.cs b/libs/cluster/ClusterFactory.cs index ef39a4fba7c..753d896def7 100644 --- a/libs/cluster/ClusterFactory.cs +++ b/libs/cluster/ClusterFactory.cs @@ -13,8 +13,8 @@ namespace Garnet.cluster public class ClusterFactory : IClusterFactory { /// - public DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default) - => new GarnetClusterCheckpointManager(deviceFactoryCreator, checkpointNamingScheme, isMainStore, logger: logger); + public DeviceLogCommitCheckpointManager CreateCheckpointManager(int aofPhysicalSublogCount, INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default) + => new GarnetClusterCheckpointManager(aofPhysicalSublogCount, deviceFactoryCreator, checkpointNamingScheme, isMainStore, logger: logger); /// public IClusterProvider CreateClusterProvider(StoreWrapper store) diff --git a/libs/cluster/CmdStrings.cs b/libs/cluster/CmdStrings.cs index d8d4dfcce36..0f9fc94e1a2 100644 --- a/libs/cluster/CmdStrings.cs +++ b/libs/cluster/CmdStrings.cs @@ -20,6 +20,7 @@ static class CmdStrings /// public static ReadOnlySpan failstopwrites => "FAILSTOPWRITES"u8; public static ReadOnlySpan failreplicationoffset => "FAILREPLICATIONOFFSET"u8; + public static ReadOnlySpan FRONTIER => "FRONTIER"u8; /// @@ -54,6 +55,7 @@ static class CmdStrings public static ReadOnlySpan RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER => "ERR value is not an integer or out of range."u8; public static ReadOnlySpan RESP_ERR_GENERIC_VALUE_IS_NOT_BOOLEAN => "ERR value is not a boolean."u8; public static ReadOnlySpan RESP_ERR_RESET_WITH_KEYS_ASSIGNED => "-ERR CLUSTER RESET can't be called with master nodes containing keys\r\n"u8; + public static ReadOnlySpan RESP_ERR_MULTI_LOG_DISABLED => "ERR Multi-log disabled"u8; public static ReadOnlySpan RESP_SYNTAX_ERROR => "ERR syntax error"u8; /// diff --git a/libs/cluster/Garnet.cluster.csproj b/libs/cluster/Garnet.cluster.csproj index 08393085ef6..06e8d930c0f 100644 --- a/libs/cluster/Garnet.cluster.csproj +++ b/libs/cluster/Garnet.cluster.csproj @@ -17,7 +17,17 @@ - + + + + $(DefineConstants);CLUSTER_PROJECT + + + + + + + \ No newline at end of file diff --git a/libs/cluster/Server/ClusterConfig.cs b/libs/cluster/Server/ClusterConfig.cs index 5727672c120..440315c01d5 100644 --- a/libs/cluster/Server/ClusterConfig.cs +++ b/libs/cluster/Server/ClusterConfig.cs @@ -40,6 +40,12 @@ internal sealed partial class ClusterConfig /// public const int MAX_HASH_SLOT_VALUE = 16384; + /// + /// Version of the cluster config serialization format. + /// Increment when the binary layout of / changes. + /// + public const byte ClusterConfigVersion = 1; + /// /// /// @@ -250,6 +256,11 @@ public bool IsKnown(string nodeid) /// Config epoch of local node. public long LocalNodeConfigEpoch => workers[LOCAL_WORKER_ID].ConfigEpoch; + /// + /// Local endpoint string + /// + public string LocalNodeEndpoint => $"{workers[LOCAL_WORKER_ID].Address}:{workers[LOCAL_WORKER_ID].Port}"; + /// /// Return endpoint of primary if this node is a replica. /// @@ -1171,7 +1182,7 @@ public ClusterConfig MergeSlotMap(ClusterConfig senderConfig, ILogger logger = n // will not be able to claim the slot without outside intervention if (currentOwnerNodeId != null && currentOwnerNodeId.Equals(senderConfig.LocalNodeId, StringComparison.OrdinalIgnoreCase)) { - logger?.LogWarning("MergeReset: {senderConfig.LocalNodeIdShort} > {i} > {LocalNodeIdShort}", senderConfig.LocalNodeIdShort, i, LocalNodeIdShort); + // logger?.LogWarning("MergeReset: {senderConfig.LocalNodeIdShort} > {i} > {LocalNodeIdShort}", senderConfig.LocalNodeIdShort, i, LocalNodeIdShort); newSlotMap[i]._workerId = RESERVED_WORKER_ID; newSlotMap[i]._state = SlotState.OFFLINE; } diff --git a/libs/cluster/Server/ClusterConfigSerializer.cs b/libs/cluster/Server/ClusterConfigSerializer.cs index ba5dc7a253f..472c53ef6c8 100644 --- a/libs/cluster/Server/ClusterConfigSerializer.cs +++ b/libs/cluster/Server/ClusterConfigSerializer.cs @@ -1,20 +1,40 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.IO; -using System.Text; namespace Garnet.cluster { internal sealed partial class ClusterConfig { + /// + /// Peek the serialization version from a config byte array without full deserialization. + /// + /// Serialized cluster config payload. + /// The version byte at the start of the payload. + /// True if the payload is large enough to contain a version byte; false otherwise. + public static bool TryPeekVersion(ReadOnlySpan data, out byte version) + { + if (data.Length < 1) + { + version = 0; + return false; + } + version = data[0]; + return true; + } + /// /// Serialize config to byte array /// public byte[] ToByteArray() { var ms = new MemoryStream(); - var writer = new BinaryWriter(ms, Encoding.ASCII); + var writer = new BinaryWriter(ms); + + // Write serialization format version + writer.Write(ClusterConfigVersion); SerializeSlotMap(ref ms, ref writer); @@ -60,6 +80,7 @@ public byte[] ToByteArray() private void SerializeSlotMap(ref MemoryStream ms, ref BinaryWriter writer) { //serialize slotMap + var segmentCountPosition = ms.Position; ms.Position += 2; ushort segmentCount = 0; ushort count = 1; @@ -93,9 +114,9 @@ private void SerializeSlotMap(ref MemoryStream ms, ref BinaryWriter writer) writer.Write(workerId); writer.Write(state); - //Write segment count in the beginning of memory stream + //Write segment count at the reserved position var _position = ms.Position; - ms.Position = 0; + ms.Position = segmentCountPosition; writer.Write(segmentCount); ms.Position = _position; } @@ -108,6 +129,13 @@ public static ClusterConfig FromByteArray(byte[] other) var ms = new MemoryStream(other); var reader = new BinaryReader(ms); + // Read and validate serialization format version + if (other.Length < 1) + throw new InvalidDataException("Invalid ClusterConfig payload: too short to contain a version"); + var version = reader.ReadByte(); + if (version != ClusterConfigVersion) + throw new InvalidDataException($"Incompatible ClusterConfig version: expected {ClusterConfigVersion}, got {version}"); + var newSlotMap = DeserializeSlotMap(ref reader); int numWorkers = reader.ReadInt32(); diff --git a/libs/cluster/Server/ClusterManager.cs b/libs/cluster/Server/ClusterManager.cs index fa247423ffa..93adc2b152d 100644 --- a/libs/cluster/Server/ClusterManager.cs +++ b/libs/cluster/Server/ClusterManager.cs @@ -22,6 +22,12 @@ internal sealed partial class ClusterManager : IDisposable readonly SectorAlignedBufferPool pool; readonly ILogger logger; + /// + /// NOTE: Unsafe! DO NOT USE, other than benchmarking + /// + /// + public void UnsafeSetConfig(ClusterConfig clusterConfig) => currentConfig = clusterConfig; + /// /// Get current config /// @@ -63,13 +69,14 @@ public ClusterManager(ClusterProvider clusterProvider, ILogger logger = null) var clusterDataPath = serverOptions.CheckpointDir + clusterFolder; var deviceFactory = serverOptions.GetInitializedDeviceFactory(clusterDataPath); + clusterConfigDevice = deviceFactory.Get(new FileDescriptor(directoryName: "", fileName: "nodes.conf")); pool = new(1, (int)clusterConfigDevice.SectorSize); var clusterEndpoint = clusterProvider.storeWrapper.GetClusterEndpoint(); this.logger = logger; - var recoverConfig = clusterConfigDevice.GetFileSize(0) > 0 && !serverOptions.CleanClusterConfig; + var recoverConfig = clusterProvider.serverOptions.ClusterConfigFlushFrequencyMs != -1 && clusterConfigDevice.GetFileSize(0) > 0 && !serverOptions.CleanClusterConfig; tlsOptions = serverOptions.TlsOptions; if (!serverOptions.CleanClusterConfig) @@ -248,27 +255,22 @@ public string GetInfo() public static string GetRange(int[] slots) { var range = "> "; - if (slots.Length >= 1) + var start = slots[0]; + var end = slots[0]; + for (var i = 1; i < slots.Length + 1; i++) { - - var start = slots[0]; - var end = slots[0]; - for (var i = 1; i < slots.Length + 1; i++) + if (i < slots.Length && slots[i] == end + 1) + end = slots[i]; + else { - if (i < slots.Length && slots[i] == end + 1) - end = slots[i]; - else + range += $"{start}-{end} "; + if (i < slots.Length) { - range += $"{start}-{end} "; - if (i < slots.Length) - { - start = slots[i]; - end = slots[i]; - } + start = slots[i]; + end = slots[i]; } } } - return range; } diff --git a/libs/cluster/Server/ClusterManagerSlotState.cs b/libs/cluster/Server/ClusterManagerSlotState.cs index 6d673a81917..17c1631ebd3 100644 --- a/libs/cluster/Server/ClusterManagerSlotState.cs +++ b/libs/cluster/Server/ClusterManagerSlotState.cs @@ -5,23 +5,10 @@ using System.Collections.Generic; using System.Text; using System.Threading; -using Garnet.common; -using Garnet.server; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.cluster { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - /// /// Cluster manager /// @@ -473,38 +460,11 @@ public void TryResetSlotState(HashSet slots) } /// - /// Methods used to cleanup keys for given slot collection in main store + /// Methods used to cleanup keys for given slot collection /// - /// + /// /// Slot list - public static unsafe void DeleteKeysInSlotsFromMainStore(BasicGarnetApi BasicGarnetApi, HashSet slots) - { - using var iter = BasicGarnetApi.IterateMainStore(); - while (iter.GetNext(out _)) - { - ref var key = ref iter.GetKey(); - var s = HashSlotUtils.HashSlot(ref key); - if (slots.Contains(s)) - _ = BasicGarnetApi.DELETE(ref key, StoreType.Main); - } - } - - /// - /// Methods used to cleanup keys for given slot collection in object store - /// - /// - /// Slot list - public static unsafe void DeleteKeysInSlotsFromObjectStore(BasicGarnetApi BasicGarnetApi, HashSet slots) - { - using var iterObject = BasicGarnetApi.IterateObjectStore(); - while (iterObject.GetNext(out _)) - { - ref var key = ref iterObject.GetKey(); - ref var value = ref iterObject.GetValue(); - var s = HashSlotUtils.HashSlot(key); - if (slots.Contains(s)) - _ = BasicGarnetApi.DELETE(key, StoreType.Object); - } - } + public static void DeleteKeysInSlots(BasicGarnetApi basicGarnetApi, HashSet slots) + => basicGarnetApi.DeleteSlotKeys(slots); } } \ No newline at end of file diff --git a/libs/cluster/Server/ClusterProvider.cs b/libs/cluster/Server/ClusterProvider.cs index 0881036db5e..b1d8f79a140 100644 --- a/libs/cluster/Server/ClusterProvider.cs +++ b/libs/cluster/Server/ClusterProvider.cs @@ -12,30 +12,13 @@ using Garnet.server.ACL; using Garnet.server.Auth; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.cluster { - using BasicContext = BasicContext, - SpanByteAllocator>>; - - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - - using VectorContext = BasicContext, SpanByteAllocator>>; - /// /// Cluster provider /// - public class ClusterProvider : IClusterProvider + public sealed partial class ClusterProvider : IClusterProvider { internal readonly ClusterManager clusterManager; internal readonly ReplicationManager replicationManager; @@ -86,7 +69,7 @@ public ClusterProvider(StoreWrapper storeWrapper) /// public bool AllowDataLoss - => serverOptions.UseAofNullDevice || (serverOptions.FastAofTruncate && !serverOptions.OnDemandCheckpoint); + => serverOptions.AllowDataLoss; /// public void Recover() @@ -96,11 +79,11 @@ public void Recover() /// public bool PreventRoleChange() - => replicationManager.BeginRecovery(RecoveryStatus.ReadRole, upgradeLock: false); + => replicationManager.BeginRecovery(RecoveryStatus.ReadRole, upgradeLock: false); /// public void AllowRoleChange() - => replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); + => replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); /// public void Start() @@ -110,8 +93,8 @@ public void Start() } /// - public IClusterSession CreateClusterSession(TransactionManager txnManager, IGarnetAuthenticator authenticator, UserHandle userHandle, GarnetSessionMetrics garnetSessionMetrics, BasicGarnetApi basicGarnetApi, BasicContext basicContext, VectorContext vectorContext, INetworkSender networkSender, ILogger logger = null) - => new ClusterSession(this, txnManager, authenticator, userHandle, garnetSessionMetrics, basicGarnetApi, basicContext, vectorContext, networkSender, logger); + public IClusterSession CreateClusterSession(TransactionManager txnManager, IGarnetAuthenticator authenticator, UserHandle userHandle, GarnetSessionMetrics garnetSessionMetrics, BasicGarnetApi basicGarnetApi, StringBasicContext stringContext, VectorBasicContext vectorContext, INetworkSender networkSender, ILogger logger = null) + => new ClusterSession(this, txnManager, authenticator, userHandle, garnetSessionMetrics, basicGarnetApi, stringContext, vectorContext, networkSender, logger); /// public void UpdateClusterAuth(string clusterUsername, string clusterPassword) @@ -167,13 +150,12 @@ public void FlushConfig() public void FlushDB(bool unsafeTruncateLog = false) { storeWrapper.store.Log.ShiftBeginAddress(storeWrapper.store.Log.TailAddress, truncateLog: unsafeTruncateLog); - storeWrapper.objectStore?.Log.ShiftBeginAddress(storeWrapper.objectStore.Log.TailAddress, truncateLog: unsafeTruncateLog); } /// - public void SafeTruncateAOF(bool full, long CheckpointCoveredAofAddress, Guid storeCheckpointToken, Guid objectStoreCheckpointToken) + public void AddNewCheckpointEntry(bool full, AofAddress CheckpointCoveredAofAddress, Guid storeCheckpointToken, Guid objectStoreCheckpointToken) { - var entry = new CheckpointEntry(); + var entry = new CheckpointEntry(storeWrapper.serverOptions.AofPhysicalSublogCount); entry.metadata.storeVersion = storeWrapper.store.CurrentVersion; entry.metadata.storeHlogToken = storeCheckpointToken; @@ -181,12 +163,6 @@ public void SafeTruncateAOF(bool full, long CheckpointCoveredAofAddress, Guid st entry.metadata.storeCheckpointCoveredAofAddress = CheckpointCoveredAofAddress; entry.metadata.storePrimaryReplId = replicationManager.PrimaryReplId; - entry.metadata.objectStoreVersion = serverOptions.DisableObjects ? -1 : storeWrapper.objectStore.CurrentVersion; - entry.metadata.objectStoreHlogToken = serverOptions.DisableObjects ? default : objectStoreCheckpointToken; - entry.metadata.objectStoreIndexToken = serverOptions.DisableObjects ? default : objectStoreCheckpointToken; - entry.metadata.objectCheckpointCoveredAofAddress = CheckpointCoveredAofAddress; - entry.metadata.objectStorePrimaryReplId = replicationManager.PrimaryReplId; - // Keep track of checkpoints for replica // Used to delete old checkpoints and cleanup and also cleanup during attachment to new primary replicationManager.AddCheckpointEntry(entry, full); @@ -196,24 +172,23 @@ public void SafeTruncateAOF(bool full, long CheckpointCoveredAofAddress, Guid st } /// - public void SafeTruncateAOF(long truncateUntil) + public void SafeTruncateAOF(in AofAddress truncateUntil) { if (clusterManager.CurrentConfig.LocalNodeRole == NodeRole.PRIMARY) - _ = replicationManager.SafeTruncateAof(truncateUntil); + replicationManager.AofSyncDriverStore.SafeTruncateAof(truncateUntil); else { if (serverOptions.FastAofTruncate) - storeWrapper.appendOnlyFile?.UnsafeShiftBeginAddress(truncateUntil, truncateLog: true); + storeWrapper.appendOnlyFile?.Log.UnsafeShiftBeginAddress(truncateUntil, truncateLog: true); else { - storeWrapper.appendOnlyFile?.TruncateUntil(truncateUntil); - if (!serverOptions.EnableFastCommit) storeWrapper.appendOnlyFile?.Commit(); + storeWrapper.appendOnlyFile?.Log.TruncateUntil(truncateUntil); } } } /// - public void OnCheckpointInitiated(out long CheckpointCoveredAofAddress) + public void OnCheckpointInitiated(ref AofAddress CheckpointCoveredAofAddress) { Debug.Assert(serverOptions.EnableCluster); if (serverOptions.EnableAOF && clusterManager.CurrentConfig.LocalNodeRole == NodeRole.REPLICA) @@ -222,12 +197,13 @@ public void OnCheckpointInitiated(out long CheckpointCoveredAofAddress) // until the checkpoint start marker. Otherwise, we will be left with an AOF that starts at the checkpoint end marker. // ReplicationCheckpointStartOffset is set by { ReplicaReplayTask.Consume -> AofProcessor.ProcessAofRecordInternal } when // it encounters the checkpoint start marker. + CheckpointCoveredAofAddress = replicationManager.ReplicationCheckpointStartOffset; } else - CheckpointCoveredAofAddress = storeWrapper.appendOnlyFile.TailAddress; + CheckpointCoveredAofAddress = storeWrapper.appendOnlyFile.Log.TailAddress; - replicationManager?.UpdateCommitSafeAofAddress(CheckpointCoveredAofAddress); + replicationManager?.UpdateCommitSafeAofAddress(ref CheckpointCoveredAofAddress); } /// @@ -251,10 +227,9 @@ public MetricsItem[] GetReplicationInfo() new("second_repl_offset", replication_offset2), new("store_current_safe_aof_address", clusterEnabled ? replicationManager.StoreCurrentSafeAofAddress.ToString() : "N/A"), new("store_recovered_safe_aof_address", clusterEnabled ? replicationManager.StoreRecoveredSafeAofTailAddress.ToString() : "N/A"), - new("object_store_current_safe_aof_address", clusterEnabled && !serverOptions.DisableObjects ? replicationManager.ObjectStoreCurrentSafeAofAddress.ToString() : "N/A"), - new("object_store_recovered_safe_aof_address", clusterEnabled && !serverOptions.DisableObjects ? replicationManager.ObjectStoreRecoveredSafeAofTailAddress.ToString() : "N/A"), new("recover_status", replicationManager.currentRecoveryStatus.ToString()), - new("last_failover_state", !clusterEnabled ? FailoverUtils.GetFailoverStatus(FailoverStatus.NO_FAILOVER) : failoverManager.GetLastFailoverStatus()) + new("last_failover_state", !clusterEnabled ? FailoverUtils.GetFailoverStatus(FailoverStatus.NO_FAILOVER) : failoverManager.GetLastFailoverStatus()), + new("sync_driver_count", !clusterEnabled ? "0" : replicationManager.AofSyncDriverStore.AofSyncDriverCount.ToString()) }; if (clusterEnabled) @@ -263,7 +238,7 @@ public MetricsItem[] GetReplicationInfo() { var (address, port) = config.GetLocalNodePrimaryAddress(); var primaryLinkStatus = clusterManager.GetPrimaryLinkStatus(config); - var replicationOffsetLag = storeWrapper.appendOnlyFile.TailAddress - replicationManager.ReplicationOffset; + var replicationOffsetLag = storeWrapper.appendOnlyFile.Log.TailAddress.AggregateDiff(replicationManager.ReplicationOffset); replicationInfo.Add(new("master_host", address)); replicationInfo.Add(new("master_port", port.ToString())); replicationInfo.Add(primaryLinkStatus[0]); @@ -287,18 +262,18 @@ public MetricsItem[] GetReplicationInfo() return [.. replicationInfo]; } + /// public MetricsItem[] GetCheckpointInfo() => [new("memory_checkpoint_entry", replicationManager.GetLatestCheckpointFromMemoryInfo()), new("disk_checkpoint_entry", replicationManager.GetLatestCheckpointFromDiskInfo())]; /// - public (long replication_offset, List replicaInfo) GetPrimaryInfo() + public (AofAddress replication_offset, List replicaInfo) GetPrimaryInfo() { if (!serverOptions.EnableCluster) { return (replicationManager.ReplicationOffset, default); } - return (replicationManager.ReplicationOffset, replicationManager.GetReplicaInfo()); } @@ -328,12 +303,6 @@ public RoleInfo GetReplicaInfo() return info; } - /// - public long GetReplicationOffset() - { - return replicationManager.ReplicationOffset; - } - /// public MetricsItem[] GetGossipStats(bool metricsDisabled) { @@ -367,85 +336,16 @@ public void PurgeBufferPool(ManagerType managerType) throw new GarnetException(); } - public void ExtractKeySpecs(RespCommandsInfo commandInfo, RespCommand cmd, ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi) - { - var specs = commandInfo.KeySpecifications; - switch (specs.Length) - { - case 1: - var searchIndex = (BeginSearchIndex)specs[0].BeginSearch; - csvi.readOnly = specs[0].Flags.HasFlag(KeySpecificationFlags.RO); - switch (specs[0].FindKeys) - { - case FindKeysRange: - var findRange = (FindKeysRange)specs[0].FindKeys; - csvi.firstKey = searchIndex.Index - 1; - csvi.lastKey = findRange.LastKey < 0 ? findRange.LastKey + parseState.Count + 1 : findRange.LastKey - searchIndex.Index + 1; - csvi.step = findRange.KeyStep; - csvi.readOnly = !specs[0].Flags.HasFlag(KeySpecificationFlags.RW); - break; - case FindKeysKeyNum: - var findKeysKeyNum = (FindKeysKeyNum)specs[0].FindKeys; - csvi.firstKey = searchIndex.Index + findKeysKeyNum.FirstKey - 1; - csvi.lastKey = csvi.firstKey + parseState.GetInt(searchIndex.Index + findKeysKeyNum.KeyNumIdx - 1); - csvi.step = findKeysKeyNum.KeyStep; - break; - case FindKeysUnknown: - default: - throw new GarnetException("FindKeys spec not known"); - } - - break; - case 2: - searchIndex = (BeginSearchIndex)specs[0].BeginSearch; - switch (specs[0].FindKeys) - { - case FindKeysRange: - csvi.firstKey = RespCommand.BITOP == cmd ? searchIndex.Index - 2 : searchIndex.Index - 1; - break; - case FindKeysKeyNum: - case FindKeysUnknown: - default: - throw new GarnetException("FindKeys spec not known"); - } - - var searchIndex1 = (BeginSearchIndex)specs[1].BeginSearch; - switch (specs[1].FindKeys) - { - case FindKeysRange: - var findRange = (FindKeysRange)specs[1].FindKeys; - csvi.lastKey = findRange.LastKey < 0 ? findRange.LastKey + parseState.Count + 1 : findRange.LastKey + searchIndex1.Index - searchIndex.Index + 1; - csvi.step = findRange.KeyStep; - break; - case FindKeysKeyNum: - var findKeysKeyNum = (FindKeysKeyNum)specs[1].FindKeys; - csvi.keyNumOffset = searchIndex1.Index + findKeysKeyNum.KeyNumIdx - 1; - csvi.lastKey = searchIndex1.Index + parseState.GetInt(csvi.keyNumOffset); - csvi.step = findKeysKeyNum.KeyStep; - break; - case FindKeysUnknown: - default: - throw new GarnetException("FindKeys spec not known"); - } - - break; - default: - throw new GarnetException("KeySpecification not supported count"); - } - } - public ValueTask ClusterPublishAsync(RespCommand cmd, Span channel, Span message) => clusterManager.TryClusterPublishAsync(cmd, channel, message); - internal GarnetClusterCheckpointManager GetReplicationLogCheckpointManager(StoreType storeType) + internal GarnetClusterCheckpointManager ReplicationLogCheckpointManager { - Debug.Assert(serverOptions.EnableCluster); - return storeType switch + get { - StoreType.Main => (GarnetClusterCheckpointManager)storeWrapper.store.CheckpointManager, - StoreType.Object => (GarnetClusterCheckpointManager)storeWrapper.objectStore?.CheckpointManager, - _ => throw new Exception($"GetCkptManager: unexpected state {storeType}") - }; + Debug.Assert(serverOptions.EnableCluster); + return (GarnetClusterCheckpointManager)storeWrapper.store.CheckpointManager; + } } /// diff --git a/libs/cluster/Server/Failover/FailoverManager.cs b/libs/cluster/Server/Failover/FailoverManager.cs index d0d3c0fa951..f1b0fd2b7a0 100644 --- a/libs/cluster/Server/Failover/FailoverManager.cs +++ b/libs/cluster/Server/Failover/FailoverManager.cs @@ -89,7 +89,7 @@ public bool TryStartReplicaFailover(FailoverOption option, TimeSpan failoverTime logger: logger); _ = Task.Run(async () => { - var success = await currentFailoverSession.BeginAsyncReplicaFailoverAsync(); + var success = await currentFailoverSession.BeginAsyncReplicaFailoverAsync().ConfigureAwait(false); lastFailoverStatus = success ? FailoverStatus.FAILOVER_COMPLETED : FailoverStatus.FAILOVER_ABORTED; Reset(); }); @@ -121,7 +121,7 @@ public bool TryStartPrimaryFailover(string replicaAddress, int replicaPort, Fail logger: logger); _ = Task.Run(async () => { - _ = await currentFailoverSession.BeginAsyncPrimaryFailoverAsync(); + _ = await currentFailoverSession.BeginAsyncPrimaryFailoverAsync().ConfigureAwait(false); Reset(); }); return true; diff --git a/libs/cluster/Server/Failover/PrimaryFailoverSession.cs b/libs/cluster/Server/Failover/PrimaryFailoverSession.cs index adebdb0134e..f0ccf9198d0 100644 --- a/libs/cluster/Server/Failover/PrimaryFailoverSession.cs +++ b/libs/cluster/Server/Failover/PrimaryFailoverSession.cs @@ -5,20 +5,21 @@ using System.Threading.Tasks; using Garnet.client; using Garnet.common; +using Garnet.server; using Microsoft.Extensions.Logging; namespace Garnet.cluster { internal sealed partial class FailoverSession : IDisposable { - private async Task CheckReplicaSyncAsync(GarnetClient gclient) + private async Task CheckReplicaSyncAsync(GarnetClient gclient) { try { if (!gclient.IsConnected) await gclient.ConnectAsync().ConfigureAwait(false); - return await gclient.FailReplicationOffsetAsync(clusterProvider.replicationManager.ReplicationOffset).WaitAsync(clusterTimeout, cts.Token).ConfigureAwait(false); + return await gclient.ExecuteClusterFailReplicationOffsetAsync(clusterProvider.replicationManager.ReplicationOffset).WaitAsync(clusterTimeout, cts.Token).ConfigureAwait(false); } catch (Exception ex) { @@ -31,7 +32,7 @@ private async Task WaitForFirstReplicaSyncAsync() { if (clients.Length > 1) { - var tasks = new Task[clients.Length + 1]; + var tasks = new Task[clients.Length + 1]; var tcount = 0; foreach (var _gclient in clients) @@ -47,12 +48,11 @@ private async Task WaitForFirstReplicaSyncAsync() return null; } - var completedTaskRes = await completedTask.ConfigureAwait(false); - // Return client for replica that has caught up with replication primary for (var i = 0; i < tasks.Length; i++) { - if (completedTask == tasks[i] && completedTaskRes == clusterProvider.replicationManager.ReplicationOffset) + var replicationOffset = AofAddress.FromString(await tasks[i].ConfigureAwait(false)); + if (completedTask == tasks[i] && replicationOffset.EqualsAll(clusterProvider.replicationManager.ReplicationOffset)) return clients[i]; } return null; @@ -70,15 +70,14 @@ private async Task WaitForFirstReplicaSyncAsync() return null; } - var syncTaskResult = await syncTask.ConfigureAwait(false); - - if (syncTaskResult != clusterProvider.replicationManager.ReplicationOffset) + var replicationOffset = AofAddress.FromString(await syncTask.ConfigureAwait(false)); + if (!replicationOffset.EqualsAll(clusterProvider.replicationManager.ReplicationOffset)) return null; else return clients[0]; } - static async Task DelayToDefaultAsync(TimeSpan failoverTimeout) + static async Task DelayToDefaultAsync(TimeSpan failoverTimeout) { await Task.Delay(failoverTimeout).ConfigureAwait(false); diff --git a/libs/cluster/Server/Failover/ReplicaFailoverSession.cs b/libs/cluster/Server/Failover/ReplicaFailoverSession.cs index d65a9f56231..c89a0996073 100644 --- a/libs/cluster/Server/Failover/ReplicaFailoverSession.cs +++ b/libs/cluster/Server/Failover/ReplicaFailoverSession.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Garnet.client; using Garnet.common; +using Garnet.server; using Microsoft.Extensions.Logging; namespace Garnet.cluster @@ -42,7 +43,7 @@ private async Task CreateConnectionAsync(string nodeId) try { if (!client.IsConnected) - await client.ReconnectAsync().WaitAsync(failoverTimeout, cts.Token); + await client.ReconnectAsync().WaitAsync(failoverTimeout, cts.Token).ConfigureAwait(false); return client; } @@ -69,7 +70,7 @@ private Task GetConnectionAsync(string nodeId) private async Task PauseWritesAndWaitForSyncAsync() { var primaryId = oldConfig.LocalNodePrimaryId; - var client = await GetConnectionAsync(primaryId); + var client = await GetConnectionAsync(primaryId).ConfigureAwait(false); try { if (client == null) @@ -84,11 +85,13 @@ private async Task PauseWritesAndWaitForSyncAsync() // Issue stop writes to the primary status = FailoverStatus.ISSUING_PAUSE_WRITES; var localIdBytes = Encoding.ASCII.GetBytes(oldConfig.LocalNodeId); - var primaryReplicationOffset = await client.FailStopWritesAsync(localIdBytes).WaitAsync(failoverTimeout, cts.Token); + + var resp = await client.ExecuteClusterFailStopWritesAsync(localIdBytes).WaitAsync(failoverTimeout, cts.Token).ConfigureAwait(false); + var primaryReplicationOffset = AofAddress.FromString(resp); // Wait for replica to catch up status = FailoverStatus.WAITING_FOR_SYNC; - while (primaryReplicationOffset > clusterProvider.replicationManager.ReplicationOffset) + while (primaryReplicationOffset.AnyGreater(clusterProvider.replicationManager.ReplicationOffset)) { // Fail if upper bound time for failover has been reached if (FailoverTimeout) @@ -119,11 +122,10 @@ private async Task TakeOverAsPrimaryAsync() try { -#if DEBUG // Exception injection point for testing: simulates TakeOverAsPrimary failure // after PauseWritesAndWaitForSync has already sent failstopwrites to the primary. ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Failover_Fail_TakeOverAsPrimary); -#endif + // Make replica syncing unavailable by setting recovery flag if (!clusterProvider.replicationManager.BeginRecovery(RecoveryStatus.ClusterFailover, upgradeLock: false)) { @@ -143,21 +145,39 @@ private async Task TakeOverAsPrimaryAsync() // Update replicationIds and replicationOffset2 clusterProvider.replicationManager.TryUpdateForFailover(); - // Reset replay iterators - clusterProvider.replicationManager.ResetReplayIterator(); + // Cancel active replication tasks + clusterProvider.replicationManager.ResetReplicaReplayDriverStore(); + + // Update sequence number generator for sharded log if needed + if (clusterProvider.serverOptions.AofPhysicalSublogCount > 1) + { + clusterProvider.storeWrapper.appendOnlyFile.ResetSequenceNumberGenerator(); + await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false); + } // Initialize checkpoint history if (!clusterProvider.replicationManager.InitializeCheckpointStore()) logger?.LogWarning("Failed acquiring latest memory checkpoint metadata at {method}", nameof(TakeOverAsPrimaryAsync)); + _ = clusterProvider.BumpAndWaitForEpochTransitionAsync().ConfigureAwait(false); + // Stop advance time task when reconfiguring node to be replica + if (clusterProvider.storeWrapper.serverOptions.AofPhysicalSublogCount > 1) + await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false); + // Resume all background maintenance that were possibly shutdown when this node became a replica clusterProvider.storeWrapper.StartPrimaryTasks(); } + catch (Exception ex) + { + logger?.LogError(ex, "{method}", nameof(TakeOverAsPrimaryAsync)); + throw; + } finally { // Disable recovering as now this node has become a primary or failed in its attempt earlier - if (acquiredLock) clusterProvider.replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); + if (acquiredLock) + clusterProvider.replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); } return true; @@ -176,7 +196,7 @@ private async Task BroadcastConfigAndRequestAttachAsync(string replicaId, byte[] var oldPrimaryId = oldConfig.LocalNodePrimaryId; var newConfig = clusterProvider.clusterManager.CurrentConfig; - var client = oldPrimaryId.Equals(replicaId) ? primaryClient : await GetConnectionAsync(replicaId); + var client = oldPrimaryId.Equals(replicaId) ? primaryClient : await GetConnectionAsync(replicaId).ConfigureAwait(false); try { @@ -196,15 +216,23 @@ private async Task BroadcastConfigAndRequestAttachAsync(string replicaId, byte[] { clusterProvider.clusterManager.gossipStats.UpdateGossipBytesRecv(resp.Length); var returnedConfigArray = resp.Span.ToArray(); - var other = ClusterConfig.FromByteArray(returnedConfigArray); - // Check if gossip is from a node that is known and trusted before merging - if (current.IsKnown(other.LocalNodeId)) - _ = clusterProvider.clusterManager.TryMerge(ClusterConfig.FromByteArray(returnedConfigArray)); + // Validate config version before full deserialization + if (!ClusterConfig.TryPeekVersion(returnedConfigArray, out var version) || version != ClusterConfig.ClusterConfigVersion) + { + logger?.LogWarning("Received failover gossip response with incompatible config version: {version}", version); + } else - logger?.LogWarning("Received gossip from unknown node: {node-id}", other.LocalNodeId); + { + var other = ClusterConfig.FromByteArray(returnedConfigArray); + + // Check if gossip is from a node that is known and trusted before merging + if (current.IsKnown(other.LocalNodeId)) + _ = clusterProvider.clusterManager.TryMerge(other); + else + logger?.LogWarning("Received gossip from unknown node: {node-id}", other.LocalNodeId); + } } - resp.Dispose(); } catch (Exception ex) { @@ -299,7 +327,7 @@ public async Task BeginAsyncReplicaFailoverAsync() try { // Issue stop writes and on ack wait for replica to catch up - if (option is FailoverOption.DEFAULT && !await PauseWritesAndWaitForSyncAsync()) + if (option is FailoverOption.DEFAULT && !await PauseWritesAndWaitForSyncAsync().ConfigureAwait(false)) { return false; } @@ -318,7 +346,7 @@ public async Task BeginAsyncReplicaFailoverAsync() failoverSucceeded = true; // Attach to old replicas, and old primary if DEFAULT option - await IssueAttachReplicasAsync(); + await IssueAttachReplicasAsync().ConfigureAwait(false); await clusterProvider.storeWrapper.SuspendReplicaOnlyTasksAsync(); clusterProvider.storeWrapper.StartPrimaryTasks(); @@ -342,7 +370,10 @@ public async Task BeginAsyncReplicaFailoverAsync() try { logger?.LogWarning("Attempting to reset primary after failed failover"); - _ = await primaryClient?.FailStopWritesAsync(Array.Empty()).WaitAsync(failoverTimeout, cts.Token); + if (primaryClient != null) + { + _ = await primaryClient.ExecuteClusterFailStopWritesAsync(Array.Empty()).WaitAsync(failoverTimeout, cts.Token).ConfigureAwait(false); + } } catch (Exception ex) { diff --git a/libs/cluster/Server/Gossip/GarnetClientExtensions.cs b/libs/cluster/Server/Gossip/GarnetClientExtensions.cs index 4989671b0ef..9fc31c92e84 100644 --- a/libs/cluster/Server/Gossip/GarnetClientExtensions.cs +++ b/libs/cluster/Server/Gossip/GarnetClientExtensions.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. using System; -using System.Text; using System.Threading; using System.Threading.Tasks; using Garnet.client; @@ -11,14 +10,11 @@ namespace Garnet.cluster { - internal static partial class GarnetClientExtensions + internal static class GarnetClientExtensions { static readonly Memory GOSSIP = "GOSSIP"u8.ToArray(); static readonly Memory WITHMEET = "WITHMEET"u8.ToArray(); - static Memory PUBLISH => "PUBLISH"u8.ToArray(); - static Memory SPUBLISH => "SPUBLISH"u8.ToArray(); - /// /// Send config /// @@ -40,32 +36,43 @@ public static Task> GossipWithMeetAsync(this GarnetClient cli => client.ExecuteForMemoryResultWithCancellationAsync(GarnetClient.CLUSTER, [GOSSIP, WITHMEET, data], cancellationToken); /// - /// Send stop writes to primary + /// Issue stop writes to primary node /// /// /// /// /// - public static async Task FailStopWritesAsync(this GarnetClient client, Memory nodeid, CancellationToken cancellationToken = default) - => await client.ExecuteForLongResultWithCancellationAsync(GarnetClient.CLUSTER, [CmdStrings.failstopwrites.ToArray(), nodeid], cancellationToken).ConfigureAwait(false); + /// + public static async Task ExecuteClusterFailStopWritesAsync(this GarnetClient client, Memory nodeid, CancellationToken cancellationToken = default) + => await client.ExecuteForStringResultWithCancellationAsync(GarnetClient.CLUSTER, [CmdStrings.failstopwrites.ToArray(), nodeid], cancellationToken).ConfigureAwait(false); /// - /// Send request to await for replication offset sync with replica + /// Acquire replication offset of primary. Used to delay failover until the calling replica catches up. /// /// /// /// /// - public static async Task FailReplicationOffsetAsync(this GarnetClient client, long primaryReplicationOffset, CancellationToken cancellationToken = default) + /// + public static async Task ExecuteClusterFailReplicationOffsetAsync(this GarnetClient client, AofAddress primaryReplicationOffset, CancellationToken cancellationToken = default) { var args = new Memory[] { CmdStrings.failreplicationoffset.ToArray(), - Encoding.ASCII.GetBytes(primaryReplicationOffset.ToString()) + primaryReplicationOffset.ToByteArray() }; - return await client.ExecuteForLongResultWithCancellationAsync(GarnetClient.CLUSTER, args, cancellationToken).ConfigureAwait(false); + return await client.ExecuteForStringResultWithCancellationAsync(GarnetClient.CLUSTER, args, cancellationToken).ConfigureAwait(false); } - public static void ClusterPublishNoResponse(this GarnetClient client, RespCommand cmd, Span channel, Span message, CancellationToken cancellationToken = default) + /// + /// Publishes a message to a specified channel in a clustered Garnet environment without waiting for a server + /// response. + /// + /// The Garnet client instance used to send the publish command. + /// The RESP command to execute. Must be either PUBLISH or SPUBLISH. + /// A span containing the channel name to which the message will be published. + /// A span containing the message to publish to the channel. + /// A cancellation token that can be used to cancel the operation. + public static void ExecuteClusterPublishNoResponse(this GarnetClient client, RespCommand cmd, Span channel, Span message, CancellationToken cancellationToken = default) => client.ExecuteNoResponse(GarnetClient.CLUSTER, RespCommand.PUBLISH == cmd ? GarnetClient.PUBLISH : GarnetClient.SPUBLISH, channel, message, cancellationToken); } } \ No newline at end of file diff --git a/libs/cluster/Server/Gossip/GarnetServerNode.cs b/libs/cluster/Server/Gossip/GarnetServerNode.cs index ecefadeb607..fc4984b9ad6 100644 --- a/libs/cluster/Server/Gossip/GarnetServerNode.cs +++ b/libs/cluster/Server/Gossip/GarnetServerNode.cs @@ -79,13 +79,15 @@ public GarnetServerNode(ClusterProvider clusterProvider, EndPoint endpoint, SslC this.clusterProvider = clusterProvider; this.EndPoint = endpoint; this.gc = new GarnetClient( - endpoint, tlsOptions, + endpoint, + tlsOptions, sendPageSize: opts.DisablePubSub ? defaultSendPageSize : Math.Max(defaultSendPageSize, (int)opts.PubSubPageSizeBytes()), maxOutstandingTasks: defaultMaxOutstandingTask, timeoutMilliseconds: opts.ClusterTimeout <= 0 ? 0 : TimeSpan.FromSeconds(opts.ClusterTimeout).Milliseconds, authUsername: clusterProvider.clusterManager.clusterProvider.ClusterUsername, authPassword: clusterProvider.clusterManager.clusterProvider.ClusterPassword, epoch: epoch, + clientName: $"Gossip-{clusterProvider.clusterManager.CurrentConfig.LocalNodeEndpoint}", logger: logger); this.initialized = 0; this.logger = logger; @@ -160,7 +162,10 @@ byte[] GetMostRecentConfig() if (conf != lastConfig) { lastConfig = conf; - if (clusterProvider.replicationManager != null) lastConfig.LazyUpdateLocalReplicationOffset(clusterProvider.replicationManager.ReplicationOffset); + if (clusterProvider.replicationManager != null) + // NOTE: We update replication offset for sublog-0 because this info is used in CLUSTER NODES + // and we cannot have multiple replication offsets without changing the expected CLUSTER NODES response + lastConfig.LazyUpdateLocalReplicationOffset(clusterProvider.replicationManager.GetReplicationOffset(0)); byteArray = lastConfig.ToByteArray(); } else @@ -179,16 +184,24 @@ private async Task GossipAsync(byte[] configByteArray) { try { - using var resp = await gc.GossipAsync(configByteArray).WaitAsync(clusterProvider.clusterManager.gossipDelay, cts.Token).ConfigureAwait(false); + using var resp = await gc.GossipAsync(configByteArray, internalCts.Token).WaitAsync(clusterProvider.clusterManager.gossipDelay, cts.Token).ConfigureAwait(false); if (resp.Length > 0) { clusterProvider.clusterManager.gossipStats.UpdateGossipBytesRecv(resp.Length); var returnedConfigArray = resp.Span.ToArray(); + + // Validate config version before full deserialization + if (!ClusterConfig.TryPeekVersion(returnedConfigArray, out var version) || version != ClusterConfig.ClusterConfigVersion) + { + logger?.LogWarning("Received gossip response with incompatible config version: {version}", version); + return; + } + var other = ClusterConfig.FromByteArray(returnedConfigArray); var current = clusterProvider.clusterManager.CurrentConfig; // Check if gossip is from a node that is known and trusted before merging if (current.IsKnown(other.LocalNodeId)) - clusterProvider.clusterManager.TryMerge(ClusterConfig.FromByteArray(returnedConfigArray)); + clusterProvider.clusterManager.TryMerge(other); else logger?.LogWarning("Received gossip from unknown node: {node-id}", other.LocalNodeId); } @@ -204,11 +217,10 @@ private async Task GossipAsync(byte[] configByteArray) /// /// /// - public async Task> TryMeetAsync(byte[] configByteArray) + public Task> TryMeetAsync(byte[] configByteArray) { UpdateGossipSend(); - var resp = await gc.GossipWithMeetAsync(configByteArray).WaitAsync(clusterProvider.clusterManager.clusterTimeout, cts.Token); - return resp; + return gc.GossipWithMeetAsync(configByteArray, internalCts.Token).WaitAsync(clusterProvider.clusterManager.clusterTimeout, cts.Token); } /// @@ -292,7 +304,7 @@ public void TryClusterPublish(RespCommand cmd, Span channel, Span me } locked = true; - gc.ClusterPublishNoResponse(cmd, channel, message); + gc.ExecuteClusterPublishNoResponse(cmd, channel, message); } finally { diff --git a/libs/cluster/Server/Gossip/Gossip.cs b/libs/cluster/Server/Gossip/Gossip.cs index ea112bcbf1e..c861c5bbdd8 100644 --- a/libs/cluster/Server/Gossip/Gossip.cs +++ b/libs/cluster/Server/Gossip/Gossip.cs @@ -180,27 +180,39 @@ public async Task TryMeetAsync(string address, int port, bool acquireLock = true // Initialize GarnetServerNode // Thread-Safe initialization executes only once - await gsn.InitializeAsync(); + await gsn.InitializeAsync().ConfigureAwait(false); // Send full config in Gossip - resp = await gsn.TryMeetAsync(conf.ToByteArray()); + resp = await gsn.TryMeetAsync(conf.ToByteArray()).ConfigureAwait(false); if (resp.Length > 0) { - var other = ClusterConfig.FromByteArray(resp.Span.ToArray()); - nodeId = other.LocalNodeId; - gsn.NodeId = nodeId; + var respArray = resp.Span.ToArray(); - logger?.LogInformation("MEET {nodeId} {address} {port}", nodeId, address, port); - // Merge without a check because node is trusted as meet was issued by admin - _ = TryMerge(other, acquireLock); + // Validate config version before full deserialization + if (!ClusterConfig.TryPeekVersion(respArray, out var version) || version != ClusterConfig.ClusterConfigVersion) + { + logger?.LogWarning("MEET response has incompatible config version: {version}", version); + if (created) gsn?.Dispose(); + gossipStats.UpdateMeetRequestsFailed(); + } + else + { + var other = ClusterConfig.FromByteArray(respArray); + nodeId = other.LocalNodeId; + gsn.NodeId = nodeId; - gossipStats.UpdateMeetRequestsSucceed(); + logger?.LogInformation("MEET {nodeId} {address} {port}", nodeId, address, port); + // Merge without a check because node is trusted as meet was issued by admin + _ = TryMerge(other, acquireLock); - // If failed to add newly created connection dispose of it to reclaim resources - // Dispose only connections that this meet task has created to avoid conflicts with existing connections from gossip main thread - // After connection is added we are no longer the owner. Background gossip task will be owner - if (created && !await clusterConnectionStore.AddConnectionAsync(gsn)) - gsn.Dispose(); + gossipStats.UpdateMeetRequestsSucceed(); + + // If failed to add newly created connection dispose of it to reclaim resources + // Dispose only connections that this meet task has created to avoid conflicts with existing connections from gossip main thread + // After connection is added we are no longer the owner. Background gossip task will be owner + if (created && !await clusterConnectionStore.AddConnectionAsync(gsn).ConfigureAwait(false)) + gsn.Dispose(); + } } } catch (Exception ex) @@ -236,99 +248,81 @@ public ValueTask TryClusterPublishAsync(RespCommand cmd, Span channel, Spa for (var entryIx = 0; entryIx < nodeEntries.Count; entryIx++) { - try - { - var (nodeId, endpoint) = nodeEntries[entryIx]; + var (nodeId, endpoint) = nodeEntries[entryIx]; - var getOrAddTask = clusterConnectionStore.GetOrAddAsync(clusterProvider, endpoint, tlsOptions, nodeId, logger: logger); + var getOrAddTask = clusterConnectionStore.GetOrAddAsync(clusterProvider, endpoint, tlsOptions, nodeId, logger: logger); - GarnetServerNode gsn; - if (getOrAddTask.IsCompletedSuccessfully) - { - // Cannot avoid blocking, but it's gated by IsCompletedSuccessfully so safe - (_, gsn) = AsyncUtils.BlockingWait(getOrAddTask); - } - else - { - // Otherwise copy channel & message and go async - return new(GoAsyncHelperAsync(getOrAddTask, default, entryIx, null, cmd, channel.ToArray(), message.ToArray())); - } + GarnetServerNode gsn; + if (getOrAddTask.IsCompletedSuccessfully) + { + // Cannot remove .GetResult here, but it's gated by IsCompletedSuccessfully so safe + (_, gsn) = AsyncUtils.BlockingWait(getOrAddTask); + } + else + { + // Otherwise copy channel & message and go async + return new(GoAsyncHelperAsync(getOrAddTask, default, entryIx, null, cmd, channel.ToArray(), message.ToArray())); + } - if (gsn == null) - continue; + if (gsn == null) + continue; - // Initialize GarnetServerNode - // Thread-Safe initialization executes only once - var initTask = gsn.InitializeAsync(); - if (initTask.IsCompletedSuccessfully) - { - // Can stay sync, so proceed - gsn.TryClusterPublish(cmd, channel, message); - } - else - { - // Copy channel & message and go async - return new(GoAsyncHelperAsync(default, initTask, entryIx, gsn, cmd, channel.ToArray(), message.ToArray())); - } + // Initialize GarnetServerNode + // Thread-Safe initialization executes only once + var initTask = gsn.InitializeAsync(); + if (initTask.IsCompletedSuccessfully) + { + // Can stay sync, so proceed + gsn.TryClusterPublish(cmd, channel, message); } - catch (Exception ex) + else { - logger?.LogWarning(ex, $"{nameof(ClusterManager)}.{nameof(TryClusterPublishAsync)}"); + // Copy channel & message and go async + return new(GoAsyncHelperAsync(default, initTask, entryIx, gsn, cmd, channel.ToArray(), message.ToArray())); } } - // Completed synchrously + // Completed synchronously return default; async Task GoAsyncHelperAsync(ValueTask<(bool Success, GarnetServerNode Node)> getOrAddTask, ValueTask initTask, int lastEntryIx, GarnetServerNode lastGsn, RespCommand cmd, Memory channel, Memory message) { // Finish the task which caused us to go async - try + if (lastGsn == null) { - if (lastGsn == null) - { - (_, lastGsn) = await getOrAddTask.ConfigureAwait(false); + (_, lastGsn) = await getOrAddTask.ConfigureAwait(false); - if (lastGsn != null) - { - await lastGsn.InitializeAsync().ConfigureAwait(false); - } - } - else + if (lastGsn != null) { - await initTask.ConfigureAwait(false); + await lastGsn.InitializeAsync().ConfigureAwait(false); } - - lastGsn?.TryClusterPublish(cmd, channel.Span, message.Span); } - catch (Exception ex) + else + { + await initTask.ConfigureAwait(false); + } + + if (lastGsn != null) { - logger?.LogWarning(ex, $"{nameof(ClusterManager)}.{nameof(TryClusterPublishAsync)} -> {nameof(GoAsyncHelperAsync)} initial completion"); + lastGsn.TryClusterPublish(cmd, channel.Span, message.Span); } // Process remainder of entries, staying async for (var entryIx = lastEntryIx + 1; entryIx < nodeEntries.Count; entryIx++) { - try - { - var (nodeId, endpoint) = nodeEntries[entryIx]; + var (nodeId, endpoint) = nodeEntries[entryIx]; - var (_, gsn) = await clusterConnectionStore.GetOrAddAsync(clusterProvider, endpoint, tlsOptions, nodeId, logger: logger).ConfigureAwait(false); + var (_, gsn) = await clusterConnectionStore.GetOrAddAsync(clusterProvider, endpoint, tlsOptions, nodeId, logger: logger).ConfigureAwait(false); - if (gsn == null) - continue; + if (gsn == null) + continue; - // Initialize GarnetServerNode - // Thread-Safe initialization executes only once - await gsn.InitializeAsync().ConfigureAwait(false); + // Initialize GarnetServerNode + // Thread-Safe initialization executes only once + await gsn.InitializeAsync().ConfigureAwait(false); - // Publish to remote nodes - gsn.TryClusterPublish(cmd, channel.Span, message.Span); - } - catch (Exception ex) - { - logger?.LogWarning(ex, $"{nameof(ClusterManager)}.{nameof(TryClusterPublishAsync)} -> {nameof(GoAsyncHelperAsync)} loop"); - } + // Publish to remote nodes + gsn.TryClusterPublish(cmd, channel.Span, message.Span); } } } @@ -343,7 +337,7 @@ async Task GossipMainAsync() { while (true) { - if (ctsGossip.Token.IsCancellationRequested) return; + ctsGossip.Token.ThrowIfCancellationRequested(); await InitConnectionsAsync().ConfigureAwait(false); // Choose between full broadcast or sample gossip to few nodes @@ -386,7 +380,7 @@ async Task InitConnectionsAsync() foreach (var a in addresses) { - if (ctsGossip.Token.IsCancellationRequested) break; + ctsGossip.Token.ThrowIfCancellationRequested(); var nodeId = a.Item1; var address = a.Item2; var port = a.Item3; @@ -401,7 +395,9 @@ async Task InitConnectionsAsync() if (gsn == null) { logger?.LogWarning("InitConnections: Could not establish connection to remote node [{nodeId} {address}:{port}] failed", nodeId, address, port); + _ = await clusterConnectionStore.TryRemoveConnectionAsync(nodeId).ConfigureAwait(false); + continue; } @@ -419,7 +415,7 @@ async Task DisposeBannedWorkerConnectionsAsync() { foreach (var w in workerBanList) { - if (ctsGossip.Token.IsCancellationRequested) return; + ctsGossip.Token.ThrowIfCancellationRequested(); var nodeId = w.Key; var expiry = w.Value; @@ -446,7 +442,7 @@ async Task BroadcastGossipSendAsync() { try { - if (ctsGossip.Token.IsCancellationRequested) return; + ctsGossip.Token.ThrowIfCancellationRequested(); // Issue gossip message to node and truck success metrics if (currNode.TryGossip()) @@ -496,7 +492,7 @@ async Task GossipSampleSendAsync() try { - if (ctsGossip.Token.IsCancellationRequested) return; + ctsGossip.Token.ThrowIfCancellationRequested(); // Issue gossip message to node and truck success metrics if (currNode.TryGossip()) diff --git a/libs/cluster/Server/Migration/MigrateOperation.cs b/libs/cluster/Server/Migration/MigrateOperation.cs index 794c98b2960..48aa19da260 100644 --- a/libs/cluster/Server/Migration/MigrateOperation.cs +++ b/libs/cluster/Server/Migration/MigrateOperation.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using Garnet.client; using Garnet.server; @@ -18,8 +19,7 @@ internal sealed partial class MigrateOperation { public readonly Sketch sketch; public readonly List keysToDelete; - public MainStoreScan mss; - public ObjectStoreScan oss; + public StoreScan storeScan; private readonly ConcurrentDictionary vectorSetsIndexKeysToMigrate; @@ -35,8 +35,14 @@ internal sealed partial class MigrateOperation public bool Contains(int slot) => session._sslots.Contains(slot); - public bool ContainsNamespace(ulong ns) => session._namespaces?.Contains(ns) ?? false; + public bool ContainsNamespace(ReadOnlySpan namespaceBytes) + { + Debug.Assert(namespaceBytes.Length == 1, "Longer namespaces note supported"); + + var ns = (ulong)namespaceBytes[0]; + return session._namespaces?.Contains(ns) ?? false; + } public void EncounteredVectorSet(byte[] key, byte[] value) => vectorSetsIndexKeysToMigrate.TryAdd(key, value); @@ -46,8 +52,7 @@ public MigrateOperation(MigrateSession session, Sketch sketch = null, int batchS gcs = session.GetGarnetClient(); localServerSession = session.GetLocalSession(); this.sketch = sketch ?? new(keyCount: batchSize << 2); - mss = new MainStoreScan(this); - oss = new ObjectStoreScan(this); + storeScan = new StoreScan(this); keysToDelete = []; vectorSetsIndexKeysToMigrate = new(ByteArrayComparer.Instance); } @@ -56,7 +61,6 @@ public async ValueTask InitializeAsync() { if (!await session.CheckConnectionAsync(gcs).ConfigureAwait(false)) return false; - gcs.InitializeIterationBuffer(session.clusterProvider.storeWrapper.loggingFrequency); return true; } @@ -70,56 +74,42 @@ public void Dispose() /// /// Perform scan to gather keys and build sketch /// - /// /// /// - public void Scan(StoreType storeType, ref long currentAddress, long endAddress) - { - if (storeType == StoreType.Main) - _ = localServerSession.BasicGarnetApi.IterateMainStore(ref mss, ref currentAddress, endAddress, endAddress, includeTombstones: true); - else if (storeType == StoreType.Object) - _ = localServerSession.BasicGarnetApi.IterateObjectStore(ref oss, ref currentAddress, endAddress, endAddress, includeTombstones: true); - } + public void Scan(ref long currentAddress, long endAddress) + => localServerSession.BasicGarnetApi.IterateStore(ref storeScan, ref currentAddress, endAddress, endAddress, + includeTombstones: true); /// /// Transmit gathered keys /// - /// /// - public async Task TransmitSlotsAsync(StoreType storeType) + public async Task TransmitSlotsAsync() { - var bufferSize = 1 << 10; - SectorAlignedMemory buffer = new(bufferSize, 1); - IntPtr bufPtr, bufPtrEnd; - SpanByteAndMemory o; - unsafe - { - bufPtr = (IntPtr)buffer.GetValidPointer(); - bufPtrEnd = bufPtr + bufferSize; - o = new SpanByteAndMemory((byte*)bufPtr, (int)(bufPtrEnd - bufPtr)); - } - var input = new RawStringInput(RespCommandAccessor.MIGRATE); + var output = new UnifiedOutput(); // TODO: initialize this based on gcs curr and end; make sure it has the initial part of the "send" set + var vectorOutput = new VectorOutput(); // TODO: initialize this based on gcs curr and end; make sure it has the initial part of the "send" set try { - if (storeType == StoreType.Main) + var input = new UnifiedInput(RespCommand.MIGRATE); + input.arg1 = session.NetworkBufferSettings.sendBufferSize - common.NetworkBufferSettings.SendBufferOverheadReserve; + + VectorInput vectorInput = new(); + vectorInput.AlignmentExpected = true; // We're moving DiskANN sourced data, so alignment is expected + vectorInput.MaxMigrationHeapAllocationSize = session.NetworkBufferSettings.sendBufferSize - common.NetworkBufferSettings.SendBufferOverheadReserve; + + foreach (var (ns, key, hasNs) in sketch.argSliceVector) { - foreach (var key in sketch.argSliceVector) + if (hasNs) { - var spanByte = key; - if (!await session.WriteOrSendMainStoreKeyValuePairAsync(gcs, localServerSession, ref spanByte, ref input, ref o, out _).ConfigureAwait(false)) + // Migrating Vector Set element data + if (!await session.WriteOrSendRecordAsync(gcs, localServerSession, ns, key, ref vectorInput, ref vectorOutput, out _).ConfigureAwait(false)) return false; - - // Reset SpanByte for next read if any but don't dispose heap buffer as we might re-use it - o.SpanByte = new SpanByte((int)(bufPtrEnd - bufPtr), (IntPtr)bufPtr); } - } - else - { - foreach (var key in sketch.argSliceVector) + else { - var argSlice = key; - if (!await session.WriteOrSendObjectStoreKeyValuePairAsync(gcs, localServerSession, ref argSlice, out _).ConfigureAwait(false)) + // Migrating everything else + if (!await session.WriteOrSendRecordAsync(gcs, localServerSession, key, ref input, ref output, out _).ConfigureAwait(false)) return false; } } @@ -130,90 +120,60 @@ public async Task TransmitSlotsAsync(StoreType storeType) } finally { - buffer.Dispose(); + output.SpanByteAndMemory.Dispose(); + vectorOutput.SpanByteAndMemory.Dispose(); } return true; } - /// - /// Move keys in sketch out of the given store, UNLESS they are also in . - /// - public async Task TransmitKeysAsync(StoreType storeType, Dictionary vectorSetKeysToIgnore) + public async Task TransmitKeysAsync(Dictionary vectorSetKeysToIgnore) { - var bufferSize = 1 << 10; - SectorAlignedMemory buffer = new(bufferSize, 1); - IntPtr bufPtr, bufPtrEnd; - SpanByteAndMemory o; - unsafe - { - bufPtr = (IntPtr)buffer.GetValidPointer(); - bufPtrEnd = bufPtr + bufferSize; - o = new SpanByteAndMemory((byte*)bufPtr, (int)(bufPtrEnd - bufPtr)); - } - var input = new RawStringInput(RespCommandAccessor.MIGRATE); + // Use this for both stores; main store will just use the SpanByteAndMemory directly. We want it to be outside iterations + // so we can reuse the SpanByteAndMemory.Memory across iterations. + // TODO: initialize 'output' based on gcs curr and end; make sure it has the initial part of the "send" set, and call gcs.IncrementRecordDirect(). + // This will still allow SBAM.Memory to be reused. + var output = new UnifiedOutput(); + +#if NET9_0_OR_GREATER + var ignoreLookup = vectorSetKeysToIgnore.GetAlternateLookup>(); +#endif try { var keys = sketch.Keys; - if (storeType == StoreType.Main) - { -#if NET9_0_OR_GREATER - var ignoreLookup = vectorSetKeysToIgnore.GetAlternateLookup>(); -#endif - for (var i = 0; i < keys.Count; i++) - { - if (keys[i].Item2) - continue; + var input = new UnifiedInput(RespCommand.MIGRATE) + { + arg1 = session.NetworkBufferSettings.sendBufferSize - 1024 // Reserve some space for overhead + }; + for (var i = 0; i < keys.Count; i++) + { + if (keys[i].Item2) + continue; - var spanByte = keys[i].Item1.SpanByte; + var spanByte = keys[i].Item1; - // Don't transmit if a Vector Set - var isVectorSet = - vectorSetKeysToIgnore.Count > 0 && + // Don't transmit if a Vector Set + var isVectorSet = + vectorSetKeysToIgnore.Count > 0 && #if NET9_0_OR_GREATER - ignoreLookup.ContainsKey(spanByte.AsReadOnlySpan()); + ignoreLookup.ContainsKey(spanByte.ReadOnlySpan); #else - vectorSetKeysToIgnore.ContainsKey(spanByte.ToByteArray()); + vectorSetKeysToIgnore.ContainsKey(spanByte.ToArray()); #endif - if (isVectorSet) - { - continue; - } - if (!await session.WriteOrSendMainStoreKeyValuePairAsync(gcs, localServerSession, ref spanByte, ref input, ref o, out var status).ConfigureAwait(false)) - return false; - - // Skip if key NOTFOUND - if (status == GarnetStatus.NOTFOUND) - continue; - - // Reset SpanByte for next read if any but don't dispose heap buffer as we might re-use it - o.SpanByte = new SpanByte((int)(bufPtrEnd - bufPtr), (IntPtr)bufPtr); - - // Mark for deletion - keys[i] = (keys[i].Item1, true); - } - } - else - { - for (var i = 0; i < keys.Count; i++) + if (isVectorSet) { - if (keys[i].Item2) - continue; - - var spanByte = keys[i].Item1.SpanByte; - if (!await session.WriteOrSendObjectStoreKeyValuePairAsync(gcs, localServerSession, ref spanByte, out var status).ConfigureAwait(false)) - return false; + continue; + } - // Skip if key NOTFOUND - if (status == GarnetStatus.NOTFOUND) - continue; + if (!await session.WriteOrSendRecordAsync(gcs, localServerSession, keys[i].Item1, ref input, ref output, out var status).ConfigureAwait(false)) + return false; - // Mark for deletion + // If key was FOUND, mark it for deletion + if (status != GarnetStatus.NOTFOUND) keys[i] = (keys[i].Item1, true); - } } // Flush final data in client buffer @@ -222,7 +182,7 @@ public async Task TransmitKeysAsync(StoreType storeType, Dictionary TransmitKeysNamespacesAsync(ILogger logger) var current = cursor; // Build Sketch migrateOperation.sketch.SetStatus(SketchStatus.INITIALIZING); - migrateOperation.Scan(StoreType.Main, ref current, workerEndAddress); + migrateOperation.Scan(ref current, workerEndAddress); // Stop if no keys have been found if (migrateOperation.sketch.argSliceVector.IsEmpty) break; @@ -261,7 +221,7 @@ public async ValueTask TransmitKeysNamespacesAsync(ILogger logger) await migrateOperation.session.WaitForConfigPropagationAsync().ConfigureAwait(false); // Transmit all keys gathered - if (!await migrateOperation.TransmitSlotsAsync(StoreType.Main).ConfigureAwait(false)) + if (!await migrateOperation.TransmitSlotsAsync().ConfigureAwait(false)) { logger?.LogWarning("TransmitSlots failed for {cursor} to {current} (with {count} keys)", cursor, current, migrateOperation.sketch.argSliceVector.Count); return false; @@ -288,16 +248,17 @@ public void DeleteKeys() return; if (session.transferOption == TransferOption.SLOTS) { - foreach (var key in sketch.argSliceVector) + foreach (var (ns, key, hasNs) in sketch.argSliceVector) { - if (key.MetadataSize == 1) + if (hasNs) { - // Namespace'd keys are not deleted here, but when migration finishes + // Namespace'd keys are deleted as part after migration completes continue; } - - var spanByte = key; - _ = localServerSession.BasicGarnetApi.DELETE(ref spanByte); + else + { + _ = localServerSession.BasicGarnetApi.DELETE(key); + } } } else @@ -305,10 +266,9 @@ public void DeleteKeys() var keys = sketch.Keys; for (var i = 0; i < keys.Count; i++) { - // Skip if key is not marked for deletion because it has not been transmitted to the target node - if (!keys[i].Item2) continue; - var spanByte = keys[i].Item1.SpanByte; - _ = localServerSession.BasicGarnetApi.DELETE(ref spanByte); + // Do not delete the key if it is not marked for deletion because it has not been transmitted to the target node + if (keys[i].Item2) + _ = localServerSession.BasicGarnetApi.DELETE(keys[i].Item1); } } } @@ -316,14 +276,14 @@ public void DeleteKeys() /// /// Delete a Vector Set after migration if _copyOption is not set. /// - public void DeleteVectorSet(ref SpanByte key) + public void DeleteVectorSet(PinnedSpanByte key) { if (session._copyOption) return; - var delRes = localServerSession.BasicGarnetApi.DELETE(ref key); + var delRes = localServerSession.BasicGarnetApi.DELETE(key); - session.logger?.LogDebug("Deleting Vector Set {key} after migration: {delRes}", System.Text.Encoding.UTF8.GetString(key.AsReadOnlySpan()), delRes); + session.logger?.LogDebug("Deleting Vector Set {key} after migration: {delRes}", System.Text.Encoding.UTF8.GetString(key), delRes); } } } diff --git a/libs/cluster/Server/Migration/MigrateScanFunctions.cs b/libs/cluster/Server/Migration/MigrateScanFunctions.cs index 2536478cd34..c8c43da7e0a 100644 --- a/libs/cluster/Server/Migration/MigrateScanFunctions.cs +++ b/libs/cluster/Server/Migration/MigrateScanFunctions.cs @@ -10,14 +10,13 @@ namespace Garnet.cluster { internal sealed unsafe partial class MigrateSession { - #region mainStoreScan - internal sealed unsafe class MainStoreScan : IScanIteratorFunctions + internal sealed unsafe class StoreScan : IScanIteratorFunctions { - readonly MigrateOperation mss; + readonly MigrateOperation migrateOperation; - internal MainStoreScan(MigrateOperation mss) + internal StoreScan(MigrateOperation migrateOperation) { - this.mss = mss; + this.migrateOperation = migrateOperation; } public bool OnStart(long beginAddress, long endAddress) => true; @@ -26,41 +25,39 @@ public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } - public unsafe bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; // default; not used here - mss.ThrowIfCancelled(); + migrateOperation.ThrowIfCancelled(); // Do not send key if it is expired - // NOTE: Because the scan executes includingTombstones, tombstone records may not have valid expiration metadata; skip expiration validation here and defer it until the actual send occurs (MigrateSessionCommonUtils.cs:WriteOrSendMainStoreKeyValuePair). - if (!recordMetadata.RecordInfo.Tombstone && ClusterSession.Expired(ref value)) + if (!srcLogRecord.Info.Tombstone && ClusterSession.Expired(in srcLogRecord)) return true; - // TODO: Some other way to detect namespaces - if (key.MetadataSize == 1) + if (srcLogRecord.HasNamespace) { - var ns = key.GetNamespaceInPayload(); - - if (mss.ContainsNamespace(ns) && !mss.sketch.TryHashAndStore(ns, key.AsSpan())) + // Migrating a Vector Set element + if (migrateOperation.ContainsNamespace(srcLogRecord.NamespaceBytes) && !migrateOperation.sketch.TryHashAndStore(srcLogRecord.NamespaceBytes, srcLogRecord.KeyBytes)) return false; } else { - var s = HashSlotUtils.HashSlot(ref key); + var key = srcLogRecord.Key; + var slot = HashSlotUtils.HashSlot(key); - // Check if key belongs to slot that is being migrated... - if (mss.Contains(s)) + // Check if key belongs to slot that is being migrated and if it can be added to our buffer + if (migrateOperation.Contains(slot)) { - if (recordMetadata.RecordInfo.VectorSet) + if (srcLogRecord.RecordType == VectorManager.RecordType) { // We can't delete the vector set _yet_ nor can we migrate it, // we just need to remember it to migrate once the associated namespaces are all moved over - mss.EncounteredVectorSet(key.ToByteArray(), value.ToByteArray()); + migrateOperation.EncounteredVectorSet(key.ToArray(), srcLogRecord.ValueSpan.ToArray()); } - else if (!mss.sketch.TryHashAndStore(key.AsSpan())) + else if (!migrateOperation.sketch.TryHashAndStore(key)) { - // Out of space, end scan for now return false; } } @@ -68,50 +65,6 @@ public unsafe bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMeta return true; } - - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - } - #endregion - - #region objectStoreScan - internal sealed unsafe class ObjectStoreScan : IScanIteratorFunctions - { - readonly MigrateOperation mss; - - internal ObjectStoreScan(MigrateOperation mss) - { - this.mss = mss; - } - - public bool OnStart(long beginAddress, long endAddress) => true; - - public void OnStop(bool completed, long numberOfRecords) { } - - public void OnException(Exception exception, long numberOfRecords) { } - - public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public unsafe bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - - mss.ThrowIfCancelled(); - - // Do not send key if it is expired - // NOTE: Because the scan executes includingTombstones, tombstone records may not have valid expiration metadata; skip expiration validation here and defer it until the actual send occurs (MigrateSessionCommonUtils.cs:WriteOrSendObjectStoreKeyValuePair). - if (!recordMetadata.RecordInfo.Tombstone && ClusterSession.Expired(ref value)) - return true; - - var s = HashSlotUtils.HashSlot(key); - // Check if key belongs to slot that is being migrated and if it can be added to our buffer - if (mss.Contains(s) && !mss.sketch.TryHashAndStore(key.AsSpan())) - return false; - - return true; - } } - #endregion } } \ No newline at end of file diff --git a/libs/cluster/Server/Migration/MigrateSession.cs b/libs/cluster/Server/Migration/MigrateSession.cs index 07861d21fee..d67d6678d21 100644 --- a/libs/cluster/Server/Migration/MigrateSession.cs +++ b/libs/cluster/Server/Migration/MigrateSession.cs @@ -75,7 +75,7 @@ internal sealed partial class MigrateSession : IDisposable /// /// Get network buffer specs /// - public NetworkBufferSettings GetNetworkBufferSettings => clusterProvider.migrationManager.GetNetworkBufferSettings; + public NetworkBufferSettings NetworkBufferSettings => clusterProvider.migrationManager.GetNetworkBufferSettings; /// /// Get network pool @@ -167,7 +167,7 @@ internal MigrateSession( public GarnetClientSession GetGarnetClient() => new( new IPEndPoint(IPAddress.Parse(_targetAddress), _targetPort), - networkBufferSettings: GetNetworkBufferSettings, + networkBufferSettings: NetworkBufferSettings, networkPool: GetNetworkPool, clusterProvider?.serverOptions.TlsOptions?.TlsClientOptions, authUsername: _username, @@ -248,8 +248,6 @@ private async ValueTask CheckConnectionAsync(GarnetClientSession client) return slotRanges; } - - /// /// Reset local slot state /// @@ -280,7 +278,5 @@ public bool RelinquishOwnership() return false; return true; } - - } } \ No newline at end of file diff --git a/libs/cluster/Server/Migration/MigrateSessionCommonUtils.cs b/libs/cluster/Server/Migration/MigrateSessionCommonUtils.cs index 044cfe20a87..347ec3a8de0 100644 --- a/libs/cluster/Server/Migration/MigrateSessionCommonUtils.cs +++ b/libs/cluster/Server/Migration/MigrateSessionCommonUtils.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Threading.Tasks; using Garnet.client; +using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -13,144 +14,125 @@ namespace Garnet.cluster { internal sealed partial class MigrateSession : IDisposable { - private ValueTask WriteOrSendMainStoreKeyValuePairAsync(GarnetClientSession gcs, LocalServerSession localServerSession, ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory o, out GarnetStatus status) + private unsafe ValueTask WriteOrSendRecordAsync(GarnetClientSession gcs, LocalServerSession localServerSession, PinnedSpanByte namespaceBytes, PinnedSpanByte key, ref VectorInput input, ref VectorOutput output, out GarnetStatus status) { - // Read value for key - status = localServerSession.BasicGarnetApi.Read_MainStore(ref key, ref input, ref o); + Debug.Assert(namespaceBytes.Length == 1, "Longer namespaces not yet supported"); - // Skip if key NOTFOUND - if (status == GarnetStatus.NOTFOUND) - return new(true); + // Must initialize this here because we use the network buffer as output. + if (gcs.NeedsInitialization) + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: false); + + // Read the value for the key. This will populate output with the entire serialized record. + var storeStatus = localServerSession.VectorBasicContext.Read(new VectorElementKey(namespaceBytes.ReadOnlySpan[0], key.ReadOnlySpan), ref input, ref output); - // Get SpanByte from stack if any - ref var value = ref o.SpanByte; - if (!o.IsSpanByte) + if (storeStatus.IsPending) { - // Reinterpret heap memory to SpanByte - value = ref SpanByte.ReinterpretWithoutLength(o.Memory.Memory.Span); + CompletePending(ref storeStatus, ref output, ref localServerSession.VectorBasicContext); } - // Map up any namespaces as needed - // TODO: Better way to do "has namespace" - if (key.MetadataSize == 1) + if (storeStatus.Found) { - var oldNs = key.GetNamespaceInPayload(); - if (_namespaceMap.TryGetValue(oldNs, out var newNs)) - { - Debug.Assert(newNs <= byte.MaxValue, "Namespace too large"); - key.SetNamespaceInPayload((byte)newNs); - } + status = GarnetStatus.OK; + } + else if (storeStatus.IsWrongType) + { + status = GarnetStatus.WRONGTYPE; + } + else + { + status = GarnetStatus.NOTFOUND; } - // If expired, skip but do not fail - if (ClusterSession.Expired(ref value)) + // Skip (but do not fail) if key NOTFOUND, WRONGTYPE, BADSTATE, etc. + if (status != GarnetStatus.OK) { return new(true); } - // Write key to network buffer, potentially flushing if buffer is full - return WriteOrSendMainStoreKeyValuePairAsync(gcs, ref key, ref value); + // Map up any namespaces as needed + VectorSessionFunctions.UpdateMigratedElementNamespaces(_namespaceMap, ref input, ref output); - ValueTask WriteOrSendMainStoreKeyValuePairAsync(GarnetClientSession gcs, ref SpanByte key, ref SpanByte value) + fixed (byte* ptr = output.SpanByteAndMemory.Span) { - // Check if we need to initialize cluster migrate command arguments - if (gcs.NeedsInitialization) - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: true, isVectorSets: false); + return WriteOrSendRecordSpanAsync(gcs, MigrationRecordSpanType.VectorSetElement, new(ptr, output.SpanByteAndMemory.Span.Length)); + } - // Try write serialized key value to client buffer - if (!gcs.TryWriteKeyValueSpanByte(ref key, ref value, out var task)) - { - // Flush key value pairs in the buffer - var handleResponseTask = HandleMigrateTaskResponseAsync(task); + // Complete reads that go pending + static void CompletePending(ref Status status, ref VectorOutput output, ref VectorBasicContext ctx) + { + _ = ctx.CompletePendingWithOutputs(out var completedOutputs, wait: true); + var more = completedOutputs.Next(); + Debug.Assert(more); + status = completedOutputs.Current.Status; + output = completedOutputs.Current.Output; + Debug.Assert(!completedOutputs.Next()); + completedOutputs.Dispose(); + } + } - // Copy key & value for async completion - var keyCopy = new byte[key.TotalSize]; - var valueCopy = new byte[value.TotalSize]; - key.CopyTo(keyCopy); - value.CopyTo(valueCopy); + private unsafe ValueTask WriteOrSendRecordAsync(GarnetClientSession gcs, LocalServerSession localServerSession, PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output, out GarnetStatus status) + { + // Must initialize this here because we use the network buffer as output. + if (gcs.NeedsInitialization) + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: false); - return new(RetryHelperAsync(handleResponseTask, gcs, _sourceNodeId, _replaceOption, keyCopy, valueCopy, logger)); - } + // Read the value for the key. This will populate output with the entire serialized record. + status = localServerSession.BasicGarnetApi.Read_UnifiedStore(key, ref input, ref output); + // Skip (but do not fail) if key NOTFOUND, WRONGTYPE, BADSTATE, etc. + if (status != GarnetStatus.OK) + { return new(true); + } - static async Task RetryHelperAsync(Task handleResponseTask, GarnetClientSession gcs, string sourceNodeId, bool replaceOption, byte[] keyCopy, byte[] valueCopy, ILogger logger) - { - try - { - if (await handleResponseTask.ConfigureAwait(false)) - { - gcs.SetClusterMigrateHeader(sourceNodeId, replaceOption, isMainStore: true, isVectorSets: false); - - unsafe - { - fixed (byte* keyCopyPtr = keyCopy, valueCopyPtr = valueCopy) - { - ref var keyCopyRef = ref SpanByte.Reinterpret(keyCopyPtr); - ref var valueCopyRef = ref SpanByte.Reinterpret(valueCopyPtr); - - if (!gcs.TryWriteKeyValueSpanByte(ref keyCopyRef, ref valueCopyRef, out _)) - { - logger?.LogCritical($"{nameof(WriteOrSendMainStoreKeyValuePairAsync)} failed on retry"); - return false; - } - } - } - - return true; - } - } - catch (Exception ex) - { - logger?.LogError(ex, "Error occurred in WriteOrSendMainStoreKeyValuePairAsync async path"); - } + fixed (byte* ptr = output.SpanByteAndMemory.Span) + { + var serializedRecordLength = new LogRecord((long)ptr).GetSerializedSize(); - return false; - } + ReadOnlySpan toWrite = new(ptr, serializedRecordLength); + + return WriteOrSendRecordSpanAsync(gcs, MigrationRecordSpanType.LogRecord, toWrite); } } - private ValueTask WriteOrSendObjectStoreKeyValuePairAsync(GarnetClientSession gcs, LocalServerSession localServerSession, ref SpanByte key, out GarnetStatus status) + /// + /// Write a serialized record directly to the client buffer; if there is not enough room, flush the buffer and retry writing. + /// + /// The client session + /// + /// + /// True on success, else false + private ValueTask WriteOrSendRecordSpanAsync(GarnetClientSession gcs, MigrationRecordSpanType type, ReadOnlySpan span) { - var keyByteArray = key.AsReadOnlySpan().ToArray(); - - ObjectInput input = default; - GarnetObjectStoreOutput value = default; - status = localServerSession.BasicGarnetApi.Read_ObjectStore(ref keyByteArray, ref input, ref value); - - // Skip if key NOTFOUND - if (status == GarnetStatus.NOTFOUND) - return new(true); + // Check if we need to initialize cluster migrate command arguments + if (gcs.NeedsInitialization) + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: false); - if (!ClusterSession.Expired(ref value.GarnetObject)) + // Try to write serialized record to client buffer + if (!gcs.TryWriteRecordSpan(span, type, out var task)) { - var objectData = GarnetObjectSerializer.Serialize(value.GarnetObject); - - return WriteOrSendObjectStoreKeyValuePairAsync(gcs, keyByteArray, objectData, value.GarnetObject.Expiration); + // Flush records in the buffer and retry + var handleTask = HandleMigrateTaskResponseAsync(task); + return new(RetryAsync(gcs, handleTask, span.ToArray())); } return new(true); - async ValueTask WriteOrSendObjectStoreKeyValuePairAsync(GarnetClientSession gcs, byte[] key, byte[] value, long expiration) + async Task RetryAsync(GarnetClientSession gcs, Task task, byte[] span) { - // Check if we need to initialize cluster migrate command arguments - if (gcs.NeedsInitialization) - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: false, isVectorSets: false); - - if (!gcs.TryWriteKeyValueByteArray(key, value, expiration, out var task)) + if (!await task.ConfigureAwait(false)) { - // Flush key value pairs in the buffer - if (!await HandleMigrateTaskResponseAsync(task).ConfigureAwait(false)) - return false; + return false; + } - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: false, isVectorSets: false); + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: false); - if (!gcs.TryWriteKeyValueByteArray(key, value, expiration, out _)) - { - logger?.LogCritical($"{nameof(WriteOrSendObjectStoreKeyValuePairAsync)} failed on retry"); - return false; - } + if (!gcs.TryWriteRecordSpan(span, type, out _)) + { + logger?.LogWarning($"TryWriteRecordSpan failed on retry"); + return false; } + return true; } } @@ -166,16 +148,14 @@ public async Task HandleMigrateTaskResponseAsync(Task task) { try { - var res = await task.WaitAsync(_timeout, _cts.Token).ConfigureAwait(false); + var resp = await task.WaitAsync(_timeout, _cts.Token).ConfigureAwait(false); - // Check if setslotsrange executed correctly - if (!res.Equals("OK", StringComparison.Ordinal)) + if (!resp.Equals("OK", StringComparison.Ordinal)) { - logger?.LogError("ClusterMigrate Keys failed with error:{error}.", res); + logger?.LogError("ClusterMigrate Keys failed with error:{error}.", resp); Status = MigrateState.FAIL; return false; } - return true; } catch (Exception ex) @@ -185,6 +165,7 @@ public async Task HandleMigrateTaskResponseAsync(Task task) return false; } } + return true; } } diff --git a/libs/cluster/Server/Migration/MigrateSessionKeyAccess.cs b/libs/cluster/Server/Migration/MigrateSessionKeyAccess.cs index 9b8e59e5263..3eb47a35ebd 100644 --- a/libs/cluster/Server/Migration/MigrateSessionKeyAccess.cs +++ b/libs/cluster/Server/Migration/MigrateSessionKeyAccess.cs @@ -4,7 +4,7 @@ using System; using System.Threading.Tasks; using Garnet.common; -using Garnet.server; +using Tsavorite.core; namespace Garnet.cluster { @@ -32,7 +32,7 @@ private Task WaitForConfigPropagationAsync() /// /// /// - public bool CanAccessKey(ref ArgSlice key, int slot, bool readOnly) + public bool CanAccessKey(PinnedSpanByte key, int slot, bool readOnly) { // Skip operation check since this session is not responsible for migrating the associated slot if (!_sslots.Contains(slot)) @@ -41,7 +41,7 @@ public bool CanAccessKey(ref ArgSlice key, int slot, bool readOnly) var state = SketchStatus.INITIALIZING; foreach (var migrateTask in migrateOperation) { - if (migrateTask.sketch.Probe(key.SpanByte, out state)) + if (migrateTask.sketch.Probe(key, out state)) goto found; } diff --git a/libs/cluster/Server/Migration/MigrateSessionKeys.cs b/libs/cluster/Server/Migration/MigrateSessionKeys.cs index 02a1adeefef..a1ca398c86d 100644 --- a/libs/cluster/Server/Migration/MigrateSessionKeys.cs +++ b/libs/cluster/Server/Migration/MigrateSessionKeys.cs @@ -2,9 +2,12 @@ // Licensed under the MIT license. using System; +using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Garnet.client; using Garnet.server; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -17,22 +20,12 @@ namespace Garnet.cluster internal sealed partial class MigrateSession : IDisposable { /// - /// Method used to migrate individual keys from main store to target node. + /// Method used to migrate individual keys from store to target node. /// Used with MIGRATE KEYS option /// /// True on success, false otherwise - private async Task MigrateKeysFromMainStoreAsync() + private async Task MigrateKeysFromStoreAsync() { - var bufferSize = 1 << 10; - SectorAlignedMemory buffer = new(bufferSize, 1); - IntPtr bufPtr, bufPtrEnd; - SpanByteAndMemory o; - unsafe - { - bufPtr = (IntPtr)buffer.GetValidPointer(); - bufPtrEnd = bufPtr + bufferSize; - o = new SpanByteAndMemory((byte*)bufPtr, (int)(bufPtrEnd - bufPtr)); - } var migrateTask = migrateOperation[0]; try @@ -53,13 +46,14 @@ private async Task MigrateKeysFromMainStoreAsync() return false; } - // Transmit keys from main store - if (!await migrateTask.TransmitKeysAsync(StoreType.Main, indexesToMigrate).ConfigureAwait(false)) + // Transmit keys from store + if (!await migrateTask.TransmitKeysAsync(indexesToMigrate).ConfigureAwait(false)) { - logger?.LogError("Failed transmitting keys from main store"); + logger?.LogError("Failed transmitting keys from store"); return false; } + // Move Vector Sets over after individual keys are moved if ((_namespaces?.Count ?? 0) > 0) { // Actually move element data over @@ -72,55 +66,54 @@ private async Task MigrateKeysFromMainStoreAsync() // Move the indexes over var gcs = migrateTask.Client; - foreach (var (key, value) in indexesToMigrate) - { - // Update the index context as we move it, so it arrives on the destination node pointed at the appropriate - // namespaces for element data - VectorManager.ReadIndex(value, out var oldContext, out _, out _, out _, out _, out _, out _, out _, out _); + var serializeBufferArr = ArrayPool.Shared.Rent(128); - var newContext = _namespaceMap[oldContext]; - VectorManager.SetContextForMigration(value, newContext); + try + { - Task pendingHandleTask; - retryKeyAndValue: - unsafe + foreach (var (key, value) in indexesToMigrate) { - fixed (byte* keyPtr = key, valuePtr = value) - { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); - var valSpan = SpanByte.FromPinnedPointer(valuePtr, value.Length); + // Update the index context as we move it, so it arrives on the destination node pointed at the appropriate + // namespaces for element data + VectorManager.ReadIndex(value, out var oldContext, out _, out _, out _, out _, out _, out _, out _); - if (gcs.NeedsInitialization) - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: true, isVectorSets: true); + var newContext = _namespaceMap[oldContext]; + VectorManager.SetContextForMigration(value, newContext); - if (!gcs.TryWriteKeyValueSpanByte(ref keySpan, ref valSpan, out var task)) - { - // Need to wait for response, but can't do so in unsafe... - pendingHandleTask = HandleMigrateTaskResponseAsync(task); - goto awaitAndRetry; - } + var neededSpace = sizeof(int) + key.Length + sizeof(int) + value.Length; - continue; + if (neededSpace > serializeBufferArr.Length) + { + ArrayPool.Shared.Return(serializeBufferArr); + serializeBufferArr = ArrayPool.Shared.Rent(neededSpace); } - } - awaitAndRetry: - if (!await pendingHandleTask.ConfigureAwait(false)) - { - unsafe { - fixed (byte* keyPtr = key) - { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); + Span serializeBuffer = serializeBufferArr; + BinaryPrimitives.WriteInt32LittleEndian(serializeBuffer, key.Length); + key.CopyTo(serializeBuffer[sizeof(int)..]); + BinaryPrimitives.WriteInt32LittleEndian(serializeBuffer[(sizeof(int) + key.Length)..], value.Length); + value.CopyTo(serializeBuffer[(sizeof(int) + key.Length + sizeof(int))..]); + } + + if (gcs.NeedsInitialization) + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: true); - logger?.LogCritical("Failed to migrate Vector Set key {key} during migration", keySpan); + while (!gcs.TryWriteRecordSpan(serializeBufferArr.AsSpan()[..neededSpace], MigrationRecordSpanType.VectorSetIndex, out var task)) + { + if (!await HandleMigrateTaskResponseAsync(task).ConfigureAwait(false)) + { + logger?.LogCritical("Failed to migrate Vector Set key {key} during migration", SpanByte.ToShortString(key)); return false; } + + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: true); } } - - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: true, isVectorSets: true); - goto retryKeyAndValue; + } + finally + { + ArrayPool.Shared.Return(serializeBufferArr); } if (!await HandleMigrateTaskResponseAsync(gcs.SendAndResetIterationBuffer()).ConfigureAwait(false)) @@ -129,47 +122,16 @@ private async Task MigrateKeysFromMainStoreAsync() return false; } } - // Final cleanup, which will also delete Vector Sets await DeleteKeysAsync().ConfigureAwait(false); } finally { - // If allocated memory in heap dispose it here. - if (o.Memory != default) - o.Memory.Dispose(); - buffer.Dispose(); - migrateOperation[0].sketch.SetStatus(SketchStatus.INITIALIZING); } return true; } - /// - /// Method used to migrate individual keys from object store to target node. - /// Used with MIGRATE KEYS option - /// - /// True on success, false otherwise - private async Task MigrateKeysFromObjectStoreAsync() - { - var migrateTask = migrateOperation[0]; - // NOTE: Any keys not found in main store are automatically set to INITIALIZING before this method is called - // Transition all INITIALIZING to TRANSMITTING state - migrateTask.sketch.SetStatus(SketchStatus.TRANSMITTING); - await WaitForConfigPropagationAsync().ConfigureAwait(false); - - // Transmit keys from object store - if (!await migrateTask.TransmitKeysAsync(StoreType.Object, new(ByteArrayComparer.Instance)).ConfigureAwait(false)) - { - logger?.LogError("Failed transmitting keys from object store"); - return false; - } - - // Delete keys if COPY option is false or transition KEYS from MIGRATING to MIGRATED status - await DeleteKeysAsync().ConfigureAwait(false); - return true; - } - /// /// Delete local copy of keys if _copyOption is set to false. /// @@ -202,15 +164,8 @@ public async Task MigrateKeysAsync() return false; // Migrate main store keys - if (!await MigrateKeysFromMainStoreAsync().ConfigureAwait(false)) + if (!await MigrateKeysFromStoreAsync().ConfigureAwait(false)) return false; - - // Migrate object store keys - if (!clusterProvider.serverOptions.DisableObjects) - { - if (!await MigrateKeysFromObjectStoreAsync().ConfigureAwait(false)) - return false; - } } catch (Exception ex) { diff --git a/libs/cluster/Server/Migration/MigrateSessionSlots.cs b/libs/cluster/Server/Migration/MigrateSessionSlots.cs index 223c8e9585e..e1035164837 100644 --- a/libs/cluster/Server/Migration/MigrateSessionSlots.cs +++ b/libs/cluster/Server/Migration/MigrateSessionSlots.cs @@ -2,15 +2,20 @@ // Licensed under the MIT license. using System; +using System.Buffers; +using System.Buffers.Binary; using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using Garnet.client; +using Garnet.server; + #if DEBUG using Garnet.common; #endif -using Garnet.server; + using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -72,40 +77,31 @@ public async Task MigrateSlotsDriverInlineAsync() { var storeBeginAddress = clusterProvider.storeWrapper.store.Log.BeginAddress; var storeTailAddress = clusterProvider.storeWrapper.store.Log.TailAddress; - var mainStorePageSize = 1 << clusterProvider.serverOptions.PageSizeBits(); + var storePageSize = 1 << clusterProvider.serverOptions.PageSizeBits(); #if DEBUG // Only on Debug mode - await ExceptionInjectionHelper.WaitOnSetAsync(ExceptionInjectionType.Migration_Slot_End_Scan_Range_Acquisition).ConfigureAwait(false); + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Migration_Slot_End_Scan_Range_Acquisition).ConfigureAwait(false); #endif - // Send main store + // Send store logger?.LogWarning("Store migrate scan range [{storeBeginAddress}, {storeTailAddress}]", storeBeginAddress, storeTailAddress); - var success = await CreateAndRunMigrateTasksAsync(StoreType.Main, storeBeginAddress, storeTailAddress, mainStorePageSize); - if (!success) return false; - // Send object store - if (!clusterProvider.serverOptions.DisableObjects) - { - var objectStoreBeginAddress = clusterProvider.storeWrapper.objectStore.Log.BeginAddress; - var objectStoreTailAddress = clusterProvider.storeWrapper.objectStore.Log.TailAddress; - var objectStorePageSize = 1 << clusterProvider.serverOptions.ObjectStorePageSizeBits(); - logger?.LogWarning("Object Store migrate scan range [{objectStoreBeginAddress}, {objectStoreTailAddress}]", objectStoreBeginAddress, objectStoreTailAddress); - success = await CreateAndRunMigrateTasksAsync(StoreType.Object, objectStoreBeginAddress, objectStoreTailAddress, objectStorePageSize); - if (!success) return false; - } + var success = await CreateAndRunMigrateTasksAsync(storeBeginAddress, storeTailAddress, storePageSize); + if (!success) return false; return true; - async Task CreateAndRunMigrateTasksAsync(StoreType storeType, long beginAddress, long tailAddress, int pageSize) + async Task CreateAndRunMigrateTasksAsync(long beginAddress, long tailAddress, int pageSize) { - logger?.LogTrace("{method} > [{storeType}] Scan in range ({BeginAddress},{TailAddress})", nameof(CreateAndRunMigrateTasksAsync), storeType, beginAddress, tailAddress); + logger?.LogTrace("{method} > Scan in range ({BeginAddress},{TailAddress})", nameof(CreateAndRunMigrateTasksAsync), beginAddress, tailAddress); var migrateOperationRunners = new Task[clusterProvider.serverOptions.ParallelMigrateTaskCount]; + var i = 0; while (i < migrateOperationRunners.Length) { var idx = i; - migrateOperationRunners[idx] = ScanStoreTaskAsync(idx, storeType, beginAddress, tailAddress, pageSize); + migrateOperationRunners[idx] = ScanStoreTaskAsync(idx, beginAddress, tailAddress, pageSize); i++; } @@ -119,56 +115,60 @@ async Task CreateAndRunMigrateTasksAsync(StoreType storeType, long beginAd } // Handle migration of discovered Vector Set keys now that they're namespaces have been moved - if (storeType == StoreType.Main) + var vectorSets = migrateOperation.SelectMany(static mo => mo.VectorSets).GroupBy(static g => g.Key, ByteArrayComparer.Instance).ToDictionary(static g => g.Key, g => g.First().Value, ByteArrayComparer.Instance); + + if (vectorSets.Count > 0) { - var vectorSets = migrateOperation.SelectMany(static mo => mo.VectorSets).GroupBy(static g => g.Key, ByteArrayComparer.Instance).ToDictionary(static g => g.Key, g => g.First().Value, ByteArrayComparer.Instance); + var gcs = migrateOperation[0].Client; - if (vectorSets.Count > 0) + var serializeBufferArr = ArrayPool.Shared.Rent(128); + try { - var gcs = migrateOperation[0].Client; - foreach (var (key, value) in vectorSets) { // Update the index context as we move it, so it arrives on the destination node pointed at the appropriate // namespaces for element data - VectorManager.ReadIndex(value, out var oldContext, out _, out _, out _, out _, out _, out _, out _, out _); + VectorManager.ReadIndex(value, out var oldContext, out _, out _, out _, out _, out _, out _, out _); var newContext = _namespaceMap[oldContext]; VectorManager.SetContextForMigration(value, newContext); - Task pendingHandleTask; - retryKeyAndValue: - unsafe + var neededSpace = sizeof(int) + key.Length + sizeof(int) + value.Length; + + if (neededSpace > serializeBufferArr.Length) + { + ArrayPool.Shared.Return(serializeBufferArr); + serializeBufferArr = ArrayPool.Shared.Rent(neededSpace); + } + + // Scope so Span doesn't cross await boundary + { + Span serializeBuffer = serializeBufferArr; + BinaryPrimitives.WriteInt32LittleEndian(serializeBuffer, key.Length); + key.CopyTo(serializeBuffer[sizeof(int)..]); + BinaryPrimitives.WriteInt32LittleEndian(serializeBuffer[(sizeof(int) + key.Length)..], value.Length); + value.CopyTo(serializeBuffer[(sizeof(int) + key.Length + sizeof(int))..]); + } + + if (gcs.NeedsInitialization) + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: true); + + while (!gcs.TryWriteRecordSpan(serializeBufferArr.AsSpan()[..neededSpace], MigrationRecordSpanType.VectorSetIndex, out var task)) { - fixed (byte* keyPtr = key, valuePtr = value) + if (!await HandleMigrateTaskResponseAsync(task).ConfigureAwait(false)) { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); - var valSpan = SpanByte.FromPinnedPointer(valuePtr, value.Length); - - if (gcs.NeedsInitialization) - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: true, isVectorSets: true); - - if (!gcs.TryWriteKeyValueSpanByte(ref keySpan, ref valSpan, out var task)) - { - // Need to wait for response, but can't do so in unsafe... - pendingHandleTask = HandleMigrateTaskResponseAsync(task); - goto awaitAndRetry; - } + logger?.LogCritical("Failed to migrate Vector Set key {key} during migration", SpanByte.ToShortString(key)); + return false; } + + gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isVectorSets: true); } // Force a flush before doing the delete, in case that fails if (!await HandleMigrateTaskResponseAsync(gcs.SendAndResetIterationBuffer()).ConfigureAwait(false)) { - unsafe - { - fixed (byte* keyPtr = key) - { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); - logger?.LogCritical("Flush failed before deletion of Vector Set {key} duration migration", keySpan); - return false; - } - } + logger?.LogCritical("Flush failed before deletion of Vector Set {key} duration migration", SpanByte.ToShortString(key)); + return false; } // Delete the index on this node now that it's moved over to the destination node @@ -176,46 +176,28 @@ async Task CreateAndRunMigrateTasksAsync(StoreType storeType, long beginAd { fixed (byte* keyPtr = key) { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); - migrateOperation[0].DeleteVectorSet(ref keySpan); + var pinnedKeySpan = PinnedSpanByte.FromPinnedPointer(keyPtr, key.Length); + migrateOperation[0].DeleteVectorSet(pinnedKeySpan); } } - - // Move to next Vector Set index key - continue; - - - awaitAndRetry: - if (!await pendingHandleTask.ConfigureAwait(false)) - { - unsafe - { - fixed (byte* keyPtr = key) - { - var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); - logger?.LogCritical("Failed to migrate Vector Set key {key} during migration", keySpan); - return false; - } - } - } - - gcs.SetClusterMigrateHeader(_sourceNodeId, _replaceOption, isMainStore: true, isVectorSets: true); - goto retryKeyAndValue; } } + finally + { + ArrayPool.Shared.Return(serializeBufferArr); + } } } catch (Exception ex) { - logger?.LogError(ex, "{CreateAndRunMigrateTasks}: {storeType} {beginAddress} {tailAddress} {pageSize}", nameof(CreateAndRunMigrateTasksAsync), storeType, beginAddress, tailAddress, pageSize); + logger?.LogError(ex, "{CreateAndRunMigrateTasks}: {beginAddress} {tailAddress} {pageSize}", nameof(CreateAndRunMigrateTasksAsync), beginAddress, tailAddress, pageSize); await _cts.CancelAsync().ConfigureAwait(false); return false; } - return true; } - async Task ScanStoreTaskAsync(int taskId, StoreType storeType, long beginAddress, long tailAddress, int pageSize) + async Task ScanStoreTaskAsync(int taskId, long beginAddress, long tailAddress, int pageSize) { // Force async await Task.Yield(); @@ -231,13 +213,13 @@ async Task ScanStoreTaskAsync(int taskId, StoreType storeType, long beginA return false; var cursor = workerStartAddress; - logger?.LogWarning("<{StoreType}:{taskId}> migrate scan range [{workerStartAddress}, {workerEndAddress}]", storeType, taskId, workerStartAddress, workerEndAddress); + logger?.LogWarning("<{taskId}> migrate scan range [{workerStartAddress}, {workerEndAddress}]", taskId, workerStartAddress, workerEndAddress); while (true) { var current = cursor; // Build Sketch migrateOperation.sketch.SetStatus(SketchStatus.INITIALIZING); - migrateOperation.Scan(storeType, ref current, workerEndAddress); + migrateOperation.Scan(ref current, workerEndAddress); // Stop if no keys have been found if (migrateOperation.sketch.argSliceVector.IsEmpty) break; @@ -250,7 +232,7 @@ async Task ScanStoreTaskAsync(int taskId, StoreType storeType, long beginA await WaitForConfigPropagationAsync().ConfigureAwait(false); // Transmit all keys gathered - if (!await migrateOperation.TransmitSlotsAsync(storeType).ConfigureAwait(false)) + if (!await migrateOperation.TransmitSlotsAsync().ConfigureAwait(false)) { logger?.LogWarning("[{taskId}> TransmitSlots failed for {cursor} to {current} (with {count} keys)", taskId, cursor, current, migrateOperation.sketch.argSliceVector.Count); return false; diff --git a/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs b/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs index 5942a72df59..eede7925209 100644 --- a/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs +++ b/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs @@ -5,8 +5,8 @@ using System.Collections.Generic; using System.Diagnostics; using Garnet.common; -using Garnet.server; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.cluster { @@ -23,7 +23,9 @@ public MigrateSessionTaskStore(ILogger logger = null) this.logger = logger; } - /// + /// + /// Disposes all managed resources held by this instance. + /// public void Dispose() { _lock.WriteLock(); @@ -39,16 +41,8 @@ public void Dispose() for (var i = 0; i < sessions.Length; i++) { - try - { - sessions[i]?.Dispose(); - } - catch (Exception e) - { - logger?.LogError(e, "Exception disposing MigrateSession instance during MigrateSessionTaskStore.Dispose"); - } + sessions[i]?.Dispose(); } - Array.Clear(sessions); } @@ -224,7 +218,7 @@ public bool TryRemove(string targetNodeId) /// /// /// True if we can operate on the key, otherwise false (i.e. key is being migrated) - public bool CanAccessKey(ref ArgSlice key, int slot, bool readOnly) + public bool CanAccessKey(PinnedSpanByte key, int slot, bool readOnly) { try { @@ -238,7 +232,7 @@ public bool CanAccessKey(ref ArgSlice key, int slot, bool readOnly) Debug.Assert(s != null); // Check owner of slot if can operate on key - if (!s.CanAccessKey(ref key, slot, readOnly)) + if (!s.CanAccessKey(key, slot, readOnly)) return false; } finally diff --git a/libs/cluster/Server/Migration/MigrationDriver.cs b/libs/cluster/Server/Migration/MigrationDriver.cs index 4f007ac8503..90e25af5313 100644 --- a/libs/cluster/Server/Migration/MigrationDriver.cs +++ b/libs/cluster/Server/Migration/MigrationDriver.cs @@ -165,7 +165,7 @@ private async Task BeginAsyncMigrationTaskAsync() // If we have any namespaces, that implies Vector Sets, and if we have any of THOSE // we need to reserve destination sets on the other side - if ((_namespaces?.Count ?? 0) > 0 && !await ReserveDestinationVectorSetsAsync()) + if ((_namespaces?.Count ?? 0) > 0 && !await ReserveDestinationVectorSetsAsync().ConfigureAwait(false)) { logger?.LogError("Failed to reserve destination vector sets, migration failed"); await TryRecoverFromFailureAsync().ConfigureAwait(false); @@ -175,21 +175,20 @@ private async Task BeginAsyncMigrationTaskAsync() #region migrateData // Migrate actual data - if (!await MigrateSlotsDriverInlineAsync()) + if (!await MigrateSlotsDriverInlineAsync().ConfigureAwait(false)) { logger?.LogError("MigrateSlotsDriver failed"); await TryRecoverFromFailureAsync().ConfigureAwait(false); Status = MigrateState.FAIL; return; } - #endregion #region transferSlotOwnnershipToTargetNode // Lock config merge to avoid a background epoch bump clusterProvider.clusterManager.SuspendConfigMerge(); configResumed = false; - await clusterProvider.clusterManager.TryMeetAsync(_targetAddress, _targetPort, acquireLock: false); + await clusterProvider.clusterManager.TryMeetAsync(_targetAddress, _targetPort, acquireLock: false).ConfigureAwait(false); // Change ownership of slots to target node. if (!await TrySetSlotRangesAsync(GetTargetNodeId, MigrateState.NODE).ConfigureAwait(false)) @@ -210,7 +209,7 @@ private async Task BeginAsyncMigrationTaskAsync() } // Gossip again to ensure that source and target agree on the slot exchange - await clusterProvider.clusterManager.TryMeetAsync(_targetAddress, _targetPort, acquireLock: false); + await clusterProvider.clusterManager.TryMeetAsync(_targetAddress, _targetPort, acquireLock: false).ConfigureAwait(false); #endregion // Enqueue success log diff --git a/libs/cluster/Server/Migration/MigrationManager.cs b/libs/cluster/Server/Migration/MigrationManager.cs index a4188d751b4..4bee4680531 100644 --- a/libs/cluster/Server/Migration/MigrationManager.cs +++ b/libs/cluster/Server/Migration/MigrationManager.cs @@ -4,8 +4,8 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using Garnet.common; -using Garnet.server; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.cluster { @@ -43,7 +43,7 @@ public MigrationManager(ClusterProvider clusterProvider, ILogger logger = null) this.clusterProvider = clusterProvider; var sendBufferSize = 1 << clusterProvider.serverOptions.PageSizeBits(); this.networkBufferSettings = new NetworkBufferSettings(sendBufferSize, initialReceiveBufferSize); - this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); + this.networkPool = networkBufferSettings.CreateBufferPool(ownerType: PoolOwnerType.Migration, logger: logger); logger?.LogInformation("NetworkBufferSettings.sendBufferSize:{sendBufferSize}", networkBufferSettings.sendBufferSize); logger?.LogInformation("NetworkBufferSettings.initialReceiveBufferSize:{initialReceiveBufferSize}", networkBufferSettings.initialReceiveBufferSize); @@ -149,7 +149,7 @@ public bool TryRemoveMigrationTask(string targetNodeId) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CanAccessKey(ref ArgSlice key, int slot, bool readOnly) - => migrationTaskStore.CanAccessKey(ref key, slot, readOnly); + public bool CanAccessKey(PinnedSpanByte key, int slot, bool readOnly) + => migrationTaskStore.CanAccessKey(key, slot, readOnly); } } \ No newline at end of file diff --git a/libs/cluster/Server/Migration/Sketch.cs b/libs/cluster/Server/Migration/Sketch.cs index 59f3d0bc4a5..cee4463cf43 100644 --- a/libs/cluster/Server/Migration/Sketch.cs +++ b/libs/cluster/Server/Migration/Sketch.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using Garnet.common; using Garnet.server; using Tsavorite.core; @@ -15,7 +16,7 @@ internal class Sketch readonly int size; public readonly ArgSliceVector argSliceVector; - public List<(ArgSlice, bool)> Keys { private set; get; } + public List<(PinnedSpanByte, bool)> Keys { private set; get; } public SketchStatus Status { private set; get; } public Sketch(int keyCount = 1 << 20) @@ -31,7 +32,7 @@ public Sketch(int keyCount = 1 << 20) #region sketchMethods - public bool TryHashAndStore(Span key) + public bool TryHashAndStore(ReadOnlySpan key) { if (!argSliceVector.TryAddItem(key)) return false; @@ -44,12 +45,14 @@ public bool TryHashAndStore(Span key) return true; } - public bool TryHashAndStore(ulong ns, Span key) + public bool TryHashAndStore(ReadOnlySpan ns, ReadOnlySpan key) { + Debug.Assert(ns.Length == 1, "Longer namespaces not yet supported"); + if (!argSliceVector.TryAddItem(ns, key)) return false; - var slot = (int)HashUtils.MurmurHash2x64A(key, seed: (uint)ns) & (size - 1); + var slot = (int)HashUtils.MurmurHash2x64A(key, seed: (uint)ns[0]) & (size - 1); var byteOffset = slot >> 3; var bitOffset = slot & 7; bitmap[byteOffset] = (byte)(bitmap[byteOffset] | (1UL << bitOffset)); @@ -61,9 +64,9 @@ public bool TryHashAndStore(ulong ns, Span key) /// Hash key to bloomfilter and store it for future use (NOTE: Use only with KEYS option) /// /// - public unsafe void HashAndStore(ref ArgSlice key) + public void HashAndStore(PinnedSpanByte key) { - var slot = (int)HashUtils.MurmurHash2x64A(key.Span) & (size - 1); + var slot = (int)HashUtils.MurmurHash2x64A(key.ReadOnlySpan) & (size - 1); var byteOffset = slot >> 3; var bitOffset = slot & 7; bitmap[byteOffset] = (byte)(bitmap[byteOffset] | (1UL << bitOffset)); @@ -76,21 +79,9 @@ public unsafe void HashAndStore(ref ArgSlice key) /// /// /// - public unsafe bool Probe(SpanByte key, out SketchStatus status) + public unsafe bool Probe(PinnedSpanByte key, out SketchStatus status) { - int slot; - - // TODO: better way to detect namespace - if (key.MetadataSize == 1) - { - var ns = key.GetNamespaceInPayload(); - slot = (int)HashUtils.MurmurHash2x64A(key.ToPointer(), key.Length, seed: (uint)ns) & (size - 1); - } - else - { - slot = (int)HashUtils.MurmurHash2x64A(key.ToPointer(), key.Length) & (size - 1); - } - + var slot = (int)HashUtils.MurmurHash2x64A(key.ToPointer(), key.Length) & (size - 1); var byteOffset = slot >> 3; var bitOffset = slot & 7; diff --git a/libs/cluster/Server/Replication/CheckpointEntry.cs b/libs/cluster/Server/Replication/CheckpointEntry.cs index c7f3ff4d300..40f9e86ce3a 100644 --- a/libs/cluster/Server/Replication/CheckpointEntry.cs +++ b/libs/cluster/Server/Replication/CheckpointEntry.cs @@ -21,40 +21,44 @@ public static void LogCheckpointEntry(this ILogger logger, LogLevel logLevel, st "storeHlogToken: {storeHlogToken}\n" + "storeIndexToken: {storeIndexToken}\n" + "storeCheckpointCoveredAofAddress: {storeCheckpointCoveredAofAddress}\n" + - "------------------------------------------------------------------------\n" + - "objectStoreVersion:{objectStoreVersion}\n" + - "objectStoreHlogToken:{objectStoreHlogToken}\n" + - "objectStoreIndexToken:{objectStoreIndexToken}\n" + - "objectCheckpointCoveredAofAddress:{objectCheckpointCoveredAofAddress}\n" + "------------------------------------------------------------------------\n", msg, entry._lock, entry.metadata.storeVersion, entry.metadata.storeHlogToken, entry.metadata.storeIndexToken, - entry.metadata.storeCheckpointCoveredAofAddress, - entry.metadata.objectStoreVersion, - entry.metadata.objectStoreHlogToken, - entry.metadata.objectStoreIndexToken, - entry.metadata.objectCheckpointCoveredAofAddress); + entry.metadata.storeCheckpointCoveredAofAddress); } } sealed class CheckpointEntry { + private const int GuidSize = 16; public CheckpointMetadata metadata; public SingleWriterMultiReaderLock _lock; public CheckpointEntry next; public CheckpointEntry() { - metadata = new(); + metadata = null; next = null; _lock = new(); } - public long GetMinAofCoveredAddress() - => Math.Max(Math.Min(metadata.storeCheckpointCoveredAofAddress, metadata.objectCheckpointCoveredAofAddress), 64); + public CheckpointEntry(int physicalSublogCount) + { + metadata = new(physicalSublogCount); + next = null; + _lock = new(); + } + + + public AofAddress GetMinAofCoveredAddress() + { + var minCoveredAofAddress = metadata.storeCheckpointCoveredAofAddress; + minCoveredAofAddress.MaxExchange(ReplicationManager.kFirstValidAofAddress); + return minCoveredAofAddress; + } /// /// Indicate addition of new reader by trying to increment reader counter @@ -89,8 +93,6 @@ public bool ContainsSharedToken(CheckpointEntry entry, CheckpointFileType fileTy { CheckpointFileType.STORE_HLOG => metadata.storeHlogToken.Equals(entry.metadata.storeHlogToken), CheckpointFileType.STORE_INDEX => metadata.storeIndexToken.Equals(entry.metadata.storeIndexToken), - CheckpointFileType.OBJ_STORE_HLOG => metadata.objectStoreHlogToken.Equals(entry.metadata.objectStoreHlogToken), - CheckpointFileType.OBJ_STORE_INDEX => metadata.objectStoreIndexToken.Equals(entry.metadata.objectStoreIndexToken), _ => throw new Exception($"Option {fileType} not supported") }; } @@ -103,32 +105,15 @@ public byte[] ToByteArray() { var ms = new MemoryStream(); var writer = new BinaryWriter(ms, Encoding.ASCII); - byte[] byteBuffer; // Write checkpoint entry data for main store writer.Write(metadata.storeVersion); - byteBuffer = metadata.storeHlogToken.ToByteArray(); - writer.Write(byteBuffer.Length); - writer.Write(byteBuffer); - byteBuffer = metadata.storeIndexToken.ToByteArray(); - writer.Write(byteBuffer.Length); - writer.Write(byteBuffer); - writer.Write(metadata.storeCheckpointCoveredAofAddress); + writer.Write(metadata.storeHlogToken.ToByteArray()); + writer.Write(metadata.storeIndexToken.ToByteArray()); + metadata.storeCheckpointCoveredAofAddress.Serialize(writer); writer.Write(metadata.storePrimaryReplId == null ? 0 : 1); if (metadata.storePrimaryReplId != null) writer.Write(metadata.storePrimaryReplId); - // Write checkpoint entry data for object store - writer.Write(metadata.objectStoreVersion); - byteBuffer = metadata.objectStoreHlogToken.ToByteArray(); - writer.Write(byteBuffer.Length); - writer.Write(byteBuffer); - byteBuffer = metadata.objectStoreIndexToken.ToByteArray(); - writer.Write(byteBuffer.Length); - writer.Write(byteBuffer); - writer.Write(metadata.objectCheckpointCoveredAofAddress); - writer.Write(metadata.objectStorePrimaryReplId == null ? 0 : 1); - if (metadata.objectStorePrimaryReplId != null) writer.Write(metadata.objectStorePrimaryReplId); - var byteArray = ms.ToArray(); writer.Dispose(); ms.Dispose(); @@ -143,35 +128,23 @@ public byte[] ToByteArray() public static CheckpointEntry FromByteArray(byte[] serialized) { if (serialized.Length == 0) return null; - var ms = new MemoryStream(serialized); - var reader = new BinaryReader(ms); - var cEntry = new CheckpointEntry + using var ms = new MemoryStream(serialized); + using var reader = new BinaryReader(ms); + var cEntry = new CheckpointEntry() { metadata = new() { storeVersion = reader.ReadInt64(), - storeHlogToken = new Guid(reader.ReadBytes(reader.ReadInt32())), - storeIndexToken = new Guid(reader.ReadBytes(reader.ReadInt32())), - storeCheckpointCoveredAofAddress = reader.ReadInt64(), - storePrimaryReplId = reader.ReadInt32() > 0 ? reader.ReadString() : default, - - objectStoreVersion = reader.ReadInt64(), - objectStoreHlogToken = new Guid(reader.ReadBytes(reader.ReadInt32())), - objectStoreIndexToken = new Guid(reader.ReadBytes(reader.ReadInt32())), - objectCheckpointCoveredAofAddress = reader.ReadInt64(), - objectStorePrimaryReplId = reader.ReadInt32() > 0 ? reader.ReadString() : default + storeHlogToken = new Guid(reader.ReadBytes(GuidSize)), + storeIndexToken = new Guid(reader.ReadBytes(GuidSize)), + storeCheckpointCoveredAofAddress = AofAddress.Deserialize(reader), + storePrimaryReplId = reader.ReadInt32() > 0 ? reader.ReadString() : default } }; - reader.Dispose(); - ms.Dispose(); return cEntry; } - /// - /// - /// - /// public override string ToString() => $"{metadata},readers={_lock}"; } } \ No newline at end of file diff --git a/libs/cluster/Server/Replication/CheckpointFileType.cs b/libs/cluster/Server/Replication/CheckpointFileType.cs index da2ac9690ba..e5721b6a6e3 100644 --- a/libs/cluster/Server/Replication/CheckpointFileType.cs +++ b/libs/cluster/Server/Replication/CheckpointFileType.cs @@ -1,9 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; -using Garnet.server; - namespace Garnet.cluster { /// @@ -16,64 +13,25 @@ enum CheckpointFileType : byte /// NONE = 0, /// - /// Store Hybrid LOG + /// Store Hybrid LOG - Main /// - STORE_HLOG, + STORE_HLOG = 1, /// - /// Store Delta Log + /// Store Hybrid LOG - Object /// - STORE_DLOG, + STORE_HLOG_OBJ = 2, + // Value 3 reserved (was STORE_DLOG, removed with incremental snapshots) /// /// Store Index /// - STORE_INDEX, - /// - /// Store Snapshot - /// - STORE_SNAPSHOT, - /// - /// Object Store Hybrid Log - Main - /// - OBJ_STORE_HLOG, - /// - /// Object Store Hybrid Log - Object - /// - OBJ_STORE_HLOG_OBJ, + STORE_INDEX = 4, /// - /// Object Store Delta Log + /// Store Snapshot - Main /// - OBJ_STORE_DLOG, + STORE_SNAPSHOT = 5, /// - /// Object Store Index + /// Store Snapshot - Object /// - OBJ_STORE_INDEX, - /// - /// Object Store Snapshot - Main - /// - OBJ_STORE_SNAPSHOT, - /// - /// Object Store Snapshot - Object - /// - OBJ_STORE_SNAPSHOT_OBJ, - } - - static class CheckpointFileTypeExtensions - { - public static StoreType ToStoreType(this CheckpointFileType type) - { - return type switch - { - CheckpointFileType.STORE_HLOG or - CheckpointFileType.STORE_DLOG or - CheckpointFileType.STORE_INDEX or - CheckpointFileType.STORE_SNAPSHOT => StoreType.Main, - CheckpointFileType.OBJ_STORE_HLOG or - CheckpointFileType.OBJ_STORE_DLOG or - CheckpointFileType.OBJ_STORE_INDEX or - CheckpointFileType.OBJ_STORE_SNAPSHOT or - CheckpointFileType.OBJ_STORE_SNAPSHOT_OBJ => StoreType.Object, - _ => throw new Exception($"ToStoreType: unexpected state {type}") - }; - } + STORE_SNAPSHOT_OBJ = 6, } } \ No newline at end of file diff --git a/libs/cluster/Server/Replication/CheckpointStore.cs b/libs/cluster/Server/Replication/CheckpointStore.cs index e3d8cc44045..f5db2cc884d 100644 --- a/libs/cluster/Server/Replication/CheckpointStore.cs +++ b/libs/cluster/Server/Replication/CheckpointStore.cs @@ -7,7 +7,6 @@ using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.cluster { @@ -40,19 +39,14 @@ public void Initialize() { head = tail = GetLatestCheckpointEntryFromDisk(); - if (tail.metadata.storeVersion == -1 && tail.metadata.objectStoreVersion == -1) + if (tail.metadata.storeVersion == -1) { head = tail = null; } else { - clusterProvider.storeWrapper.StoreCheckpointManager.RecoveredSafeAofAddress = tail.metadata.storeCheckpointCoveredAofAddress; + clusterProvider.storeWrapper.StoreCheckpointManager.SetRecoveredSafeAofAddress(ref tail.metadata.storeCheckpointCoveredAofAddress); clusterProvider.storeWrapper.StoreCheckpointManager.RecoveredHistoryId = tail.metadata.storePrimaryReplId; - if (!storeWrapper.serverOptions.DisableObjects) - { - clusterProvider.storeWrapper.ObjectStoreCheckpointManager.RecoveredSafeAofAddress = tail.metadata.storeCheckpointCoveredAofAddress; - clusterProvider.storeWrapper.ObjectStoreCheckpointManager.RecoveredHistoryId = tail.metadata.storePrimaryReplId; - } } // This purge does not check for active readers @@ -85,13 +79,11 @@ public void PurgeAllCheckpointsExceptEntry(CheckpointEntry entry = null) entry ??= GetLatestCheckpointEntryFromDisk(); if (entry == null) return; logger?.LogCheckpointEntry(LogLevel.Trace, nameof(PurgeAllCheckpointsExceptEntry), entry); - PurgeAllCheckpointsExceptTokens(StoreType.Main, entry.metadata.storeHlogToken, entry.metadata.storeIndexToken); - if (!clusterProvider.serverOptions.DisableObjects) - PurgeAllCheckpointsExceptTokens(StoreType.Object, entry.metadata.objectStoreHlogToken, entry.metadata.objectStoreIndexToken); + PurgeAllCheckpointsExceptTokens(entry.metadata.storeHlogToken, entry.metadata.storeIndexToken); - void PurgeAllCheckpointsExceptTokens(StoreType storeType, Guid logToken, Guid indexToken) + void PurgeAllCheckpointsExceptTokens(Guid logToken, Guid indexToken) { - var ckptManager = clusterProvider.GetReplicationLogCheckpointManager(storeType); + var ckptManager = clusterProvider.ReplicationLogCheckpointManager; // Delete log checkpoints foreach (var toDeletelogToken in ckptManager.GetLogCheckpointTokens()) @@ -129,7 +121,6 @@ public void AddCheckpointEntry(CheckpointEntry entry, bool fullCheckpoint = fals { var lastEntry = tail ?? throw new GarnetException($"Checkpoint history unavailable, need full checkpoint for {entry}"); entry.metadata.storeIndexToken = lastEntry.metadata.storeIndexToken; - entry.metadata.objectStoreIndexToken = lastEntry.metadata.objectStoreIndexToken; } _ = ValidateCheckpointEntry(entry); @@ -156,9 +147,6 @@ bool ValidateCheckpointEntry(CheckpointEntry entry) if (!clusterProvider.replicationManager.TryAcquireSettledMetadataForMainStore(entry, out _, out _)) throw new GarnetException("Failed to validate main store metadata at insertion"); - if (!clusterProvider.serverOptions.DisableObjects && !clusterProvider.replicationManager.TryAcquireSettledMetadataForObjectStore(entry, out _, out _)) - throw new GarnetException("Failed to validate object store metadata at insertion"); - return true; } catch (Exception ex) @@ -191,22 +179,11 @@ private void DeleteOutdatedCheckpoints() // Below check each checkpoint token separately if it is eligible for deletion if (!CanDeleteToken(curr, CheckpointFileType.STORE_HLOG)) break; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main).DeleteLogCheckpoint(curr.metadata.storeHlogToken); + clusterProvider.ReplicationLogCheckpointManager.DeleteLogCheckpoint(curr.metadata.storeHlogToken); if (!CanDeleteToken(curr, CheckpointFileType.STORE_INDEX)) break; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main).DeleteIndexCheckpoint(curr.metadata.storeIndexToken); - - if (!clusterProvider.serverOptions.DisableObjects) - { - if (!CanDeleteToken(curr, CheckpointFileType.OBJ_STORE_HLOG)) - break; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object).DeleteLogCheckpoint(curr.metadata.objectStoreHlogToken); - - if (!CanDeleteToken(curr, CheckpointFileType.OBJ_STORE_INDEX)) - break; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object).DeleteIndexCheckpoint(curr.metadata.objectStoreIndexToken); - } + clusterProvider.ReplicationLogCheckpointManager.DeleteIndexCheckpoint(curr.metadata.storeIndexToken); logger?.LogCheckpointEntry(LogLevel.Warning, "Deleting outdated checkpoint", curr); @@ -216,6 +193,22 @@ private void DeleteOutdatedCheckpoints() curr = next; } + // Safely truncate hlog segments up to the oldest active checkpoint's begin address. + // This is safe because curr is the oldest entry still referenced by active readers. + if (curr != null) + { + try + { + var hlogSize = storeWrapper.store.GetLogFileSize(curr.metadata.storeHlogToken); + storeWrapper.store.Log.ShiftBeginAddress(hlogSize.hybridLogFileStartAddress, truncateLog: true); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Skipping hlog truncation for checkpoint {token} because checkpoint metadata could not be read", + curr.metadata.storeHlogToken); + } + } + // Update head after delete head = curr; logger?.LogCheckpointEntry(LogLevel.Trace, "Current Head", head); @@ -257,13 +250,9 @@ public bool TryGetLatestCheckpointEntryFromMemory(out CheckpointEntry cEntry) var _tail = tail; if (_tail == null) { - cEntry = new CheckpointEntry() + cEntry = new CheckpointEntry { - metadata = new() - { - storeCheckpointCoveredAofAddress = 0, - objectCheckpointCoveredAofAddress = clusterProvider.serverOptions.DisableObjects ? long.MaxValue : 0 - } + metadata = new(storeWrapper.serverOptions.AofPhysicalSublogCount) }; _ = cEntry.TryAddReader(); return true; @@ -282,52 +271,30 @@ public bool TryGetLatestCheckpointEntryFromMemory(out CheckpointEntry cEntry) /// public CheckpointEntry GetLatestCheckpointEntryFromDisk() { - Guid objectStoreHLogToken = default; - Guid objectStoreIndexToken = default; - var objectStoreVersion = -1L; + var storeCheckpointCoveredAofAddress = server.AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, 0); storeWrapper.store.GetLatestCheckpointTokens(out var storeHLogToken, out var storeIndexToken, out var storeVersion); - storeWrapper.objectStore?.GetLatestCheckpointTokens(out objectStoreHLogToken, out objectStoreIndexToken, out objectStoreVersion); - var (storeCheckpointCoveredAofAddress, storePrimaryReplId) = GetCheckpointCookieMetadata(StoreType.Main, storeHLogToken); - var (objectCheckpointCoveredAofAddress, objectStorePrimaryReplId) = objectStoreHLogToken == default ? (long.MaxValue, null) : GetCheckpointCookieMetadata(StoreType.Object, objectStoreHLogToken); + GetCheckpointCookieMetadata(StoreType.Main, storeHLogToken, ref storeCheckpointCoveredAofAddress, out var storePrimaryReplId); CheckpointEntry entry = new() { - metadata = new() + metadata = new(storeWrapper.serverOptions.AofPhysicalSublogCount) { storeVersion = storeVersion, storeHlogToken = storeHLogToken, storeIndexToken = storeIndexToken, storeCheckpointCoveredAofAddress = storeCheckpointCoveredAofAddress, storePrimaryReplId = storePrimaryReplId, - - objectStoreVersion = objectStoreVersion, - objectStoreHlogToken = objectStoreHLogToken, - objectStoreIndexToken = objectStoreIndexToken, - objectCheckpointCoveredAofAddress = objectCheckpointCoveredAofAddress, - objectStorePrimaryReplId = objectStorePrimaryReplId, } }; return entry; - (long RecoveredSafeAofAddress, string RecoveredReplicationId) GetCheckpointCookieMetadata(StoreType storeType, Guid fileToken) + void GetCheckpointCookieMetadata(StoreType storeType, Guid fileToken, ref AofAddress recoveredSafeAofAddress, out string RecoveredReplicationId) { - if (fileToken == default) return (0, null); - var ckptManager = clusterProvider.GetReplicationLogCheckpointManager(storeType); - var pageSizeBits = storeType == StoreType.Main ? clusterProvider.serverOptions.PageSizeBits() : clusterProvider.serverOptions.ObjectStorePageSizeBits(); - using (var deltaFileDevice = ckptManager.GetDeltaLogDevice(fileToken)) - { - if (deltaFileDevice is not null) - { - deltaFileDevice.Initialize(-1); - if (deltaFileDevice.GetFileSize(0) > 0) - { - var deltaLog = new DeltaLog(deltaFileDevice, pageSizeBits, -1); - deltaLog.InitializeForReads(); - return ckptManager.GetCheckpointCookieMetadata(fileToken, deltaLog, true, -1); - } - } - } - return ckptManager.GetCheckpointCookieMetadata(fileToken, null, false, -1); + RecoveredReplicationId = null; + if (fileToken == default) return; + var ckptManager = clusterProvider.ReplicationLogCheckpointManager; + + ckptManager.GetCheckpointCookieMetadata(fileToken, ref recoveredSafeAofAddress, out RecoveredReplicationId); } } diff --git a/libs/cluster/Server/Replication/GarnetClusterCheckpointManager.cs b/libs/cluster/Server/Replication/GarnetClusterCheckpointManager.cs index 3bf7ec5b5fe..ec270b4237c 100644 --- a/libs/cluster/Server/Replication/GarnetClusterCheckpointManager.cs +++ b/libs/cluster/Server/Replication/GarnetClusterCheckpointManager.cs @@ -2,10 +2,8 @@ // Licensed under the MIT license. using System; -using System.Diagnostics; using System.IO; using System.Text; -using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -26,19 +24,25 @@ internal sealed class GarnetClusterCheckpointManager : GarnetCheckpointManager, readonly ILogger logger; public GarnetClusterCheckpointManager( + int aofPhysicalSublogCount, INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, bool safelyRemoveOutdated = false, int fastCommitThrottleFreq = 0, ILogger logger = null) - : base(deviceFactoryCreator, checkpointNamingScheme, removeOutdated: false, fastCommitThrottleFreq, logger) + : base(aofPhysicalSublogCount, deviceFactoryCreator, checkpointNamingScheme, removeOutdated: false, fastCommitThrottleFreq, logger) { this.isMainStore = isMainStore; this.safelyRemoveOutdated = safelyRemoveOutdated; this.logger = logger; } + /// + /// Cluster mode manages checkpoint cleanup externally via CheckpointStore with reader-safety checks. + /// + public override bool PerformAutomaticCleanup => false; + public override void CheckpointVersionShiftStart(long oldVersion, long newVersion, bool isStreaming) => checkpointVersionShiftStart?.Invoke(isMainStore, oldVersion, newVersion, isStreaming); @@ -55,13 +59,9 @@ public IDevice GetDevice(CheckpointFileType retStateType, Guid fileToken) { var device = retStateType switch { - CheckpointFileType.STORE_DLOG => GetDeltaLogDevice(fileToken), CheckpointFileType.STORE_INDEX => GetIndexDevice(fileToken), CheckpointFileType.STORE_SNAPSHOT => GetSnapshotLogDevice(fileToken), - CheckpointFileType.OBJ_STORE_DLOG => GetDeltaLogDevice(fileToken), - CheckpointFileType.OBJ_STORE_INDEX => GetIndexDevice(fileToken), - CheckpointFileType.OBJ_STORE_SNAPSHOT => GetSnapshotLogDevice(fileToken), - CheckpointFileType.OBJ_STORE_SNAPSHOT_OBJ => GetSnapshotObjectLogDevice(fileToken), + CheckpointFileType.STORE_SNAPSHOT_OBJ => GetSnapshotObjectLogDevice(fileToken), _ => throw new Exception($"RetrieveCheckpointFile: unexpected state{retStateType}") }; return device; @@ -71,80 +71,20 @@ public IDevice GetDevice(CheckpointFileType retStateType, Guid fileToken) private HybridLogRecoveryInfo ConvertMetadata(byte[] checkpointMetadata) { - // NOTE: this conversion should be simplified after suspending support for the old format which assumed the cookie is stored in the prefix. - var success = true; HybridLogRecoveryInfo recoveryInfo = new(); // Try to parse new format where cookie is embedded inside the HybridLogRecoveryInfo try { - using (StreamReader s = new(new MemoryStream(checkpointMetadata))) - { - recoveryInfo.Initialize(s); - } + using var s = new StreamReader(new MemoryStream(checkpointMetadata)); + recoveryInfo.Initialize(s); } catch (Exception ex) { logger?.LogError(ex, "Best effort read of checkpoint metadata failed"); - success = false; + throw; } - if (!success) - { - // If failed to parse above cookie is at prefix - // so extract it and convert it to new format - // NOTE: this needs to be deprecated at some point after 1.0.61 because conversion will not be necessary. - var metadataWithoutCookie = ExtractCookie(checkpointMetadata); - try - { - using (StreamReader s = new(new MemoryStream(metadataWithoutCookie))) - { - recoveryInfo.Initialize(s); - } - - var cookieSize = checkpointMetadata.Length - metadataWithoutCookie.Length; - var cookie = new byte[cookieSize]; - Array.Copy(checkpointMetadata, cookie, cookieSize); - recoveryInfo.cookie = cookie; - } - catch (Exception ex) - { - logger?.LogError(ex, "Old format checkpoint metadata failed"); - throw ex.InnerException; - } - - byte[] ExtractCookie(byte[] commitMetadataWithCookie) - { - var cookieTotalSize = GetCookieData(commitMetadataWithCookie, out var recoveredSafeAofAddress, out var recoveredReplicationId); - RecoveredSafeAofAddress = recoveredSafeAofAddress; - RecoveredHistoryId = recoveredReplicationId; - var payloadSize = commitMetadataWithCookie.Length - cookieTotalSize; - - var commitMetadata = new byte[payloadSize]; - Array.Copy(commitMetadataWithCookie, cookieTotalSize, commitMetadata, 0, payloadSize); - return commitMetadata; - - unsafe int GetCookieData(byte[] commitMetadataWithCookie, out long checkpointCoveredAddress, out string primaryReplId) - { - checkpointCoveredAddress = -1; - primaryReplId = null; - var size = sizeof(int); - fixed (byte* ptr = commitMetadataWithCookie) - { - if (commitMetadataWithCookie.Length < 4) throw new Exception($"invalid metadata length: {commitMetadataWithCookie.Length} < 4"); - var cookieSize = *(int*)ptr; - size += cookieSize; - - if (commitMetadataWithCookie.Length < 12) throw new Exception($"invalid metadata length: {commitMetadataWithCookie.Length} < 12"); - checkpointCoveredAddress = *(long*)(ptr + 4); - - if (commitMetadataWithCookie.Length < 52) throw new Exception($"invalid metadata length: {commitMetadataWithCookie.Length} < 52"); - primaryReplId = Encoding.ASCII.GetString(ptr + 12, 40); - } - return size; - } - } - } return recoveryInfo; } @@ -163,24 +103,18 @@ public void CommitLogCheckpointSendFromPrimary(Guid logToken, byte[] checkpointM /// Retrieve RecoveredSafeAofAddress and RecoveredReplicationId for checkpoint /// /// - /// - /// - /// - /// + /// + /// /// - public unsafe (long RecoveredSafeAofAddress, string RecoveredReplicationId) GetCheckpointCookieMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta, long recoverTo) + public unsafe void GetCheckpointCookieMetadata(Guid logToken, ref AofAddress recoveredSafeAofAddress, out string recoveredReplicationId) { - var metadata = GetLogCheckpointMetadata(logToken, deltaLog, scanDelta, recoverTo); + var metadata = GetLogCheckpointMetadata(logToken); var hlri = ConvertMetadata(metadata); - var bytesRead = GetCookieData(hlri, out var RecoveredSafeAofAddress, out var RecoveredReplicationId); - Debug.Assert(bytesRead == 52); - return (RecoveredSafeAofAddress, RecoveredReplicationId); - static unsafe int GetCookieData(HybridLogRecoveryInfo hlri, out long checkpointCoveredAddress, out string primaryReplId) + recoveredReplicationId = null; + if (RecoveredSafeAofAddress.Length == 1) { - checkpointCoveredAddress = -1; - primaryReplId = null; - + // Legacy single log deserialization for backward compatibility var bytesRead = sizeof(int); fixed (byte* ptr = hlri.cookie) { @@ -189,48 +123,27 @@ static unsafe int GetCookieData(HybridLogRecoveryInfo hlri, out long checkpointC bytesRead += cookieSize; if (hlri.cookie.Length < 12) throw new Exception($"invalid metadata length: {hlri.cookie.Length} < 12"); - checkpointCoveredAddress = *(long*)(ptr + 4); + recoveredSafeAofAddress[0] = *(long*)(ptr + 4); if (hlri.cookie.Length < 52) throw new Exception($"invalid metadata length: {hlri.cookie.Length} < 52"); - primaryReplId = Encoding.ASCII.GetString(ptr + 12, 40); + recoveredReplicationId = Encoding.ASCII.GetString(ptr + 12, 40); } - return bytesRead; + } + else + { + // Multi-log cookie + using var ms = new MemoryStream(hlri.cookie); + using var reader = new BinaryReader(ms, Encoding.ASCII); + recoveredReplicationId = reader.ReadInt32() > 0 ? reader.ReadString() : null; + recoveredSafeAofAddress = AofAddress.Deserialize(reader); + reader.Dispose(); + ms.Dispose(); } } - public override byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta, long recoverTo) + public override byte[] GetLogCheckpointMetadata(Guid logToken) { - byte[] metadata = null; HybridLogRecoveryInfo hlri; - if (deltaLog != null && scanDelta) - { - // Try to get latest valid metadata from delta-log - deltaLog.Reset(); - while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out var type)) - { - switch (type) - { - case DeltaLogEntryType.DELTA: - // consider only metadata records - continue; - case DeltaLogEntryType.CHECKPOINT_METADATA: - metadata = new byte[entryLength]; - unsafe - { - fixed (byte* m = metadata) - Buffer.MemoryCopy((void*)physicalAddress, m, entryLength, entryLength); - } - hlri = ConvertMetadata(metadata); - if (hlri.version == recoverTo || hlri.version < recoverTo && hlri.nextVersion > recoverTo) goto LoopEnd; - continue; - default: - throw new GarnetException("Unexpected entry type"); - } - LoopEnd: - break; - } - if (metadata != null) return metadata; - } var device = deviceFactory.Get(checkpointNamingScheme.LogCheckpointMetadata(logToken)); diff --git a/libs/cluster/Server/Replication/IOCallbackContext.cs b/libs/cluster/Server/Replication/IOCallbackContext.cs new file mode 100644 index 00000000000..7a98c035fec --- /dev/null +++ b/libs/cluster/Server/Replication/IOCallbackContext.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Roots the buffer so the pinned byte[] is not collected while IO is in-flight. + /// If the caller abandons the wait (timeout/cancellation), the buffer is intentionally + /// not returned to the pool — the GC will collect it after the IO completes and + /// the callback releases this context. + /// + internal sealed class IOCallbackContext + { + public SectorAlignedMemory Buffer; + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriver.cs b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriver.cs new file mode 100644 index 00000000000..2d4650dc321 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriver.cs @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Garnet.common; +using Garnet.server; +using Microsoft.Extensions.Logging; + +namespace Garnet.cluster +{ + internal sealed partial class AofSyncDriver : IDisposable + { + readonly ClusterProvider clusterProvider; + readonly AofSyncDriverStore aofSyncDriverStore; + readonly string localNodeId; + readonly string remoteNodeId; + readonly ILogger logger; + readonly CancellationTokenSource cts; + + readonly AofSyncTask[] aofSyncTasks; + + /// + /// Check if client connection is healthy + /// + public bool IsConnected + => aofSyncTasks.Select(x => x.IsConnected ? 1 : 0).Sum() == aofSyncTasks.Length; + + /// + /// Node-id associated with this AofSyncTask + /// + public string RemoteNodeId => remoteNodeId; + + /// + /// Active worker monitor for AofSyncDriver tasks + /// + readonly ActiveWorkerMonitor activeWorkerMonitor = new(); + + /// + /// Return start address for underlying AofSyncTask + /// + public AofAddress StartAddress + { + get + { + var startAddress = AofAddress.Create(aofSyncTasks.Length, 0); + for (var i = 0; i < aofSyncTasks.Length; i++) + startAddress[i] = aofSyncTasks[i].StartAddress; + return startAddress; + } + } + + /// + /// Return previous address for underlying AofSyncTask + /// + public AofAddress PreviousAddress + { + get + { + var previousAddress = AofAddress.Create(aofSyncTasks.Length, 0); + for (var i = 0; i < aofSyncTasks.Length; i++) + previousAddress[i] = aofSyncTasks[i].PreviousAddress; + return previousAddress; + } + } + + /// + /// Return previous address for a specific sublog without copying the full AofAddress struct + /// + /// Index of the physical sublog. + /// The previous address of the specified sublog's sync task. + public long GetPreviousAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].PreviousAddress; + + /// + /// Return start address for a specific sublog without copying the full AofAddress struct + /// + /// Index of the physical sublog. + /// The start address of the specified sublog's sync task. + public long GetStartAddress(int physicalSublogIdx) => aofSyncTasks[physicalSublogIdx].StartAddress; + + /// + /// Replica endpoint + /// + readonly IPEndPoint endPoint; + + public AofSyncDriver( + ClusterProvider clusterProvider, + AofSyncDriverStore aofSyncDriverStore, + string localNodeId, + string remoteNodeId, + IPEndPoint endPoint, + ref AofAddress startAddress, + ILogger logger) + { + this.clusterProvider = clusterProvider; + this.aofSyncDriverStore = aofSyncDriverStore; + this.localNodeId = localNodeId; + this.remoteNodeId = remoteNodeId; + this.endPoint = endPoint; + cts = new(); + this.logger = logger; + + aofSyncTasks = new AofSyncTask[clusterProvider.serverOptions.AofPhysicalSublogCount]; + for (var physicalSublogIdx = 0; physicalSublogIdx < aofSyncTasks.Length; physicalSublogIdx++) + aofSyncTasks[physicalSublogIdx] = new AofSyncTask(clusterProvider, physicalSublogIdx, endPoint, startAddress[physicalSublogIdx], localNodeId, remoteNodeId, cts, logger); + } + + /// + /// Dispose AofSyncDriver + /// + public void Dispose() + { + // Cancel cts + cts?.Cancel(); + + // Dispose sync tasks + foreach (var aofSyncTask in aofSyncTasks) + aofSyncTask?.Dispose(); + + // Wait for tasks to exit + activeWorkerMonitor.Dispose(); + + // Finally, dispose the cts + cts?.Dispose(); + } + + /// + /// Dispose alls clients associated with this aof sync driver + /// + public void DisposeClient() + { + foreach (var aofSyncTask in aofSyncTasks) + aofSyncTask.garnetClient?.Dispose(); + } + + /// + /// Main replica aof sync task. + /// + public async Task RunAsync() + { + logger?.LogInformation("Starting ReplicationManager.ReplicaSyncTask for remote node {remoteNodeId} starting from address {address}", remoteNodeId, StartAddress); + + try + { + if (clusterProvider.serverOptions.AofPhysicalSublogCount == 1) + { + await aofSyncTasks[0].RunAofSyncTaskAsync(this).ConfigureAwait(false); + } + else + { + var tasks = new Task[aofSyncTasks.Length + 1]; + tasks[0] = AdvancePhysicalSublogTimeAsync(); + for (var i = 0; i < aofSyncTasks.Length; i++) + tasks[i + 1] = aofSyncTasks[i].RunAofSyncTaskAsync(this); + + _ = await Task.WhenAny(tasks).ConfigureAwait(false); + } + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.ReplicaSyncTask - terminating"); + } + finally + { + var (address, port) = clusterProvider.clusterManager.CurrentConfig.GetWorkerAddressFromNodeId(remoteNodeId); + logger?.LogWarning("AofSyncDriver terminated for remote node replicaId:{remoteNodeId} [{address}:{port}] startAddress: {startAddress}, previousAddress:{previousAddress}", remoteNodeId, address, port, StartAddress, PreviousAddress); + + if (!aofSyncDriverStore.TryRemove(this)) + logger?.LogError("Unable to remove {remoteNodeId} from aofTaskStore at end of ReplicaSyncTask", remoteNodeId); + } + } + + /// + /// Advance physical sublog time background task. + /// + /// + /// + /// + async Task AdvancePhysicalSublogTimeAsync() + { + var enteredMonitor = false; + var client = new GarnetClientSession( + endPoint, + clusterProvider.replicationManager.GetAofSyncNetworkBufferSettings, + clusterProvider.replicationManager.GetNetworkPool, + tlsOptions: this.clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, + authUsername: this.clusterProvider.ClusterUsername, + authPassword: this.clusterProvider.ClusterPassword, + logger: logger); + + try + { + enteredMonitor = activeWorkerMonitor.TryEnter(); + if (!enteredMonitor) + throw new GarnetException($"Failed to acquire read lock at {nameof(AdvancePhysicalSublogTimeAsync)}"); + + // Connect to replica + await client.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, cts.Token).ConfigureAwait(false); + + var appendOnlyFile = clusterProvider.storeWrapper.appendOnlyFile; + var previousTailAddress = AofAddress.Create(appendOnlyFile.Log.Size, 0); + + while (!cts.IsCancellationRequested) + { + await Task.Delay(clusterProvider.serverOptions.AofTailWitnessFreqMs, cts.Token).ConfigureAwait(false); + var currentTailAddress = appendOnlyFile.Log.TailAddress; + var newWrites = previousTailAddress.AnyLesser(currentTailAddress); + + if (newWrites) + { + var sequenceNumber = appendOnlyFile.GetLargerThanMaximumSequenceNumber(); + _ = await client.ExecuteClusterAdvanceTime(sequenceNumber, currentTailAddress.Span). + WaitAsync(clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token). + ConfigureAwait(false); + previousTailAddress.MonotonicUpdate(ref currentTailAddress); + } + } + } + finally + { + if (enteredMonitor) + _ = activeWorkerMonitor.Exit(); + client?.Dispose(); + } + } + + #region DisklesSyncInterface + public async Task ConnectClientsAsync() + { + if (!IsConnected) + foreach (var aofSyncTask in aofSyncTasks) + await aofSyncTask.garnetClient.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, cts.Token).ConfigureAwait(false); + } + + public Task IssuesFlushAllAsync() + => aofSyncTasks[0].garnetClient.ExecuteAsync(["CLUSTER", "FLUSHALL"]); + + public void InitializeIterationBuffer() + => aofSyncTasks[0].garnetClient.InitializeIterationBuffer(clusterProvider.storeWrapper.loggingFrequency); + + public void InitializeIfNeeded() + { + if (aofSyncTasks[0].garnetClient.NeedsInitialization) + aofSyncTasks[0].garnetClient.SetClusterSyncHeader(clusterProvider.clusterManager.CurrentConfig.LocalNodeId); + } + + public Task ExecuteAttachSyncAsync(SyncMetadata syncMetadata) + => aofSyncTasks[0].garnetClient.ExecuteClusterAttachSync(syncMetadata.ToByteArray()); + + public bool TryWriteRecordSpan(ReadOnlySpan recordSpan, MigrationRecordSpanType type, out Task task) + => aofSyncTasks[0].garnetClient.TryWriteRecordSpan(recordSpan, type, out task); + + public Task SendAndResetIterationBufferAsync() + => aofSyncTasks[0].garnetClient.SendAndResetIterationBuffer(); + #endregion + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriverStore.cs b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriverStore.cs new file mode 100644 index 00000000000..8fe8778113a --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncDriverStore.cs @@ -0,0 +1,579 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net; +using Garnet.common; +using Garnet.server; +using Microsoft.Extensions.Logging; + +namespace Garnet.cluster +{ + /// + /// Storage provider for AOF tasks + /// + internal sealed class AofSyncDriverStore : IDisposable + { + internal sealed class LogShiftTailCallback(int physicalSublogIdx, AofSyncDriverStore store) + { + readonly int physicalSublogIdx = physicalSublogIdx; + readonly AofSyncDriverStore store = store; + + internal void SafeTailPageShiftCallback(long oldTailAddress, long newTailAddress) + { + // Truncate 2 pages above ReadOnly mark, so that we have sufficient time to shift begin before we flush. + // Make sure this is page-aligned, in case we go to a non-page-aligned ReadOnlyAddress. + var truncateUntilAddress = store.clusterProvider.storeWrapper.appendOnlyFile.Log.UnsafeGetReadOnlyAddressAbove(physicalSublogIdx, newTailAddress, numPagesAbove: 2); + if (truncateUntilAddress > 0) + _ = store.SafeTruncateAof(truncateUntilAddress, physicalSublogIdx); + } + } + + readonly ClusterProvider clusterProvider; + readonly ILogger logger; + + AofSyncDriver[] syncDrivers; + int numDrivers; + SingleWriterMultiReaderLock _lock; + bool _disposed; + internal AofAddress TruncatedUntil; + + public int AofSyncDriverCount => numDrivers; + + public AofSyncDriverStore(ClusterProvider clusterProvider, int initialSize = 1, ILogger logger = null) + { + this.clusterProvider = clusterProvider; + this.logger = logger; + syncDrivers = new AofSyncDriver[initialSize]; + numDrivers = 0; + if (clusterProvider.storeWrapper.appendOnlyFile != null) + { + if (clusterProvider.serverOptions.FastAofTruncate) + { + for (var i = 0; i < clusterProvider.serverOptions.AofPhysicalSublogCount; i++) + { + var logShiftTailCallback = new LogShiftTailCallback(i, this); + clusterProvider.storeWrapper.appendOnlyFile.Log.SetLogShiftTailCallback(i, logShiftTailCallback.SafeTailPageShiftCallback); + } + } + } + TruncatedUntil = AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, 0); + } + + /// + /// Safely truncate AOF sublog + /// + /// + /// + /// + long SafeTruncateAof(long truncateUntil, int physicalSublogIdx) + { + _lock.WriteLock(); + + if (_disposed) + { + _lock.WriteUnlock(); + return -1; + } + + // Calculate min address of all iterators + var TruncatedUntil = truncateUntil; + try + { + for (var i = 0; i < numDrivers; i++) + { + Debug.Assert(syncDrivers[i] != null, $"syncDriver cannot be null at {nameof(SafeTruncateAof)}"); + var prevAddress = syncDrivers[i].GetPreviousAddress(physicalSublogIdx); + if (prevAddress < TruncatedUntil) + TruncatedUntil = prevAddress; + } + + // Inform that we have logically truncatedUntil + this.TruncatedUntil.MonotonicUpdate(TruncatedUntil, physicalSublogIdx); + } + finally + { + // Release lock early + _lock.WriteUnlock(); + } + + if (clusterProvider.serverOptions.FastAofTruncate) + { + clusterProvider.storeWrapper.appendOnlyFile?.Log.UnsafeShiftBeginAddress(physicalSublogIdx, TruncatedUntil, snapToPageStart: true, truncateLog: true); + } + else + { + clusterProvider.storeWrapper.appendOnlyFile?.Log.TruncateUntil(physicalSublogIdx, TruncatedUntil); + clusterProvider.storeWrapper.appendOnlyFile?.Log.Commit(); + } + + return TruncatedUntil; + } + + /// + /// Safely truncate AOF until provided address by checking against active AofSyncDrivers + /// + /// + public void SafeTruncateAof(in AofAddress truncateUntil) + { + _lock.WriteLock(); + + if (_disposed) + { + _lock.WriteUnlock(); + return; + } + + // Calculate min address of all iterators + var TruncatedUntil = truncateUntil; + try + { + for (var i = 0; i < numDrivers; i++) + { + Debug.Assert(syncDrivers[i] != null, $"syncDriver cannot be null {nameof(SafeTruncateAof)}"); + var previousAddress = syncDrivers[i].PreviousAddress; + for (var physicalSublogIdx = 0; physicalSublogIdx < previousAddress.Length; physicalSublogIdx++) + { + if (previousAddress[physicalSublogIdx] < TruncatedUntil[physicalSublogIdx]) + TruncatedUntil[physicalSublogIdx] = previousAddress[physicalSublogIdx]; + } + } + // Inform that we have logically truncatedUntil + this.TruncatedUntil.MonotonicUpdate(ref TruncatedUntil); + } + finally + { + // Release lock early + _lock.WriteUnlock(); + } + + if (clusterProvider.serverOptions.FastAofTruncate) + { + clusterProvider.storeWrapper.appendOnlyFile?.Log.UnsafeShiftBeginAddress(TruncatedUntil, snapToPageStart: true, truncateLog: true); + } + else + { + clusterProvider.storeWrapper.appendOnlyFile?.Log.TruncateUntil(TruncatedUntil); + clusterProvider.storeWrapper.appendOnlyFile?.Log.Commit(); + } + } + + /// + /// Get relevant information for all attached replicas + /// + /// + /// + public List GetReplicaInfo(in AofAddress PrimaryReplicationOffset) + { + // secondary0: ip=127.0.0.1,port=7001,state=online,offset=56,lag=0 + List replicaInfo = new(numDrivers); + + _lock.ReadLock(); + var current = clusterProvider.clusterManager.CurrentConfig; + try + { + if (_disposed) return replicaInfo; + + for (var i = 0; i < numDrivers; ++i) + { + var cr = syncDrivers[i]; + var (address, port) = current.GetWorkerAddressFromNodeId(cr.RemoteNodeId); + replicaInfo.Add(new() + { + address = address, + port = port, + replication_state = cr.IsConnected ? "online" : "offline", + replication_offset = cr.PreviousAddress, + replication_lag = cr.PreviousAddress.Diff(PrimaryReplicationOffset) + }); + } + } + finally + { + _lock.ReadUnlock(); + } + return replicaInfo; + } + + /// + /// Dispose the AofSyncDriverStore + /// + public void Dispose() + { + try + { + _lock.WriteLock(); + if (_disposed) return; + _disposed = true; + } + finally + { + _lock.WriteUnlock(); + } + + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + syncDrivers[i] = null; + try + { + syncDriver?.Dispose(); + } + catch { } + } + numDrivers = 0; + Array.Clear(syncDrivers); + } + + /// + /// Add a new AofSyncDriver + /// + /// + /// + /// + /// + /// + public bool TryAddReplicationDriver(string remoteNodeId, ref AofAddress startAddress, out AofSyncDriver aofSyncDriver) + { + aofSyncDriver = null; + + startAddress.SetValueIf(ReplicationManager.kFirstValidAofAddress, 0); + var success = false; + var current = clusterProvider.clusterManager.CurrentConfig; + var (address, port) = current.GetWorkerAddressFromNodeId(remoteNodeId); + + // If address is null or port is not valid, we cannot create a task + if (address == null || port <= 0 || ExceptionInjectionHelper.TriggerCondition(ExceptionInjectionType.Replication_Failed_To_AddAofSyncTask_UnknownNode)) + throw new GarnetException($"Failed to create AOF sync task for {remoteNodeId} with address {address} and port {port}"); + + // Create AofSyncTask + try + { + aofSyncDriver = new AofSyncDriver( + clusterProvider, + this, + current.LocalNodeId, + remoteNodeId, + new IPEndPoint(IPAddress.Parse(address), port), + ref startAddress, + logger); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An error occurred at TryAddReplicationTask task creation for {remoteNodeId}", remoteNodeId); + return false; + } + + Debug.Assert(aofSyncDriver != null, $"aofSyncTaskInfo should not be null {nameof(TryAddReplicationDriver)}"); + + // Lock to prevent add/remove tasks and truncate operations + _lock.WriteLock(); + try + { + if (_disposed) return success; + + // Fail adding the task if truncation has happened, and we are not in AllowDataLoss mode + if (startAddress.AnyLesser(TruncatedUntil) && !clusterProvider.AllowDataLoss) + { + logger?.LogWarning("AOF sync task for {remoteNodeId}, with start address {startAddress}, could not be added, local AOF is truncated until {truncatedUntil}", remoteNodeId, startAddress, TruncatedUntil); + return success; + } + + // Iterate array of existing tasks and update associated task if it already exists + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + Debug.Assert(syncDriver != null, "syncDriver should not be null"); + if (syncDriver.RemoteNodeId == remoteNodeId) + { + syncDrivers[i] = aofSyncDriver; + syncDriver.Dispose(); + success = true; + break; + } + } + + // If task did not exist we add it here + if (!success) + { + if (numDrivers == syncDrivers.Length) + { + var old_tasks = syncDrivers; + var _tasks = new AofSyncDriver[syncDrivers.Length * 2]; + Array.Copy(syncDrivers, _tasks, syncDrivers.Length); + syncDrivers = _tasks; + Array.Clear(old_tasks); + } + syncDrivers[numDrivers++] = aofSyncDriver; + success = true; + } + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An error occurred at TryAddReplicationTask task addition for {remoteNodeId}", remoteNodeId); + } + finally + { + _lock.WriteUnlock(); + if (!success) + { + aofSyncDriver?.Dispose(); + aofSyncDriver = null; + } + } + + return success; + } + + /// + /// Add AofSyncDrivers collection + /// + /// + /// + /// + /// + public bool TryAddReplicationDrivers(ReplicaSyncSession[] replicaSyncSessions, ref AofAddress startAddress) + { + var current = clusterProvider.clusterManager.CurrentConfig; + var success = true; + startAddress.SetValueIf(ReplicationManager.kFirstValidAofAddress, 0); + + // First iterate through all sync sessions and add an AOF sync task + // All tasks will be + foreach (var rss in replicaSyncSessions) + { + if (rss == null) continue; + var replicaNodeId = rss.replicaSyncMetadata.originNodeId; + var (address, port) = current.GetWorkerAddressFromNodeId(replicaNodeId); + + // If address is null or port is not valid, we cannot create a task + if (address == null || port <= 0) + throw new GarnetException($"Failed to create AOF sync task for {replicaNodeId} with address {address} and port {port}"); + + try + { + rss.AddAofSyncTask(new AofSyncDriver( + clusterProvider, + this, + current.LocalNodeId, + replicaNodeId, + new IPEndPoint(IPAddress.Parse(address), port), + ref startAddress, + logger)); + } + catch (Exception ex) + { + logger?.LogError(ex, "{method} creating AOF sync task for {replicaNodeId} failed", nameof(TryAddReplicationDrivers), replicaNodeId); + return false; + } + } + + _lock.WriteLock(); + try + { + if (_disposed) return false; + + // Fail adding the task if truncation has happened + if (startAddress.AnyLesser(TruncatedUntil) && !clusterProvider.AllowDataLoss) + { + logger?.LogError("{method} failed to add tasks for AOF sync {startAddress} {truncatedUntil}", nameof(TryAddReplicationDrivers), startAddress, TruncatedUntil); + return false; + } + + foreach (var rss in replicaSyncSessions) + { + if (rss == null) continue; + + var added = false; + // Find if AOF sync task already exists + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + Debug.Assert(syncDriver != null, $"syncDrive should not be null {nameof(TryAddReplicationDrivers)}"); + if (syncDriver.RemoteNodeId == rss.replicaSyncMetadata.originNodeId) + { + syncDrivers[i] = rss.AofSyncDriver; + syncDriver.Dispose(); + added = true; + break; + } + } + + if (added) continue; + + // If AOF sync task did not exist and was not added we added below + // Check if array can hold a new AOF sync task + if (numDrivers == syncDrivers.Length) + { + var old_tasks = syncDrivers; + var _tasks = new AofSyncDriver[syncDrivers.Length * 2]; + Array.Copy(syncDrivers, _tasks, syncDrivers.Length); + syncDrivers = _tasks; + Array.Clear(old_tasks); + } + // Add new AOF sync task + syncDrivers[numDrivers++] = rss.AofSyncDriver; + } + + success = true; + } + finally + { + _lock.WriteUnlock(); + + if (!success) + { + foreach (var rss in replicaSyncSessions) + { + if (rss == null) continue; + rss.AofSyncDriver?.Dispose(); + } + } + } + + return true; + } + + /// + /// Remove provided AofSyncDriver + /// + /// + /// + public bool TryRemove(AofSyncDriver aofSyncDriver) + { + // Lock addition of new tasks + _lock.WriteLock(); + + var success = false; + try + { + if (_disposed) return success; + + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + Debug.Assert(syncDriver != null, $"syncDriver should not be null at {nameof(TryRemove)}"); + if (syncDriver == aofSyncDriver) + { + syncDrivers[i] = null; + if (i < numDrivers - 1) + { + // Swap the last task into the free slot + syncDrivers[i] = syncDrivers[numDrivers - 1]; + syncDrivers[numDrivers - 1] = null; + } + // Reduce the number of tasks + numDrivers--; + + // Kill the task + try + { + syncDriver.Dispose(); + } + catch { } + success = true; + break; + } + } + } + finally + { + _lock.WriteUnlock(); + } + return success; + } + + /// + /// Count the number of AofSyncDriver connections + /// + /// + public int CountConnectedReplicas() + { + var count = 0; + _lock.ReadLock(); + try + { + if (_disposed) return 0; + + for (var i = 0; i < numDrivers; i++) + { + var t = syncDrivers[i]; + count += t.IsConnected ? 1 : 0; + } + } + finally + { + _lock.ReadUnlock(); + } + return count; + } + + /// + /// Update TruncatedUntil address + /// + /// + public void UpdateTruncatedUntil(AofAddress truncatedUntil) + { + try + { + _lock.WriteLock(); + TruncatedUntil.MonotonicUpdate(ref truncatedUntil); + } + finally + { + _lock.WriteUnlock(); + } + } + + /// + /// Remove and dispose all active aof sync drivers + /// + public void Reset() + { + try + { + _lock.WriteLock(); + if (_disposed) return; + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + syncDrivers[i] = null; + try + { + syncDriver?.Dispose(); + } + catch { } + } + numDrivers = 0; + Array.Clear(syncDrivers); + } + finally + { + _lock.WriteUnlock(); + } + } + + [Conditional("DEBUG")] + public void AssertDoesNotExist(string remoteNodeId) + { + _lock.ReadLock(); + try + { + if (_disposed) return; + + for (var i = 0; i < numDrivers; i++) + { + var syncDriver = syncDrivers[i]; + if (syncDriver.RemoteNodeId == remoteNodeId) + Debug.Fail($"syncDriver with {remoteNodeId} should not exist!"); + } + } + finally + { + _lock.ReadUnlock(); + } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncTask.cs b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncTask.cs new file mode 100644 index 00000000000..bc678f3869d --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/AofOperations/AofSyncTask.cs @@ -0,0 +1,225 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + internal sealed partial class AofSyncDriver : IDisposable + { + public class AofSyncTask : IBulkLogEntryConsumer, IDisposable + { + readonly ClusterProvider clusterProvider; + readonly int physicalSublogIdx; + public readonly GarnetClientSession garnetClient; + readonly string localNodeId; + readonly string remoteNodeId; + readonly CancellationTokenSource cts; + readonly long startAddress; + TsavoriteLogScanSingleIterator iter; + long previousAddress; + + /// + /// Return start address for this AofSyncTask + /// + public long StartAddress => startAddress; + + /// + /// Return previous address for this AofSyncTask + /// + public long PreviousAddress => previousAddress; + + /// + /// Check if client connection is healthy + /// + public bool IsConnected => garnetClient != null && garnetClient.IsConnected; + + /// + /// Logger instance + /// + readonly ILogger logger; + + /// + /// AofSyncTask constructor + /// + /// + /// + /// + /// + /// + /// + /// + /// + public AofSyncTask( + ClusterProvider clusterProvider, + int physicalSublogIdx, + IPEndPoint endPoint, + long startAddress, + string localNodeId, + string remoteNodeId, + CancellationTokenSource cts, + ILogger logger) + { + var currentConfig = clusterProvider.clusterManager.CurrentConfig; + this.clusterProvider = clusterProvider; + this.physicalSublogIdx = physicalSublogIdx; + this.startAddress = startAddress; + previousAddress = startAddress; + this.localNodeId = localNodeId; + this.remoteNodeId = remoteNodeId; + this.cts = cts; + garnetClient = new GarnetClientSession( + endPoint, + this.clusterProvider.replicationManager.GetAofSyncNetworkBufferSettings, + this.clusterProvider.replicationManager.GetNetworkPool, + tlsOptions: this.clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, + authUsername: this.clusterProvider.ClusterUsername, + authPassword: this.clusterProvider.ClusterPassword, + clientName: $"AofSyncTask-{physicalSublogIdx}:({currentConfig.LocalNodeEndpoint})", + logger: logger); + this.logger = logger; + } + + public void Dispose() + { + try + { + // Dispose GarnetClient + garnetClient?.Dispose(); + } + catch { } + + try + { + // This forces the background sync task to stop, + // unless the cancelled cts already signaled it to stop + iter?.Dispose(); + iter = null; + } + catch { } + } + + /// + /// Consume AOF records generated at the primary + /// + /// + /// + /// + /// + /// + public unsafe void Consume(byte* payloadPtr, int payloadLength, long currentAddress, long nextAddress, bool isProtected) + { + try + { + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Aof_Sync_Task_Consume); + + // logger?.LogInformation("Sending {payloadLength} bytes to {remoteNodeId} at address {currentAddress}-{nextAddress}", payloadLength, remoteNodeId, currentAddress, nextAddress); + + // This is called under epoch protection, so we have to wait for appending to complete + garnetClient.ExecuteClusterAppendLog( + localNodeId, + physicalSublogIdx, + previousAddress, + currentAddress, + nextAddress, + (long)payloadPtr, + payloadLength); + + // Set task address to nextAddress, as the iterator is currently at nextAddress + // (records at currentAddress are already sent above) + previousAddress = nextAddress; + } + catch (Exception ex) + { + logger?.LogError( + ex, + "{Consume}[{taskId}]: exception consuming AOF payload to sync {remoteNodeId} ({currenAddress}, {nextAddress})", + nameof(AofSyncTask.Consume), + physicalSublogIdx, + remoteNodeId, + currentAddress, + nextAddress); + throw; + } + } + + public void Throttle() + { + cts.Token.ThrowIfCancellationRequested(); + + if (!garnetClient.IsConnected) + ExceptionUtils.ThrowException(new GarnetException($"AOF stream client disconnected! [{physicalSublogIdx}]:({startAddress},{previousAddress})")); + + // Trigger flush while we are out of epoch protection + garnetClient.CompletePending(false); + garnetClient.Throttle(); + } + + public async Task RunAofSyncTaskAsync(AofSyncDriver aofSyncDriver) + { + var enteredMonitor = false; + try + { + enteredMonitor = aofSyncDriver.activeWorkerMonitor.TryEnter(); + if (!enteredMonitor) + ExceptionUtils.ThrowException(new GarnetException($"[{physicalSublogIdx}] Failed to acquire lock at {nameof(RunAofSyncTaskAsync)}")); + + logger?.LogInformation( + "{RunAofSyncTask}[{taskId}]: syncing {remoteNodeId} starting from address {address}", + nameof(AofSyncTask.RunAofSyncTaskAsync), + physicalSublogIdx, + remoteNodeId, + startAddress); + + if (!IsConnected) + await garnetClient.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, cts.Token).ConfigureAwait(false); + + LogRunAofSyncTask(physicalSublogIdx, startAddress, previousAddress, logger); + + iter = clusterProvider.storeWrapper.appendOnlyFile.Log.ScanSingle(physicalSublogIdx, startAddress, long.MaxValue, scanUncommitted: true, recover: false, logger: logger); + + // Send ping to initialize replication stream + var resp = await garnetClient.ExecuteClusterAppendLogInit(localNodeId, physicalSublogIdx, -1, -1, -1); + if (!resp.Equals("OK")) + throw new GarnetException("Failed to initialize AofSync stream!"); + + await iter.BulkConsumeAllAsync( + this, + clusterProvider.serverOptions.ReplicaSyncDelayMs, + maxChunkSize: 1 << 20, + cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + logger?.LogError(ex, "[{sublogIdx}]({method})", physicalSublogIdx, nameof(RunAofSyncTaskAsync)); + } + finally + { + if (enteredMonitor) + _ = aofSyncDriver.activeWorkerMonitor.Exit(); + garnetClient?.Dispose(); + } + + [Conditional("DEBUG")] + static void LogRunAofSyncTask(int physicalSublogIdx, long startAddress, long previousAddress, ILogger logger) + { + var state = new GarnetTestLoggingEvent() + { + Type = GarnetTestLoggingEventType.LogRunAofSyncTask, + Message = $"physicalSublogIdx:{physicalSublogIdx}, startAddress: {startAddress}, previousAddress: {previousAddress}", + }; + + logger?.LogTesting(state); + } + } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/AofSyncTaskInfo.cs b/libs/cluster/Server/Replication/PrimaryOps/AofSyncTaskInfo.cs deleted file mode 100644 index 6b60cf429d1..00000000000 --- a/libs/cluster/Server/Replication/PrimaryOps/AofSyncTaskInfo.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Garnet.client; -using Garnet.common; -using Microsoft.Extensions.Logging; -using Tsavorite.core; - -namespace Garnet.cluster -{ - internal sealed class AofSyncTaskInfo : IBulkLogEntryConsumer, IDisposable - { - readonly ClusterProvider clusterProvider; - readonly AofTaskStore aofTaskStore; - readonly string localNodeId; - public readonly string remoteNodeId; - readonly ILogger logger; - public readonly GarnetClientSession garnetClient; - readonly CancellationTokenSource cts; - TsavoriteLogScanSingleIterator iter; - readonly long startAddress; - public long previousAddress; - - /// - /// Check if client connection is healthy - /// - public bool IsConnected => garnetClient != null && garnetClient.IsConnected; - - /// - /// Return start address for this AOF iterator - /// - public long StartAddress => startAddress; - - public AofSyncTaskInfo( - ClusterProvider clusterProvider, - AofTaskStore aofTaskStore, - string localNodeId, - string remoteNodeId, - GarnetClientSession garnetClient, - long startAddress, - ILogger logger) - { - this.clusterProvider = clusterProvider; - this.aofTaskStore = aofTaskStore; - this.localNodeId = localNodeId; - this.remoteNodeId = remoteNodeId; - this.logger = logger; - this.garnetClient = garnetClient; - this.startAddress = startAddress; - previousAddress = startAddress; - cts = new CancellationTokenSource(); - } - - public void Dispose() - { - // First cancel the token - cts?.Cancel(); - - // Then, dispose the iterator. This will also signal the iterator so that it can observe the canceled token - iter?.Dispose(); - - // Finally, dispose the cts - cts?.Dispose(); - } - - public unsafe void Consume(byte* payloadPtr, int payloadLength, long currentAddress, long nextAddress, bool isProtected) - { - try - { - ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Aof_Sync_Task_Consume); - - // logger?.LogInformation("Sending {payloadLength} bytes to {remoteNodeId} at address {currentAddress}-{nextAddress}", payloadLength, remoteNodeId, currentAddress, nextAddress); - - // This is called under epoch protection, so we have to wait for appending to complete - garnetClient.ExecuteClusterAppendLog(localNodeId, previousAddress, currentAddress, nextAddress, (long)payloadPtr, payloadLength); - - // Set task address to nextAddress, as the iterator is currently at nextAddress - // (records at currentAddress are already sent above) - previousAddress = nextAddress; - } - catch (Exception ex) - { - logger?.LogWarning(ex, "An exception occurred at ReplicationManager.AofSyncTaskInfo.Consume"); - throw; - } - } - - public void Throttle() - { - if (!garnetClient.IsConnected) - ExceptionUtils.ThrowException(new GarnetException("AOF stream client disconnected!")); - - // Trigger flush while we are out of epoch protection - garnetClient.CompletePending(false); - garnetClient.Throttle(); - } - - /// - /// Main replica aof sync task. - /// - public async Task ReplicaSyncTaskAsync() - { - logger?.LogInformation("Starting ReplicationManager.ReplicaSyncTask for remote node {remoteNodeId} starting from address {address}", remoteNodeId, startAddress); - - try - { - if (!IsConnected) garnetClient.Connect(); - - iter = clusterProvider.storeWrapper.appendOnlyFile.ScanSingle(startAddress, long.MaxValue, scanUncommitted: true, recover: false, logger: logger); - - await iter.BulkConsumeAllAsync( - this, - clusterProvider.serverOptions.ReplicaSyncDelayMs, - maxChunkSize: 1 << 20, - cts.Token).ConfigureAwait(false); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "An exception occurred at ReplicationManager.ReplicaSyncTask - terminating"); - } - finally - { - garnetClient.Dispose(); - var (address, port) = clusterProvider.clusterManager.CurrentConfig.GetWorkerAddressFromNodeId(remoteNodeId); - logger?.LogWarning("AofSync task terminated; client disposed {remoteNodeId} {address} {port} {currentAddress}", remoteNodeId, address, port, previousAddress); - - if (!aofTaskStore.TryRemove(this)) - { - logger?.LogInformation("Did not remove {remoteNodeId} from aofTaskStore at end of ReplicaSyncTask", remoteNodeId); - } - } - } - } -} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/AofTaskStore.cs b/libs/cluster/Server/Replication/PrimaryOps/AofTaskStore.cs deleted file mode 100644 index 2b3985c8ffb..00000000000 --- a/libs/cluster/Server/Replication/PrimaryOps/AofTaskStore.cs +++ /dev/null @@ -1,441 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Net; -using Garnet.client; -using Garnet.common; -using Garnet.server; -using Microsoft.Extensions.Logging; - -namespace Garnet.cluster -{ - /// - /// Storage provider for AOF tasks - /// - internal sealed class AofTaskStore : IDisposable - { - readonly ClusterProvider clusterProvider; - readonly ILogger logger; - readonly int logPageSizeBits, logPageSizeMask; - readonly long TruncateLagAddress; - - AofSyncTaskInfo[] tasks; - int numTasks; - SingleWriterMultiReaderLock _lock; - bool _disposed; - public int Count => numTasks; - long TruncatedUntil; - - public AofTaskStore(ClusterProvider clusterProvider, int initialSize = 1, ILogger logger = null) - { - this.clusterProvider = clusterProvider; - this.logger = logger; - tasks = new AofSyncTaskInfo[initialSize]; - numTasks = 0; - if (clusterProvider.storeWrapper.appendOnlyFile != null) - { - logPageSizeBits = clusterProvider.storeWrapper.appendOnlyFile.UnsafeGetLogPageSizeBits(); - int logPageSize = 1 << logPageSizeBits; - logPageSizeMask = logPageSize - 1; - if (clusterProvider.serverOptions.FastAofTruncate) - clusterProvider.storeWrapper.appendOnlyFile.SafeTailShiftCallback = SafeTailShiftCallback; - TruncateLagAddress = clusterProvider.storeWrapper.appendOnlyFile.UnsafeGetReadOnlyAddressLagOffset() - 2 * logPageSize; - } - TruncatedUntil = 0; - } - - internal long AofTruncatedUntil => TruncatedUntil; - - internal void SafeTailShiftCallback(long oldTailAddress, long newTailAddress) - { - long oldPage = oldTailAddress >> logPageSizeBits; - long newPage = newTailAddress >> logPageSizeBits; - // Call truncate only once per page - if (oldPage != newPage) - { - // Truncate 2 pages after ReadOnly mark, so that we have sufficient time to shift begin before we flush - long truncateUntilAddress = (newTailAddress & ~logPageSizeMask) - TruncateLagAddress; - // Do not truncate beyond new tail (to handle corner cases) - if (truncateUntilAddress > newTailAddress) truncateUntilAddress = newTailAddress; - if (truncateUntilAddress > 0) - SafeTruncateAof(truncateUntilAddress); - } - } - - public List GetReplicaInfo(long PrimaryReplicationOffset) - { - // secondary0: ip=127.0.0.1,port=7001,state=online,offset=56,lag=0 - List replicaInfo = new(numTasks); - - _lock.ReadLock(); - var current = clusterProvider.clusterManager.CurrentConfig; - try - { - if (_disposed) return replicaInfo; - - for (int i = 0; i < numTasks; ++i) - { - var cr = tasks[i]; - var (address, port) = current.GetWorkerAddressFromNodeId(cr.remoteNodeId); - - replicaInfo.Add(new() - { - address = address, - port = port, - replication_state = cr.garnetClient.IsConnected ? "online" : "offline", - replication_offset = cr.previousAddress, - replication_lag = cr.previousAddress - PrimaryReplicationOffset - }); - } - } - finally - { - _lock.ReadUnlock(); - } - return replicaInfo; - } - - public void Dispose() - { - try - { - _lock.WriteLock(); - _disposed = true; - for (var i = 0; i < numTasks; i++) - { - var task = tasks[i]; - task.Dispose(); - } - numTasks = 0; - Array.Clear(tasks); - } - finally - { - _lock.WriteUnlock(); - } - } - - public bool TryAddReplicationTask(string remoteNodeId, long startAddress, out AofSyncTaskInfo aofSyncTaskInfo) - { - aofSyncTaskInfo = null; - - if (startAddress == 0) startAddress = ReplicationManager.kFirstValidAofAddress; - var success = false; - var current = clusterProvider.clusterManager.CurrentConfig; - var (address, port) = current.GetWorkerAddressFromNodeId(remoteNodeId); - - // If address is null or port is not valid, we cannot create a task - if (address == null || port <= 0 || ExceptionInjectionHelper.TriggerCondition(ExceptionInjectionType.Replication_Failed_To_AddAofSyncTask_UnknownNode)) - throw new GarnetException($"Failed to create AOF sync task for {remoteNodeId} with address {address} and port {port}"); - - // Create AofSyncTask - try - { - aofSyncTaskInfo = new AofSyncTaskInfo( - clusterProvider, - this, - current.LocalNodeId, - remoteNodeId, - new GarnetClientSession( - new IPEndPoint(IPAddress.Parse(address), port), - clusterProvider.replicationManager.GetAofSyncNetworkBufferSettings, - clusterProvider.replicationManager.GetNetworkPool, - tlsOptions: clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, - authUsername: clusterProvider.ClusterUsername, - authPassword: clusterProvider.ClusterPassword, - logger: logger), - startAddress, - logger); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "An error occurred at TryAddReplicationTask task creation for {remoteNodeId}", remoteNodeId); - return false; - } - - Debug.Assert(aofSyncTaskInfo != null); - - // Lock to prevent add/remove tasks and truncate operations - _lock.WriteLock(); - try - { - if (_disposed) return success; - - // Fail adding the task if truncation has happened, and we are not in AllowDataLoss mode - if (startAddress < TruncatedUntil && !clusterProvider.AllowDataLoss) - { - logger?.LogWarning("AOF sync task for {remoteNodeId}, with start address {startAddress}, could not be added, local AOF is truncated until {truncatedUntil}", remoteNodeId, startAddress, TruncatedUntil); - return success; - } - - // Iterate array of existing tasks and update associated task if it already exists - for (int i = 0; i < numTasks; i++) - { - var t = tasks[i]; - Debug.Assert(t != null); - if (t.remoteNodeId == remoteNodeId) - { - tasks[i] = aofSyncTaskInfo; - t.Dispose(); - success = true; - break; - } - } - - // If task did not exist we add it here - if (!success) - { - if (numTasks == tasks.Length) - { - var old_tasks = tasks; - var _tasks = new AofSyncTaskInfo[tasks.Length * 2]; - Array.Copy(tasks, _tasks, tasks.Length); - tasks = _tasks; - Array.Clear(old_tasks); - } - tasks[numTasks++] = aofSyncTaskInfo; - success = true; - } - } - catch (Exception ex) - { - logger?.LogWarning(ex, "An error occurred at TryAddReplicationTask task addition for {remoteNodeId}", remoteNodeId); - } - finally - { - _lock.WriteUnlock(); - if (!success) - { - aofSyncTaskInfo?.Dispose(); - aofSyncTaskInfo = null; - } - } - - return success; - } - - public bool TryAddReplicationTasks(ReplicaSyncSession[] replicaSyncSessions, long startAddress) - { - var current = clusterProvider.clusterManager.CurrentConfig; - var success = true; - if (startAddress == 0) startAddress = ReplicationManager.kFirstValidAofAddress; - - // First iterate through all sync sessions and add an AOF sync task - // All tasks will be - foreach (var rss in replicaSyncSessions) - { - if (rss == null) continue; - var replicaNodeId = rss.replicaSyncMetadata.originNodeId; - var (address, port) = current.GetWorkerAddressFromNodeId(replicaNodeId); - - // If address is null or port is not valid, we cannot create a task - if (address == null || port <= 0) - throw new GarnetException($"Failed to create AOF sync task for {replicaNodeId} with address {address} and port {port}"); - - try - { - rss.AddAofSyncTask(new AofSyncTaskInfo( - clusterProvider, - this, - current.LocalNodeId, - replicaNodeId, - new GarnetClientSession( - new IPEndPoint(IPAddress.Parse(address), port), - clusterProvider.replicationManager.GetAofSyncNetworkBufferSettings, - clusterProvider.replicationManager.GetNetworkPool, - tlsOptions: clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, - authUsername: clusterProvider.ClusterUsername, - authPassword: clusterProvider.ClusterPassword, - logger: logger), - startAddress, - logger)); - } - catch (Exception ex) - { - logger?.LogError(ex, "{method} creating AOF sync task for {replicaNodeId} failed", nameof(TryAddReplicationTasks), replicaNodeId); - return false; - } - } - - _lock.WriteLock(); - try - { - if (_disposed) return false; - - // Fail adding the task if truncation has happened - if (startAddress < TruncatedUntil && !clusterProvider.AllowDataLoss) - { - logger?.LogError("{method} failed to add tasks for AOF sync {startAddress} {truncatedUntil}", nameof(TryAddReplicationTasks), startAddress, TruncatedUntil); - return false; - } - - foreach (var rss in replicaSyncSessions) - { - if (rss == null) continue; - - var added = false; - // Find if AOF sync task already exists - for (var i = 0; i < numTasks; i++) - { - var t = tasks[i]; - Debug.Assert(t != null); - if (t.remoteNodeId == rss.replicaNodeId) - { - tasks[i] = rss.AofSyncTask; - t.Dispose(); - added = true; - break; - } - } - - if (added) continue; - - // If AOF sync task did not exist and was not added we added below - // Check if array can hold a new AOF sync task - if (numTasks == tasks.Length) - { - var old_tasks = tasks; - var _tasks = new AofSyncTaskInfo[tasks.Length * 2]; - Array.Copy(tasks, _tasks, tasks.Length); - tasks = _tasks; - Array.Clear(old_tasks); - } - // Add new AOF sync task - tasks[numTasks++] = rss.AofSyncTask; - } - - success = true; - } - finally - { - _lock.WriteUnlock(); - - if (!success) - { - foreach (var rss in replicaSyncSessions) - { - if (rss == null) continue; - rss.AofSyncTask?.Dispose(); - } - } - } - - return true; - } - - public bool TryRemove(AofSyncTaskInfo aofSyncTask) - { - // Lock addition of new tasks - _lock.WriteLock(); - - var success = false; - try - { - if (_disposed) return success; - - for (var i = 0; i < numTasks; i++) - { - var t = tasks[i]; - Debug.Assert(t != null); - if (t == aofSyncTask) - { - tasks[i] = null; - if (i < numTasks - 1) - { - // Swap the last task into the free slot - tasks[i] = tasks[numTasks - 1]; - tasks[numTasks - 1] = null; - } - // Reduce the number of tasks - numTasks--; - - // Kill the task - t.Dispose(); - success = true; - break; - } - } - } - finally - { - _lock.WriteUnlock(); - } - return success; - } - - /// - /// Safely truncate iterator - /// - /// - /// - public long SafeTruncateAof(long CheckpointCoveredAofAddress = long.MaxValue) - { - _lock.WriteLock(); - - if (_disposed) - { - _lock.WriteUnlock(); - return -1; - } - - // Calculate min address of all iterators - long TruncatedUntil = CheckpointCoveredAofAddress; - for (int i = 0; i < numTasks; i++) - { - Debug.Assert(tasks[i] != null); - if (tasks[i].previousAddress < TruncatedUntil) - TruncatedUntil = tasks[i].previousAddress; - } - - // Inform that we have logically truncatedUntil - _ = Tsavorite.core.Utility.MonotonicUpdate(ref this.TruncatedUntil, TruncatedUntil, out _); - // Release lock early - _lock.WriteUnlock(); - - if (TruncatedUntil is > 0 and < long.MaxValue) - { - if (clusterProvider.serverOptions.FastAofTruncate) - { - clusterProvider.storeWrapper.appendOnlyFile?.UnsafeShiftBeginAddress(TruncatedUntil, snapToPageStart: true, truncateLog: true); - } - else - { - clusterProvider.storeWrapper.appendOnlyFile?.TruncateUntil(TruncatedUntil); - clusterProvider.storeWrapper.appendOnlyFile?.Commit(); - } - } - return TruncatedUntil; - } - - public int CountConnectedReplicas() - { - var count = 0; - _lock.ReadLock(); - try - { - if (_disposed) return 0; - - for (var i = 0; i < numTasks; i++) - { - var t = tasks[i]; - count += t.garnetClient.IsConnected ? 1 : 0; - } - } - finally - { - _lock.ReadUnlock(); - } - return count; - } - - public void UpdateTruncatedUntil(long truncatedUntil) - { - _lock.WriteLock(); - Tsavorite.core.Utility.MonotonicUpdate(ref TruncatedUntil, truncatedUntil, out _); - _lock.WriteUnlock(); - } - } -} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/DataSourceReadResult.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/DataSourceReadResult.cs new file mode 100644 index 00000000000..b9410e4e1f0 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/DataSourceReadResult.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Represents the result of a chunk read from a checkpoint data source. + /// For device-backed sources, is set and is empty. + /// For memory-backed sources, is set and is null. + /// + internal readonly struct DataSourceReadResult + { + /// + /// The buffer containing the read data (device-backed sources). + /// May be null for memory-backed sources. + /// + public readonly SectorAlignedMemory Buffer; + + /// + /// The in-memory data (memory-backed sources). + /// Null for device-backed sources. + /// + public readonly byte[] Data; + + /// + /// The number of bytes read. + /// + public readonly int BytesRead; + + /// + /// The start address of this chunk in the source. + /// + public readonly long ChunkStartAddress; + + /// + /// Creates a device-backed chunk read result. + /// + public DataSourceReadResult(SectorAlignedMemory buffer, int bytesRead, long chunkStartAddress) + { + Buffer = buffer; + Data = default; + BytesRead = bytesRead; + ChunkStartAddress = chunkStartAddress; + } + + /// + /// Creates a memory-backed chunk read result. + /// + public DataSourceReadResult(byte[] data, long chunkStartAddress) + { + Buffer = null; + Data = data; + BytesRead = data.Length; + ChunkStartAddress = chunkStartAddress; + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileDataSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileDataSource.cs new file mode 100644 index 00000000000..ffeacec191a --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileDataSource.cs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Base class for checkpoint data sources that reads from an IDevice using sector-aligned async I/O. + /// Subclasses can override to customize read behavior. + /// + internal class FileDataSource : ISnapshotDataSource + { + /// + /// Default batch size for non-segmented checkpoint files. + /// + internal const int DefaultBatchSize = 1 << 17; + + private readonly int maxBatchSize; + private readonly TimeSpan timeout; + private readonly ILogger logger; + private readonly SectorAlignedBufferPool bufferPool; + private readonly SemaphoreSlim signalCompletion; + private readonly IOCallbackContext ioContext = new(); + private volatile uint lastIOErrorCode; + + public CheckpointFileType Type { get; } + public Guid Token { get; } + public IDevice Device { get; } + public long StartOffset { get; } + public long CurrentOffset { get; private set; } + public long EndOffset { get; } + + public bool HasNextChunk => CurrentOffset < EndOffset; + + /// + /// Creates a new FileDataSource. + /// + /// The checkpoint file type. + /// The checkpoint token. + /// The initialized device to read from. + /// The start offset. + /// The end offset. + /// Maximum bytes to read per chunk (will be further capped by sector alignment). + /// Timeout for async read operations. + /// Shared sector-aligned buffer pool for read operations. + /// Shared semaphore for async I/O completion signaling. + /// Optional logger. + public FileDataSource( + CheckpointFileType type, + Guid token, + IDevice device, + long startOffset, + long endOffset, + int maxBatchSize, + TimeSpan timeout, + SectorAlignedBufferPool bufferPool, + SemaphoreSlim signalCompletion, + ILogger logger = null) + { + Type = type; + Token = token; + Device = device; + StartOffset = startOffset; + CurrentOffset = startOffset; + EndOffset = endOffset; + this.maxBatchSize = maxBatchSize; + this.timeout = timeout; + this.bufferPool = bufferPool; + this.signalCompletion = signalCompletion; + this.logger = logger; + } + + /// + public void Dispose() + { + Device?.Dispose(); + } + + /// + public async Task ReadNextChunkAsync(CancellationToken cancellationToken = default) + { + var chunkStartAddress = CurrentOffset; + var remainingBytes = EndOffset - CurrentOffset; + var size = (int)Math.Min(remainingBytes, maxBatchSize); + + var (buffer, bytesRead) = await ReadIntoAsync(Device, (ulong)CurrentOffset, size, cancellationToken).ConfigureAwait(false); + CurrentOffset += bytesRead; + + return new DataSourceReadResult(buffer, bytesRead, chunkStartAddress); + } + + /// + /// Reads data from the device into a sector-aligned buffer. + /// Override this method to customize how reads are performed (e.g., segmented reads). + /// + /// The device to read from. + /// The address to read from. + /// The requested number of bytes (will be sector-aligned). + /// Cancellation token. + /// A tuple of the allocated buffer and the actual (sector-aligned) bytes read. + protected virtual async Task<(SectorAlignedMemory buffer, int bytesRead)> ReadIntoAsync( + IDevice device, + ulong address, + int size, + CancellationToken cancellationToken = default) + { + long numBytesToRead = size; + numBytesToRead = (numBytesToRead + (device.SectorSize - 1)) & ~(device.SectorSize - 1); + + var buffer = bufferPool.Get((int)numBytesToRead); + ioContext.Buffer = buffer; + + unsafe + { + device.ReadAsync(address, (IntPtr)buffer.aligned_pointer, (uint)numBytesToRead, IOCallback, ioContext); + } + + // The IOCallbackContext roots the buffer for GC safety while the IO is in-flight. + // On timeout or cancellation the buffer is intentionally abandoned (not returned to + // the pool) — the exception aborts the replication session, so the stale semaphore + // count left by the callback is harmless. + if (!await signalCompletion.WaitAsync(timeout, cancellationToken).ConfigureAwait(false)) + { + logger?.LogWarning("Timed out reading {type} checkpoint file at address {address}", Type, address); + ExceptionUtils.ThrowException(new GarnetException( + $"Timed out reading {Type} checkpoint file at address {address} (requested {numBytesToRead} bytes)")); + } + + return HandleIOError(buffer, address, numBytesToRead); + } + + private (SectorAlignedMemory buffer, int bytesRead) HandleIOError( + SectorAlignedMemory buffer, ulong address, long numBytesToRead) + { + var errorCode = lastIOErrorCode; + Debug.Assert(errorCode == 0, $"I/O error {errorCode} reading {Type} checkpoint file at address {address}"); + if (errorCode != 0) + { + ExceptionUtils.ThrowException(new GarnetException( + $"I/O error {errorCode} reading {Type} checkpoint file at address {address} (requested {numBytesToRead} bytes)")); + } + + return (buffer, (int)numBytesToRead); + } + + private void IOCallback(uint errorCode, uint numBytes, object context) + { + lastIOErrorCode = errorCode; + if (errorCode != 0) + { + var errorMessage = Utility.GetCallbackErrorMessage(errorCode, numBytes, context); + logger?.LogError("[CheckpointDataSource] ReadAsync error: {errorCode} msg: {errorMessage}", errorCode, errorMessage); + } + + try + { + _ = signalCompletion.Release(); + } + catch (ObjectDisposedException) { } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileTransmitSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileTransmitSource.cs new file mode 100644 index 00000000000..e8029ecb062 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/FileTransmitSource.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Transmits checkpoint file segments over the network using chunked reads from its owned . + /// Sends each chunk via + /// followed by an empty end-of-transmission packet. + /// + internal sealed class FileTransmitSource : ISnapshotTransmitSource + { + readonly ILogger logger; + + public ISnapshotDataSource DataSource { get; } + + public FileTransmitSource(ISnapshotDataSource dataSource, ILogger logger = null) + { + DataSource = dataSource; + this.logger = logger; + } + + /// + public async Task TransmitAsync(GarnetClientSession gcs, TimeSpan timeout, CancellationToken cancellationToken = default) + { + var fileTokenBytes = DataSource.Token.ToByteArray(); + + while (DataSource.HasNextChunk) + { + var result = await DataSource.ReadNextChunkAsync(cancellationToken).ConfigureAwait(false); + try + { + var resp = await gcs.ExecuteClusterSnapshotData( + fileTokenBytes, + (int)DataSource.Type, + startAddress: result.ChunkStartAddress, + result.Buffer.GetSlice(result.BytesRead)).WaitAsync(timeout, cancellationToken).ConfigureAwait(false); + + if (!resp.Equals("OK")) + ExceptionUtils.ThrowException(new GarnetException( + $"Primary error at TransmitAsync {DataSource.Type} {resp} [{DataSource.StartOffset},{DataSource.CurrentOffset},{DataSource.EndOffset}]")); + } + finally + { + result.Buffer.Return(); + } + } + + // Send empty package to indicate end of transmission + var endResp = await gcs.ExecuteClusterSnapshotData( + fileTokenBytes, (int)DataSource.Type, DataSource.CurrentOffset, []) + .WaitAsync(timeout, cancellationToken).ConfigureAwait(false); + + if (!endResp.Equals("OK")) + ExceptionUtils.ThrowException(new GarnetException( + $"Primary error at TransmitAsync Completion {DataSource.Type} {endResp}")); + } + + public void Dispose() + { + DataSource?.Dispose(); + } + } + + internal static unsafe class SectorAlignedMemoryExtensions + { + public static Span GetSlice(this SectorAlignedMemory pbuffer, int length) + { + return new Span(pbuffer.aligned_pointer, length); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotDataSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotDataSource.cs new file mode 100644 index 00000000000..5f199b8cbc8 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotDataSource.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Garnet.cluster +{ + /// + /// Interface for a checkpoint data source that provides chunk-based reading. + /// + internal interface ISnapshotDataSource : IDisposable + { + /// + /// The type of checkpoint file this data source represents. + /// + CheckpointFileType Type { get; } + + /// + /// The token identifying the checkpoint file. + /// + Guid Token { get; } + + /// + /// The start offset in the source. + /// + long StartOffset { get; } + + /// + /// The current read offset in the source. + /// + long CurrentOffset { get; } + + /// + /// The end offset in the source. + /// + long EndOffset { get; } + + /// + /// Whether there are remaining chunks to read. + /// + bool HasNextChunk { get; } + + /// + /// Reads the next chunk from the underlying source asynchronously. + /// Advances CurrentOffset by the number of bytes read. + /// For device-backed sources, the caller is responsible for returning the buffer via .Return(). + /// + /// Cancellation token. + /// A containing the data, bytes read, and chunk start address. + Task ReadNextChunkAsync(CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotReader.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotReader.cs new file mode 100644 index 00000000000..3f076f734a4 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotReader.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; + +namespace Garnet.cluster +{ + /// + /// Interface for a checkpoint reader that provides an enumeration of transmit sources. + /// + internal interface ISnapshotReader : IDisposable + { + /// + /// Returns an enumeration of transmit sources with initialized data sources. + /// The caller is responsible for disposing each transmit source after use. + /// + IEnumerable GetTransmitSources(); + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotTransmitSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotTransmitSource.cs new file mode 100644 index 00000000000..581074b54e8 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ISnapshotTransmitSource.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; + +namespace Garnet.cluster +{ + /// + /// Interface for transmitting checkpoint data over the network. + /// Implementations own an internally + /// and define how to read and ship the data to a replica. + /// + internal interface ISnapshotTransmitSource : IDisposable + { + /// + /// The underlying data source owned by this transmitter. + /// + ISnapshotDataSource DataSource { get; } + + /// + /// Transmits data from the owned data source to the replica via the provided client session. + /// + /// The client session connected to the replica. + /// Timeout for network operations. + /// Cancellation token. + Task TransmitAsync(GarnetClientSession gcs, TimeSpan timeout, CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ReplicaSyncSession.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ReplicaSyncSession.cs new file mode 100644 index 00000000000..6154e9e7ec5 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/ReplicaSyncSession.cs @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Garnet.common; +using Garnet.server; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + internal sealed partial class ReplicaSyncSession( + StoreWrapper storeWrapper, + ClusterProvider clusterProvider, + AofAddress replicaAofBeginAddress, + AofAddress replicaAofTailAddress, + SyncMetadata replicaSyncMetadata = null, + CancellationToken token = default, + string replicaNodeId = null, + string replicaAssignedPrimaryId = null, + CheckpointEntry replicaCheckpointEntry = null, + ILogger logger = null) : IDisposable + { + readonly StoreWrapper storeWrapper = storeWrapper; + readonly ClusterProvider clusterProvider = clusterProvider; + public readonly SyncMetadata replicaSyncMetadata = replicaSyncMetadata; + readonly CancellationToken token = token; + readonly CancellationTokenSource cts = new(); + readonly SemaphoreSlim signalCompletion = new(0); + + public readonly string replicaNodeId = replicaNodeId; + public readonly string replicaAssignedPrimaryId = replicaAssignedPrimaryId; + private readonly AofAddress replicaAofBeginAddress = replicaAofBeginAddress; + private readonly AofAddress replicaAofTailAddress = replicaAofTailAddress; + + private readonly CheckpointEntry replicaCheckpointEntry = replicaCheckpointEntry; + + private readonly ILogger logger = logger; + + public string errorMsg = default; + + const int validateMetadataMaxRetryCount = 10; + + public void Dispose() + { + AofSyncDriver?.DisposeClient(); + AofSyncDriver = null; + cts.Cancel(); + cts.Dispose(); + signalCompletion?.Dispose(); + } + + private bool ValidateMetadata( + CheckpointEntry localEntry, + out long index_size, + out LogFileInfo hlog_size, + out bool skipLocalMainStoreCheckpoint) + { + hlog_size = default; + index_size = -1L; + + // Local and remote checkpoints are of same history if both of the following hold + // 1. There is a checkpoint available at remote node + // 2. Remote and local checkpoints contain the same PrimaryReplId + var sameMainStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.storePrimaryReplId) && replicaCheckpointEntry.metadata.storePrimaryReplId.Equals(localEntry.metadata.storePrimaryReplId); + // We will not send the latest local checkpoint if any of the following hold + // 1. Local node does not have any checkpoints + // 2. Local checkpoint is of same version and history as the remote checkpoint + skipLocalMainStoreCheckpoint = localEntry.metadata.storeHlogToken == default || (sameMainStoreCheckpointHistory && localEntry.metadata.storeVersion == replicaCheckpointEntry.metadata.storeVersion); + + // Acquire metadata for main store + // If failed then this checkpoint is not usable because it is corrupted + if (!skipLocalMainStoreCheckpoint && !clusterProvider.replicationManager.TryAcquireSettledMetadataForMainStore(localEntry, out hlog_size, out index_size)) + return false; + + return true; + } + + /// + /// Start sending the latest checkpoint to replica + /// + public async Task SendCheckpointAsync() + { + errorMsg = default; + var current = clusterProvider.clusterManager.CurrentConfig; + var (address, port) = current.GetWorkerAddressFromNodeId(replicaNodeId); + + if (address == null || port == -1) + { + errorMsg = $"PRIMARY-ERR don't know about replicaId: {replicaNodeId}"; + logger?.LogError("{errorMsg}", errorMsg); + return false; + } + + GarnetClientSession gcs = new( + new IPEndPoint(IPAddress.Parse(address), port), + clusterProvider.replicationManager.GetRSSNetworkBufferSettings, + clusterProvider.replicationManager.GetNetworkPool, + tlsOptions: clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, + authUsername: clusterProvider.ClusterUsername, + authPassword: clusterProvider.ClusterPassword, + clientName: nameof(ReplicaSyncSession.SendCheckpointAsync), + logger: logger); + CheckpointEntry localEntry = default; + AofSyncDriver aofSyncDriver = null; + + try + { + logger?.LogInformation("Replica replicaId:{replicaId} requesting checkpoint replicaStoreVersion:{replicaStoreVersion}", + replicaNodeId, replicaCheckpointEntry.metadata.storeVersion); + + logger?.LogInformation("Attempting to acquire checkpoint"); + + (localEntry, aofSyncDriver) = await AcquireCheckpointEntryAsync().ConfigureAwait(false); + logger?.LogInformation("Checkpoint search completed"); + + await gcs.ConnectAsync((int)storeWrapper.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, cts.Token).ConfigureAwait(false); + + var index_size = -1L; + var hlog_size = default(LogFileInfo); + var skipLocalMainStoreCheckpoint = false; + var retryCount = validateMetadataMaxRetryCount; + while (!ValidateMetadata(localEntry, out index_size, out hlog_size, out skipLocalMainStoreCheckpoint)) + { + logger?.LogError("Failed to validate metadata. Retrying...."); + await Task.Yield(); + if (retryCount-- <= 0) + throw new GarnetException("Failed to validate metadata!"); + } + + #region sendStoresSnapshotData + if (!skipLocalMainStoreCheckpoint) + { + logger?.LogInformation("Sending main store checkpoint {version} {storeHlogToken} {storeIndexToken} to replica", localEntry.metadata.storeVersion, localEntry.metadata.storeHlogToken, localEntry.metadata.storeIndexToken); + + using var checkpointTransmissionDriver = new SnapshotTransmissionDriver(gcs, storeWrapper.serverOptions.ReplicaSyncTimeout, logger); + checkpointTransmissionDriver.AddReader(new TsavoriteSnapshotReader(clusterProvider, localEntry, hlog_size, index_size, storeWrapper.serverOptions.ReplicaSyncTimeout, logger)); + await checkpointTransmissionDriver.SendCheckpointAsync(cts.Token).ConfigureAwait(false); + } + #endregion + + #region startAofSync + var recoverFromRemote = !skipLocalMainStoreCheckpoint; + var checkpointAofBeginAddress = localEntry.GetMinAofCoveredAddress(); + var beginAddress = checkpointAofBeginAddress; + var sameHistory2 = string.IsNullOrEmpty(clusterProvider.replicationManager.PrimaryReplId2) && clusterProvider.replicationManager.PrimaryReplId2.Equals(replicaAssignedPrimaryId); + + // Calculate replay AOF range + var sameMainStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.storePrimaryReplId) && replicaCheckpointEntry.metadata.storePrimaryReplId.Equals(localEntry.metadata.storePrimaryReplId); + var replayAOFMap = clusterProvider.storeWrapper.appendOnlyFile.ComputeAofSyncReplayAddress( + recoverFromRemote, + sameMainStoreCheckpointHistory, + sameHistory2, + clusterProvider.replicationManager.ReplicationOffset2, + replicaAofBeginAddress, + replicaAofTailAddress, + beginAddress, + ref checkpointAofBeginAddress); + + // Signal replica to recover from local/remote checkpoint + // Make replica replayAOF if needed and replay from provided beginAddress to RecoveredReplication Address + var resp = await gcs.ExecuteClusterBeginReplicaRecover( + !skipLocalMainStoreCheckpoint, + replayAOFMap, + clusterProvider.replicationManager.PrimaryReplId, + localEntry.ToByteArray(), + beginAddress.Span, + checkpointAofBeginAddress.Span).WaitAsync(storeWrapper.serverOptions.ReplicaSyncTimeout, cts.Token).ConfigureAwait(false); + var syncFromAofAddress = AofAddress.FromString(resp); + + // Assert that AOF address the replica will be requesting can be served, except in case of: + // Possible AOF data loss: { using null AOF device } OR { main memory replication AND no on-demand checkpoints } + var possibleAofDataLoss = clusterProvider.serverOptions.UseAofNullDevice || + (clusterProvider.serverOptions.FastAofTruncate && !clusterProvider.serverOptions.OnDemandCheckpoint); + clusterProvider.storeWrapper.appendOnlyFile.DataLossCheck(possibleAofDataLoss, syncFromAofAddress, logger); + + // Check what happens if we fail after recovery and start AOF stream + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Replication_Fail_Before_Background_AOF_Stream_Task_Start); + + // We have already added the iterator for the covered address above but replica might request an address + // that is ahead of the covered address so we should start streaming from that address in order not to + // introduce duplicate insertions. + if (!clusterProvider.replicationManager.AofSyncDriverStore.TryAddReplicationDriver(replicaNodeId, ref syncFromAofAddress, out aofSyncDriver)) + throw new GarnetException("Failed trying to try update replication task"); + if (!clusterProvider.replicationManager.TryConnectToReplica(replicaNodeId, ref syncFromAofAddress, aofSyncDriver, out _)) + throw new GarnetException("Failed connecting to replica for aofSync"); + #endregion + } + catch (Exception ex) + { + if (localEntry != null) + logger?.LogCheckpointEntry(LogLevel.Error, "Error at attaching", localEntry); + else + logger?.LogError("Error at attaching: {ex}", ex.Message); + + if (aofSyncDriver != null) + _ = clusterProvider.replicationManager.AofSyncDriverStore.TryRemove(aofSyncDriver); + errorMsg = ex.Message;// this is error sent to remote client + return false; + } + finally + { + // At this point the replica has received the most recent checkpoint data + // and recovered from it so primary can release and delete it safely + localEntry?.RemoveReader(); + gcs.Dispose(); + } + return true; + } + + private async Task<(CheckpointEntry, AofSyncDriver)> AcquireCheckpointEntryAsync() + { + AofSyncDriver aofSyncDriver; + CheckpointEntry cEntry; + + // This loop tries to provide the following two guarantees + // 1. Retrieve latest checkpoint and lock it to prevent deletion before it is send to the replica + // 2. Guard against truncation of AOF in between the retrieval of the checkpoint metadata and start of the aofSyncTask + var iteration = 0; + var numOdcAttempts = 0; + const int maxOdcAttempts = 2; + + while (true) + { + cts.Token.ThrowIfCancellationRequested(); + logger?.LogInformation("AcquireCheckpointEntry iteration {iteration}", iteration); + iteration++; + + aofSyncDriver = null; + cEntry = default; + + // Acquire startSaveTime to identify if an external task might have taken the checkpoint for us + // This is only useful for MainMemoryReplication where we might have multiple replicas attaching + // We want to share the on-demand checkpoint and ensure that only one replica should succeed when calling TakeOnDemandCheckpoint + var lastSaveTime = storeWrapper.lastSaveTime; + + var exceptionInjected = ExceptionInjectionHelper.TriggerCondition(ExceptionInjectionType.Replication_Acquire_Checkpoint_Entry_Fail_Condition); + + // Retrieve latest checkpoint and lock it from deletion operations + var addedReader = !exceptionInjected && clusterProvider.replicationManager.TryGetLatestCheckpointEntryFromMemory(out cEntry); + + if (!addedReader) + { + // Fail to acquire lock, could mean that a writer might be trying to delete + logger?.LogWarning("Could not acquire lock for existing checkpoint, retrying."); + + // Go back to re-acquire the latest checkpoint + await Task.Yield(); + continue; + } + +#if DEBUG + // Only on Debug mode + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Replication_Wait_After_Checkpoint_Acquisition).ConfigureAwait(false); +#endif + + // Calculate the minimum start address covered by this checkpoint + var startAofAddress = cEntry.GetMinAofCoveredAddress(); + + // If there is possible AOF data loss and we need to take an on-demand checkpoint, + // then we should take the checkpoint before we register the sync task, because + // TryAddReplicationTask is guaranteed to return true in this scenario. + var validMetadata = ValidateMetadata(cEntry, out _, out _, out _); + if (clusterProvider.serverOptions.OnDemandCheckpoint && + (startAofAddress.AnyLesser(clusterProvider.replicationManager.AofSyncDriverStore.TruncatedUntil) || !validMetadata)) + { + if (numOdcAttempts >= maxOdcAttempts && clusterProvider.AllowDataLoss) + { + logger?.LogWarning("Failed to acquire checkpoint after {numOdcAttempts} on-demand checkpoint attempts. Possible data loss, startAofAddress:{startAofAddress} < truncatedUntil:{truncatedUntil}.", numOdcAttempts, startAofAddress, clusterProvider.replicationManager.AofSyncDriverStore.TruncatedUntil); + } + else + { + cEntry.RemoveReader(); + numOdcAttempts++; + logger?.LogInformation("Taking on-demand checkpoint, attempt {numOdcAttempts}.", numOdcAttempts); + await storeWrapper.TakeOnDemandCheckpointAsync(lastSaveTime).ConfigureAwait(false); + await Task.Yield(); + continue; + } + } + + // Validate that AofSyncDriver has been terminated + clusterProvider.replicationManager.AofSyncDriverStore.AssertDoesNotExist(replicaNodeId); + + // Enqueue AOF sync task with startAofAddress to prevent future AOF truncations + // and check if truncation has happened in between retrieving the latest checkpoint and enqueuing the aofSyncTask + if (clusterProvider.replicationManager.AofSyncDriverStore.TryAddReplicationDriver(replicaNodeId, ref startAofAddress, out aofSyncDriver)) + break; + + // Unlock last checkpoint because associated startAofAddress is no longer available + cEntry.RemoveReader(); + + // Go back to re-acquire checkpoint + await Task.Yield(); + } + + return (cEntry, aofSyncDriver); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/SnapshotTransmissionDriver.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/SnapshotTransmissionDriver.cs new file mode 100644 index 00000000000..439380479b0 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/SnapshotTransmissionDriver.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Microsoft.Extensions.Logging; + +namespace Garnet.cluster +{ + /// + /// Drives checkpoint transmission by iterating over checkpoint readers and sending segments. + /// + internal sealed class SnapshotTransmissionDriver : IDisposable + { + readonly List checkpointReaders = []; + readonly GarnetClientSession gcs; + readonly TimeSpan timeout; + readonly ILogger logger; + + public SnapshotTransmissionDriver(GarnetClientSession gcs, TimeSpan timeout, ILogger logger = null) + { + this.gcs = gcs; + this.timeout = timeout; + this.logger = logger; + } + + /// + /// Adds a checkpoint reader whose transmit sources will be sent during . + /// The driver takes ownership and will dispose the reader in . + /// + public void AddReader(ISnapshotReader reader) => checkpointReaders.Add(reader); + + public void Dispose() + { + foreach (var reader in checkpointReaders) + { + try { reader.Dispose(); } + catch (Exception ex) { logger?.LogError(ex, "Error disposing checkpoint reader"); } + } + } + + /// + /// Sends all checkpoint data by iterating transmit sources from each reader. + /// For each source, delegates transmission to the . + /// + public async Task SendCheckpointAsync(CancellationToken cancellationToken = default) + { + foreach (var checkpointReader in checkpointReaders) + { + foreach (var transmitSource in checkpointReader.GetTransmitSources()) + { + try + { + logger?.LogInformation(" fileDataSources = []; + readonly List metadataDataSources = []; + SectorAlignedBufferPool bufferPool; + readonly SemaphoreSlim signalCompletion = new(0); + + /// + /// Computes the maximum batch size for a given checkpoint file type. + /// For segmented types (HLOG, SNAPSHOT), returns the segment size. + /// For other types, returns the default batch size. + /// The actual read batch is capped at min(DefaultBatchSize, GetMaxBatchSize). + /// + public static int GetMaxBatchSize(CheckpointFileType type, GarnetServerOptions serverOptions) + { + return type switch + { + CheckpointFileType.STORE_HLOG or CheckpointFileType.STORE_SNAPSHOT => 1 << serverOptions.SegmentSizeBits(isObj: false), + CheckpointFileType.STORE_HLOG_OBJ or CheckpointFileType.STORE_SNAPSHOT_OBJ => 1 << serverOptions.SegmentSizeBits(isObj: true), + _ => FileDataSource.DefaultBatchSize + }; + } + + public TsavoriteSnapshotReader( + ClusterProvider clusterProvider, + CheckpointEntry checkpointEntry, + LogFileInfo logFileInfo, + long indexSize, + TimeSpan timeout, + ILogger logger = null) + { + this.clusterProvider = clusterProvider; + this.timeout = timeout; + this.logger = logger; + + // 1. send hlog file segments + if (clusterProvider.serverOptions.EnableStorageTier && logFileInfo.hybridLogFileEndAddress > PageHeader.Size) + { + fileDataSources.Add(CreateFileDataSource( + CheckpointFileType.STORE_HLOG, + checkpointEntry.metadata.storeHlogToken, + logFileInfo.hybridLogFileStartAddress, + logFileInfo.hybridLogFileEndAddress)); + + if (logFileInfo.hasSnapshotObjects) + fileDataSources.Add(CreateFileDataSource( + CheckpointFileType.STORE_HLOG_OBJ, + checkpointEntry.metadata.storeHlogToken, + logFileInfo.hybridLogObjectFileStartAddress, + logFileInfo.hybridLogObjectFileEndAddress)); + } + + // 2. Send index file segments + fileDataSources.Add(CreateFileDataSource( + CheckpointFileType.STORE_INDEX, + checkpointEntry.metadata.storeIndexToken, + 0, + indexSize)); + + // 3. Send snapshot files + if (logFileInfo.snapshotFileEndAddress > PageHeader.Size) + { + fileDataSources.Add(CreateFileDataSource( + CheckpointFileType.STORE_SNAPSHOT, + checkpointEntry.metadata.storeHlogToken, + 0, + logFileInfo.snapshotFileEndAddress)); + + if (logFileInfo.hasSnapshotObjects) + fileDataSources.Add(CreateFileDataSource( + CheckpointFileType.STORE_SNAPSHOT_OBJ, + checkpointEntry.metadata.storeHlogToken, + 0, + logFileInfo.snapshotObjectFileEndAddress)); + } + + // 4. Metadata sources + var storeCkptManager = clusterProvider.ReplicationLogCheckpointManager; + + metadataDataSources.Add(new TsavoriteMetadataSource( + CheckpointFileType.STORE_INDEX, + checkpointEntry.metadata.storeIndexToken, + () => checkpointEntry.metadata.storeIndexToken != default + ? storeCkptManager.GetIndexCheckpointMetadata(checkpointEntry.metadata.storeIndexToken) + : [])); + + metadataDataSources.Add(new TsavoriteMetadataSource( + CheckpointFileType.STORE_SNAPSHOT, + checkpointEntry.metadata.storeHlogToken, + () => checkpointEntry.metadata.storeHlogToken != default + ? storeCkptManager.GetLogCheckpointMetadata(checkpointEntry.metadata.storeHlogToken) + : [])); + } + + /// + public IEnumerable GetTransmitSources() + { + foreach (var dataSource in fileDataSources) + { + yield return new FileTransmitSource(dataSource, logger); + } + + foreach (var dataSource in metadataDataSources) + { + yield return new TsavoriteMetadataTransmitSource(dataSource, logger); + } + } + + private FileDataSource CreateFileDataSource(CheckpointFileType type, Guid token, long startOffset, long endOffset) + { + var device = CreateCheckpointDevice(type, token); + bufferPool ??= new SectorAlignedBufferPool(1, (int)device.SectorSize); + var maxBatchSize = Math.Min(FileDataSource.DefaultBatchSize, GetMaxBatchSize(type, clusterProvider.serverOptions)); + + return new FileDataSource( + type, + token, + device, + startOffset, + endOffset, + maxBatchSize, + timeout, + bufferPool, + signalCompletion, + logger); + } + + private IDevice CreateCheckpointDevice(CheckpointFileType type, Guid token) + { + var device = type switch + { + CheckpointFileType.STORE_HLOG => GetStoreHLogDevice(isObj: false), + CheckpointFileType.STORE_HLOG_OBJ => GetStoreHLogDevice(isObj: true), + _ => clusterProvider.ReplicationLogCheckpointManager.GetDevice(type, token), + }; + + var segmentSize = GetMaxBatchSize(type, clusterProvider.serverOptions); + switch (type) + { + case CheckpointFileType.STORE_HLOG: + case CheckpointFileType.STORE_SNAPSHOT: + case CheckpointFileType.STORE_HLOG_OBJ: + case CheckpointFileType.STORE_SNAPSHOT_OBJ: + device.Initialize(segmentSize: segmentSize); + break; + } + + return device; + } + + private IDevice GetStoreHLogDevice(bool isObj) + { + var opts = clusterProvider.serverOptions; + if (opts.EnableStorageTier) + { + var LogDir = !string.IsNullOrEmpty(opts.LogDir) ? opts.LogDir : Directory.GetCurrentDirectory(); + var logFactory = opts.GetInitializedDeviceFactory(LogDir); + + // These must match GarnetServerOptions.GetSettings, EnableStorageTier + return logFactory.Get(new FileDescriptor("Store", isObj ? "hlog_objs" : "hlog")); + } + return null; + } + + public void Dispose() + { + foreach (var ds in fileDataSources) + { + try { ds.Dispose(); } + catch (Exception ex) { logger?.LogError(ex, "Error disposing file data source {type} {token}", ds.Type, ds.Token); } + } + fileDataSources.Clear(); + + foreach (var ds in metadataDataSources) + { + try { ds.Dispose(); } + catch (Exception ex) { logger?.LogError(ex, "Error disposing metadata data source {type} {token}", ds.Type, ds.Token); } + } + metadataDataSources.Clear(); + + signalCompletion?.Dispose(); + bufferPool?.Free(); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataSource.cs new file mode 100644 index 00000000000..fdbab44c709 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataSource.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Garnet.cluster +{ + /// + /// A checkpoint data source backed by in-memory byte data (e.g., checkpoint metadata). + /// Returns the byte array as a single chunk via . + /// + internal sealed class TsavoriteMetadataSource : ISnapshotDataSource + { + private readonly Func dataFactory; + private bool consumed; + + /// + public CheckpointFileType Type { get; } + + /// + public Guid Token { get; } + + /// + public long StartOffset => 0; + + /// + public long CurrentOffset => consumed ? EndOffset : 0; + + /// + public long EndOffset { get; private set; } + + /// + public bool HasNextChunk => !consumed; + + /// + /// Creates a metadata data source with a factory that lazily produces the byte array. + /// + /// The checkpoint file type (e.g., STORE_INDEX or STORE_SNAPSHOT). + /// The checkpoint token. + /// A factory that returns the metadata bytes. Called on first read. + public TsavoriteMetadataSource(CheckpointFileType type, Guid token, Func dataFactory) + { + Type = type; + Token = token; + this.dataFactory = dataFactory; + } + + /// + public void Dispose() + { + } + + /// + public Task ReadNextChunkAsync(CancellationToken cancellationToken = default) + { + if (consumed) + throw new InvalidOperationException("TsavoriteMetadataSource has already been consumed."); + + var data = dataFactory() ?? []; + EndOffset = data.Length; + consumed = true; + + return Task.FromResult(new DataSourceReadResult(data, chunkStartAddress: 0)); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataTransmitSource.cs b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataTransmitSource.cs new file mode 100644 index 00000000000..aad494fe215 --- /dev/null +++ b/libs/cluster/Server/Replication/PrimaryOps/DiskbasedReplication/TsavoriteMetadataTransmitSource.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Garnet.client; +using Garnet.common; +using Microsoft.Extensions.Logging; + +namespace Garnet.cluster +{ + /// + /// Transmits checkpoint metadata over the network. + /// Reads the metadata bytes from the owned data source and sends them via + /// . + /// Includes retry logic for transient failures. + /// + internal sealed class TsavoriteMetadataTransmitSource : ISnapshotTransmitSource + { + const int MaxRetryCount = 10; + readonly ILogger logger; + + public ISnapshotDataSource DataSource { get; } + + public TsavoriteMetadataTransmitSource(ISnapshotDataSource dataSource, ILogger logger = null) + { + DataSource = dataSource; + this.logger = logger; + } + + /// + public async Task TransmitAsync(GarnetClientSession gcs, TimeSpan timeout, CancellationToken cancellationToken = default) + { + var retryCount = MaxRetryCount; + + while (true) + { + try + { + logger?.LogInformation(" /// Get the associated aof sync task instance with this replica sync session /// - public AofSyncTaskInfo AofSyncTask { get; private set; } = null; + public AofSyncDriver AofSyncDriver { get; private set; } = null; - public bool IsConnected => AofSyncTask != null && AofSyncTask.IsConnected; + public bool IsConnected => AofSyncDriver != null && AofSyncDriver.IsConnected; public bool Failed => ssInfo.syncStatus == SyncStatus.FAILED; @@ -31,32 +32,26 @@ internal sealed partial class ReplicaSyncSession public long currentStoreVersion; - public long currentObjectStoreVersion; - /// /// Pessimistic checkpoint covered AOF address /// - public long checkpointCoveredAofAddress; + public AofAddress checkpointCoveredAofAddress; #region NetworkMethods /// /// Connect client /// - public void Connect() - { - if (!AofSyncTask.IsConnected) - AofSyncTask.garnetClient.Connect(); - } + public Task ConnectAsync() + => AofSyncDriver.ConnectClientsAsync(); /// - /// Execute async command + /// Issue FlushAll /// - /// /// - public async Task ExecuteAsync(params string[] commands) + public async Task IssueFlushAllAsync() { await WaitForFlushAsync().ConfigureAwait(false); - return await AofSyncTask.garnetClient.ExecuteAsync(commands).ConfigureAwait(false); + return await AofSyncDriver.IssuesFlushAllAsync().ConfigureAwait(false); } /// @@ -65,45 +60,26 @@ public async Task ExecuteAsync(params string[] commands) public void InitializeIterationBuffer() { AsyncUtils.BlockingWait(WaitForFlushAsync()); - AofSyncTask.garnetClient.InitializeIterationBuffer(clusterProvider.storeWrapper.loggingFrequency); + AofSyncDriver.InitializeIterationBuffer(); } /// /// Set Cluster Sync header /// - /// - public void SetClusterSyncHeader(bool isMainStore) + public void SetClusterSyncHeader() { AsyncUtils.BlockingWait(WaitForFlushAsync()); - if (AofSyncTask.garnetClient.NeedsInitialization) - AofSyncTask.garnetClient.SetClusterSyncHeader(clusterProvider.clusterManager.CurrentConfig.LocalNodeId, isMainStore: isMainStore); + AofSyncDriver.InitializeIfNeeded(); } /// - /// Try write main store key value pair + /// Try to write the span of an entire record. /// - /// - /// - /// /// - public bool TryWriteKeyValueSpanByte(ref SpanByte key, ref SpanByte value, out Task task) + public bool TryWriteRecordSpan(ReadOnlySpan recordSpan, MigrationRecordSpanType recordSpanType, out Task task) { AsyncUtils.BlockingWait(WaitForFlushAsync()); - return AofSyncTask.garnetClient.TryWriteKeyValueSpanByte(ref key, ref value, out task); - } - - /// - /// Try write object store key value pair - /// - /// - /// - /// - /// - /// - public bool TryWriteKeyValueByteArray(byte[] key, byte[] value, long expiration, out Task task) - { - AsyncUtils.BlockingWait(WaitForFlushAsync()); - return AofSyncTask.garnetClient.TryWriteKeyValueByteArray(key, value, expiration, out task); + return AofSyncDriver.TryWriteRecordSpan(recordSpan, recordSpanType, out task); } /// @@ -113,15 +89,15 @@ public bool TryWriteKeyValueByteArray(byte[] key, byte[] value, long expiration, public void SendAndResetIterationBuffer() { AsyncUtils.BlockingWait(WaitForFlushAsync()); - SetFlushTask(AofSyncTask.garnetClient.SendAndResetIterationBuffer()); + SetFlushTask(AofSyncDriver.SendAndResetIterationBufferAsync()); } #endregion /// /// Associated aof sync task instance with this replica sync session /// - /// - public void AddAofSyncTask(AofSyncTaskInfo aofSyncTask) => AofSyncTask = aofSyncTask; + /// + public void AddAofSyncTask(AofSyncDriver aofSyncDriver) => AofSyncDriver = aofSyncDriver; /// /// Set status of replica sync session @@ -138,8 +114,11 @@ public void SetStatus(SyncStatus status, string error = null) switch (status) { case SyncStatus.SUCCESS: + _ = signalCompletion.Release(); + break; case SyncStatus.FAILED: - signalCompletion.Release(); + _ = clusterProvider.replicationManager.AofSyncDriverStore.TryRemove(AofSyncDriver); + _ = signalCompletion.Release(); break; } } @@ -214,21 +193,19 @@ public bool NeedToFullSync() var localPrimaryReplId = clusterProvider.replicationManager.PrimaryReplId; var sameHistory = localPrimaryReplId.Equals(replicaSyncMetadata.currentPrimaryReplId, StringComparison.Ordinal); var sendMainStore = !sameHistory || replicaSyncMetadata.currentStoreVersion != currentStoreVersion; - var sendObjectStore = !sameHistory || replicaSyncMetadata.currentObjectStoreVersion != currentObjectStoreVersion; - var aofBeginAddress = clusterProvider.storeWrapper.appendOnlyFile.BeginAddress; - var aofTailAddress = clusterProvider.storeWrapper.appendOnlyFile.TailAddress; - var outOfRangeAof = replicaSyncMetadata.currentAofTailAddress < aofBeginAddress || replicaSyncMetadata.currentAofTailAddress > aofTailAddress; + var aofBeginAddress = clusterProvider.storeWrapper.appendOnlyFile.Log.BeginAddress; + var aofTailAddress = clusterProvider.storeWrapper.appendOnlyFile.Log.TailAddress; + var outOfRangeAof = replicaSyncMetadata.currentAofTailAddress.IsOutOfRange(aofBeginAddress, aofTailAddress); - var aofTooLarge = (aofTailAddress - replicaSyncMetadata.currentAofTailAddress) > clusterProvider.serverOptions.ReplicaDisklessSyncFullSyncAofThresholdValue(); + var aofTooLarge = aofTailAddress.AggregateDiff(replicaSyncMetadata.currentAofTailAddress) > clusterProvider.serverOptions.ReplicaDisklessSyncFullSyncAofThresholdValue(); // We need to stream checkpoint if any of the following conditions are met: // 1. Replica has different history than primary - // 2. Replica has different main store version than primary - // 3. Replica has different object store version than primary - // 4. Replica has truncated AOF - // 5. The AOF to be replayed in case of a partial sync is larger than the specified threshold - fullSync = sendMainStore || sendObjectStore || outOfRangeAof || aofTooLarge; + // 2. Replica has different store version than primary + // 3. Replica has truncated AOF + // 4. The AOF to be replayed in case of a partial sync is larger than the specified threshold + fullSync = sendMainStore || outOfRangeAof || aofTooLarge; return fullSync; } @@ -237,11 +214,11 @@ public bool NeedToFullSync() /// public async Task BeginAofSyncAsync() { - var aofSyncTask = AofSyncTask; + var aofSyncDriver = AofSyncDriver; try { - var currentAofBeginAddress = fullSync ? checkpointCoveredAofAddress : aofSyncTask.StartAddress; - var currentAofTailAddress = clusterProvider.storeWrapper.appendOnlyFile.TailAddress; + var currentAofBeginAddress = fullSync ? checkpointCoveredAofAddress : aofSyncDriver.StartAddress; + var currentAofTailAddress = clusterProvider.storeWrapper.appendOnlyFile.Log.TailAddress; var recoverSyncMetadata = new SyncMetadata( fullSync: fullSync, @@ -249,19 +226,13 @@ public async Task BeginAofSyncAsync() originNodeId: clusterProvider.clusterManager.CurrentConfig.LocalNodeId, currentPrimaryReplId: clusterProvider.replicationManager.PrimaryReplId, currentStoreVersion: currentStoreVersion, - currentObjectStoreVersion: currentObjectStoreVersion, currentAofBeginAddress: currentAofBeginAddress, currentAofTailAddress: currentAofTailAddress, currentReplicationOffset: clusterProvider.replicationManager.ReplicationOffset, checkpointEntry: null); - var result = await aofSyncTask.garnetClient.ExecuteAttachSync(recoverSyncMetadata.ToByteArray()).ConfigureAwait(false); - if (!long.TryParse(result, out var syncFromAofAddress)) - { - logger?.LogError("Failed to parse syncFromAddress at {method}", nameof(BeginAofSyncAsync)); - SetStatus(SyncStatus.FAILED, "Failed to parse recovery offset"); - return; - } + var result = await aofSyncDriver.ExecuteAttachSyncAsync(recoverSyncMetadata).ConfigureAwait(false); + var syncFromAddress = AofAddress.FromString(result); logger?.LogSyncMetadata(LogLevel.Trace, "BeginAofSync", replicaSyncMetadata, recoverSyncMetadata); @@ -271,16 +242,15 @@ public async Task BeginAofSyncAsync() // We have already added the iterator for the covered address above but replica might request an address // that is ahead of the covered address so we should start streaming from that address in order not to // introduce duplicate insertions. - if (!clusterProvider.replicationManager.TryAddReplicationTask(replicaSyncMetadata.originNodeId, syncFromAofAddress, out aofSyncTask)) + if (!clusterProvider.replicationManager.AofSyncDriverStore.TryAddReplicationDriver(replicaSyncMetadata.originNodeId, ref syncFromAddress, out aofSyncDriver)) throw new GarnetException("Failed trying to try update replication task"); - if (!clusterProvider.replicationManager.TryConnectToReplica(replicaSyncMetadata.originNodeId, syncFromAofAddress, aofSyncTask, out _)) + if (!clusterProvider.replicationManager.TryConnectToReplica(replicaSyncMetadata.originNodeId, ref syncFromAddress, aofSyncDriver, out _)) throw new GarnetException("Failed connecting to replica for aofSync"); } catch (Exception ex) { logger?.LogError(ex, "{method}", $"{nameof(BeginAofSyncAsync)}"); SetStatus(SyncStatus.FAILED, ex.Message); - _ = clusterProvider.replicationManager.TryRemoveReplicationTask(AofSyncTask); } } } diff --git a/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSnapshotIterator.cs b/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSnapshotIterator.cs index b4159ada522..c5750f7349f 100644 --- a/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSnapshotIterator.cs +++ b/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSnapshotIterator.cs @@ -2,7 +2,9 @@ // Licensed under the MIT license. using System; +using System.Buffers; using System.Threading; +using Garnet.client; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -13,12 +15,15 @@ namespace Garnet.cluster internal sealed unsafe class SnapshotIteratorManager { public readonly ReplicationSyncManager replicationSyncManager; - public readonly TimeSpan timeout; public readonly CancellationToken cancellationToken; public readonly ILogger logger; - public MainStoreSnapshotIterator mainStoreSnapshotIterator; - public ObjectStoreSnapshotIterator objectStoreSnapshotIterator; + public StoreSnapshotIterator StoreSnapshotIterator; + + // For serialization from LogRecord to DiskLogRecord + SpanByteAndMemory serializationOutput; + GarnetObjectSerializer valueObjectSerializer; + MemoryPool memoryPool; readonly ReplicaSyncSession[] sessions; readonly int numSessions; @@ -27,7 +32,7 @@ internal sealed unsafe class SnapshotIteratorManager long currentFlushEventCount = 0; long lastFlushEventCount = 0; - public long CheckpointCoveredAddress { get; private set; } + AofAddress CheckpointCoveredAddress { get; set; } public SnapshotIteratorManager(ReplicationSyncManager replicationSyncManager, CancellationToken cancellationToken, ILogger logger = null) { @@ -38,16 +43,17 @@ public SnapshotIteratorManager(ReplicationSyncManager replicationSyncManager, Ca sessions = replicationSyncManager.Sessions; numSessions = replicationSyncManager.NumSessions; - CheckpointCoveredAddress = replicationSyncManager.ClusterProvider.storeWrapper.appendOnlyFile.TailAddress; + CheckpointCoveredAddress = replicationSyncManager.ClusterProvider.storeWrapper.appendOnlyFile.Log.TailAddress; for (var i = 0; i < numSessions; i++) { if (!replicationSyncManager.IsActive(i)) continue; sessions[i].checkpointCoveredAofAddress = CheckpointCoveredAddress; } - mainStoreSnapshotIterator = new MainStoreSnapshotIterator(this); - if (!replicationSyncManager.ClusterProvider.serverOptions.DisableObjects) - objectStoreSnapshotIterator = new ObjectStoreSnapshotIterator(this); + StoreSnapshotIterator = new StoreSnapshotIterator(this); + + memoryPool = MemoryPool.Shared; + valueObjectSerializer = new(customCommandManager: default); } /// @@ -68,7 +74,7 @@ public bool IsProgressing() } } - public bool OnStart(Guid checkpointToken, long currentVersion, long targetVersion, bool isMainStore) + public bool OnStart(Guid checkpointToken, long currentVersion, long targetVersion) { if (cancellationToken.IsCancellationRequested) { @@ -81,28 +87,33 @@ public bool OnStart(Guid checkpointToken, long currentVersion, long targetVersio for (var i = 0; i < numSessions; i++) { - if (!replicationSyncManager.IsActive(i)) continue; + if (!replicationSyncManager.IsActive(i)) + continue; sessions[i].InitializeIterationBuffer(); - if (isMainStore) - sessions[i].currentStoreVersion = targetVersion; - else - sessions[i].currentObjectStoreVersion = targetVersion; + sessions[i].currentStoreVersion = targetVersion; } - logger?.LogTrace("{OnStart} {store} {token} {currentVersion} {targetVersion}", - nameof(OnStart), isMainStore ? "MAIN STORE" : "OBJECT STORE", checkpointToken, currentVersion, targetVersion); + logger?.LogTrace("{OnStart} {token} {currentVersion} {targetVersion}", + nameof(OnStart), checkpointToken, currentVersion, targetVersion); return true; } - public bool Reader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords) + public bool StringReader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords) + where TSourceLogRecord : ISourceLogRecord { if (!firstRead) { + var key = srcLogRecord.Key; + var value = srcLogRecord.ValueSpan; logger?.LogTrace("Start Streaming {key} {value}", key.ToString(), value.ToString()); firstRead = true; } + // Note: We may be sending to multiple replicas, so serialize LogRecords to a local then copy to the multiple network buffers + // rather than issuing multiple serialization calls. + _ = DiskLogRecord.Serialize(in srcLogRecord, maxHeapAllocationSize: -1, valueObjectSerializer: default, memoryPool, ref serializationOutput); + var needToFlush = false; while (true) { @@ -115,20 +126,22 @@ public bool Reader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMe // Write key value pair to network buffer for (var i = 0; i < numSessions; i++) { - if (!replicationSyncManager.IsActive(i)) continue; + if (!replicationSyncManager.IsActive(i)) + continue; // Initialize header if necessary - sessions[i].SetClusterSyncHeader(isMainStore: true); + sessions[i].SetClusterSyncHeader(); // Try to write to network buffer. If failed we need to retry - if (!sessions[i].TryWriteKeyValueSpanByte(ref key, ref value, out var task)) + if (!sessions[i].TryWriteRecordSpan(serializationOutput.MemorySpan, MigrationRecordSpanType.LogRecord, out var task)) { sessions[i].SetFlushTask(task); needToFlush = true; } } - if (!needToFlush) break; + if (!needToFlush) + break; // Wait for flush to complete for all and retry to enqueue previous keyValuePair above AsyncUtils.BlockingWait(replicationSyncManager.WaitForFlushAsync()); @@ -139,16 +152,22 @@ public bool Reader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMe return true; } - public bool Reader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords) + public bool ObjectReader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords) + where TSourceLogRecord : ISourceLogRecord { if (!firstRead) { + var key = srcLogRecord.Key; + var value = srcLogRecord.ValueObject; logger?.LogTrace("Start Streaming {key} {value}", key.ToString(), value.ToString()); firstRead = true; } + // Note: We may be sending to multiple replicas, so cannot serialize LogRecords directly to the network buffer + var maxHeapAllocationSize = replicationSyncManager.ClusterProvider.replicationManager.networkBufferSettings.sendBufferSize; + var recordSize = DiskLogRecord.Serialize(in srcLogRecord, maxHeapAllocationSize, valueObjectSerializer, memoryPool, ref serializationOutput); + var needToFlush = false; - var objectData = GarnetObjectSerializer.Serialize(value); while (true) { if (cancellationToken.IsCancellationRequested) @@ -160,20 +179,22 @@ public bool Reader(ref byte[] key, ref IGarnetObject value, RecordMetadata recor // Write key value pair to network buffer for (var i = 0; i < numSessions; i++) { - if (!replicationSyncManager.IsActive(i)) continue; + if (!replicationSyncManager.IsActive(i)) + continue; // Initialize header if necessary - sessions[i].SetClusterSyncHeader(isMainStore: false); + sessions[i].SetClusterSyncHeader(); // Try to write to network buffer. If failed we need to retry - if (!sessions[i].TryWriteKeyValueByteArray(key, objectData, value.Expiration, out var task)) + if (!sessions[i].TryWriteRecordSpan(serializationOutput.MemorySpan.Slice(0, recordSize), MigrationRecordSpanType.LogRecord, out var task)) { sessions[i].SetFlushTask(task); needToFlush = true; } } - if (!needToFlush) break; + if (!needToFlush) + break; // Wait for flush to complete for all and retry to enqueue previous keyValuePair above AsyncUtils.BlockingWait(replicationSyncManager.WaitForFlushAsync()); @@ -183,67 +204,51 @@ public bool Reader(ref byte[] key, ref IGarnetObject value, RecordMetadata recor return true; } - public void OnStop(bool completed, long numberOfRecords, bool isMainStore, long targetVersion) + public void OnStop(bool completed, long numberOfRecords, long targetVersion) { // Flush remaining data for (var i = 0; i < numSessions; i++) { - if (!replicationSyncManager.IsActive(i)) continue; - sessions[i].SendAndResetIterationBuffer(); + if (replicationSyncManager.IsActive(i)) + sessions[i].SendAndResetIterationBuffer(); } // Wait for flush and response to complete AsyncUtils.BlockingWait(replicationSyncManager.WaitForFlushAsync()); - logger?.LogTrace("{OnStop} {store} {numberOfRecords} {targetVersion}", - nameof(OnStop), isMainStore ? "MAIN STORE" : "OBJECT STORE", numberOfRecords, targetVersion); + logger?.LogTrace("{OnStop} {numberOfRecords} {targetVersion}", + nameof(OnStop), numberOfRecords, targetVersion); // Reset read marker firstRead = false; + + serializationOutput.Dispose(); } } - internal sealed unsafe class MainStoreSnapshotIterator(SnapshotIteratorManager snapshotIteratorManager) : - IStreamingSnapshotIteratorFunctions + internal sealed unsafe class StoreSnapshotIterator(SnapshotIteratorManager snapshotIteratorManager) : + IStreamingSnapshotIteratorFunctions { - readonly SnapshotIteratorManager snapshotIteratorManager = snapshotIteratorManager; long targetVersion; public bool OnStart(Guid checkpointToken, long currentVersion, long targetVersion) { this.targetVersion = targetVersion; - return snapshotIteratorManager.OnStart(checkpointToken, currentVersion, targetVersion, isMainStore: true); + return snapshotIteratorManager.OnStart(checkpointToken, currentVersion, targetVersion); } - public bool Reader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords) - => snapshotIteratorManager.Reader(ref key, ref value, recordMetadata, numberOfRecords); - - public void OnException(Exception exception, long numberOfRecords) - => snapshotIteratorManager.logger?.LogError(exception, $"{nameof(MainStoreSnapshotIterator)}"); - - public void OnStop(bool completed, long numberOfRecords) - => snapshotIteratorManager.OnStop(completed, numberOfRecords, isMainStore: true, targetVersion); - } - - internal sealed unsafe class ObjectStoreSnapshotIterator(SnapshotIteratorManager snapshotIteratorManager) : - IStreamingSnapshotIteratorFunctions - { - readonly SnapshotIteratorManager snapshotIteratorManager = snapshotIteratorManager; - long targetVersion; - - public bool OnStart(Guid checkpointToken, long currentVersion, long targetVersion) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords) + where TSourceLogRecord : ISourceLogRecord { - this.targetVersion = targetVersion; - return snapshotIteratorManager.OnStart(checkpointToken, currentVersion, targetVersion, isMainStore: false); + return srcLogRecord.Info.ValueIsObject + ? snapshotIteratorManager.ObjectReader(in srcLogRecord, recordMetadata, numberOfRecords) + : snapshotIteratorManager.StringReader(in srcLogRecord, recordMetadata, numberOfRecords); } - public bool Reader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords) - => snapshotIteratorManager.Reader(ref key, ref value, recordMetadata, numberOfRecords); - public void OnException(Exception exception, long numberOfRecords) - => snapshotIteratorManager.logger?.LogError(exception, $"{nameof(ObjectStoreSnapshotIterator)}"); + => snapshotIteratorManager.logger?.LogError(exception, $"{nameof(StoreSnapshotIterator)}"); public void OnStop(bool completed, long numberOfRecords) - => snapshotIteratorManager.OnStop(completed, numberOfRecords, isMainStore: false, targetVersion); + => snapshotIteratorManager.OnStop(completed, numberOfRecords, targetVersion); } } \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSyncManager.cs b/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSyncManager.cs index f30483a0996..e9423f10f77 100644 --- a/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSyncManager.cs +++ b/libs/cluster/Server/Replication/PrimaryOps/DisklessReplication/ReplicationSyncManager.cs @@ -40,12 +40,11 @@ public ReplicationSyncManager(ClusterProvider clusterProvider, ILogger logger = public void Dispose() { - // Return if original value is true, hence already disposed - disposed.WriteLock(); cts?.Cancel(); + syncInProgress.WriteLock(); + disposed.WriteLock(); cts?.Dispose(); cts = null; - syncInProgress.WriteLock(); } /// @@ -92,7 +91,14 @@ public async Task WaitForFlushAsync() /// public bool AddReplicaSyncSession(SyncMetadata replicaSyncMetadata, out ReplicaSyncSession replicaSyncSession) { - replicaSyncSession = new ReplicaSyncSession(ClusterProvider.storeWrapper, ClusterProvider, replicaSyncMetadata, cts.Token, logger: logger); + replicaSyncSession = new ReplicaSyncSession( + ClusterProvider.storeWrapper, + ClusterProvider, + replicaAofBeginAddress: default, + replicaAofTailAddress: default, + replicaSyncMetadata, + cts.Token, + logger: logger); replicaSyncSession.SetStatus(SyncStatus.INITIALIZING); try { @@ -127,12 +133,14 @@ public async Task ReplicationSyncDriverAsync(ReplicaSyncSession // This will be the task added first in the replica sync session array. if (isLeader) { + using SemaphoreSlim signalCompletion = new(0); // Launch a background task to sync the attached replicas using streaming snapshot - _ = Task.Run(MainStreamingSnapshotDriverAsync); + _ = Task.Run(() => MainStreamingSnapshotDriverAsync(signalCompletion)); + await signalCompletion.WaitAsync().ConfigureAwait(false); } // Wait for main sync driver to complete - await replicaSyncSession.WaitForSyncCompletionAsync(); + await replicaSyncSession.WaitForSyncCompletionAsync().ConfigureAwait(false); // If session faulted return early if (replicaSyncSession.Failed) @@ -144,7 +152,7 @@ public async Task ReplicationSyncDriverAsync(ReplicaSyncSession } // Start AOF sync background task for this replica - await replicaSyncSession.BeginAofSyncAsync(); + await replicaSyncSession.BeginAofSyncAsync().ConfigureAwait(false); return replicaSyncSession.GetSyncStatusInfo; } finally @@ -154,18 +162,22 @@ public async Task ReplicationSyncDriverAsync(ReplicaSyncSession } /// - /// Streaming snapshot driver + /// Coordinates the main streaming snapshot synchronization process across replica sessions. /// - /// - async Task MainStreamingSnapshotDriverAsync() + /// A semaphore used to signal completion of the main synchronization task. The method releases this semaphore + /// when the synchronization process finishes. + /// A task that represents the asynchronous operation of the streaming snapshot synchronization driver. + /// Thrown if the streaming checkpoint operation fails during synchronization. + async Task MainStreamingSnapshotDriverAsync(SemaphoreSlim signalCompletion) { // Parameters for sync operation - var disableObjects = ClusterProvider.serverOptions.DisableObjects; - + var syncInProgressAcquired = false; try { // Lock to avoid the addition of new replica sync sessions while sync is in progress - syncInProgress.WriteLock(); + syncInProgressAcquired = syncInProgress.TryWriteLock(); + if (!syncInProgressAcquired) + throw new GarnetException("Failed to acquire write syncInProgress lock!"); // Get sync session info NumSessions = GetSessionStore.GetNumSessions(); @@ -183,7 +195,7 @@ async Task MainStreamingSnapshotDriverAsync() await Task.Yield(); // Choose to perform a full sync or not - var fullSync = await PrepareForSyncAsync(); + var fullSync = await PrepareForSyncAsync().ConfigureAwait(false); // If at least one replica requires a full sync, take a streaming checkpoint // NOTE: @@ -191,7 +203,7 @@ async Task MainStreamingSnapshotDriverAsync() // It is possible that some replicas may not require a full sync and can continue with partial sync. // See #chooseBetweenFullAndPartialSync if (fullSync) - await TakeStreamingCheckpointAsync(); + await TakeStreamingCheckpointAsync().ConfigureAwait(false); // Notify sync session of success success for (var i = 0; i < NumSessions; i++) @@ -212,7 +224,11 @@ async Task MainStreamingSnapshotDriverAsync() ClusterProvider.storeWrapper.ResumeCheckpoints(); // Unlock sync session lock - syncInProgress.WriteUnlock(); + if (syncInProgressAcquired) + syncInProgress.WriteUnlock(); + + // Release to indicate completion of the main sync task + signalCompletion.Release(); } // Acquire checkpoint and lock AOF if possible @@ -223,12 +239,12 @@ async Task PrepareForSyncAsync() while (true) { // Minimum address that we can serve assuming aof-locking and no aof-null-device - var minServiceableAofAddress = ClusterProvider.storeWrapper.appendOnlyFile.BeginAddress; + var minServiceableAofAddress = ClusterProvider.storeWrapper.appendOnlyFile.Log.BeginAddress; // Lock AOF address for sync streaming // If clusterProvider.allowDataLoss is set the addition never fails, // otherwise failure occurs if AOF has been truncated beyond minServiceableAofAddress - if (ClusterProvider.replicationManager.TryAddReplicationTasks(GetSessionStore.GetSessions(), minServiceableAofAddress)) + if (ClusterProvider.replicationManager.AofSyncDriverStore.TryAddReplicationDrivers(GetSessionStore.GetSessions(), ref minServiceableAofAddress)) break; // Retry if failed to lock AOF address because truncation occurred @@ -243,11 +259,10 @@ async Task PrepareForSyncAsync() try { // Initialize connections - Sessions[i].Connect(); + await Sessions[i].ConnectAsync().ConfigureAwait(false); // Set store version to operate on Sessions[i].currentStoreVersion = ClusterProvider.storeWrapper.store.CurrentVersion; - Sessions[i].currentObjectStoreVersion = disableObjects ? -1 : ClusterProvider.storeWrapper.objectStore.CurrentVersion; // If checkpoint is not needed mark this sync session as complete // to avoid waiting for other replicas which may need to receive the latest checkpoint @@ -259,7 +274,7 @@ async Task PrepareForSyncAsync() else { // Reset replica database in preparation for full sync - Sessions[i].SetFlushTask(Sessions[i].ExecuteAsync(["CLUSTER", "FLUSHALL"])); + Sessions[i].SetFlushTask(Sessions[i].IssueFlushAllAsync()); fullSync = true; } } @@ -280,27 +295,17 @@ async Task PrepareForSyncAsync() // Stream Diskless async Task TakeStreamingCheckpointAsync() { - // Main snapshot iterator manager + // Store snapshot iterator manager var manager = new SnapshotIteratorManager(this, cts.Token, logger); - // Iterate through main store + // Iterate through store var mainStoreCheckpointTask = ClusterProvider.storeWrapper.store. - TakeFullCheckpointAsync(CheckpointType.StreamingSnapshot, cancellationToken: cts.Token, streamingSnapshotIteratorFunctions: manager.mainStoreSnapshotIterator); + TakeFullCheckpointAsync(CheckpointType.StreamingSnapshot, cancellationToken: cts.Token, streamingSnapshotIteratorFunctions: manager.StoreSnapshotIterator); - var result = await WaitOrDieAsync(checkpointTask: mainStoreCheckpointTask, iteratorManager: manager); - if (!result.success) + var (success, _) = await WaitOrDieAsync(checkpointTask: mainStoreCheckpointTask, iteratorManager: manager).ConfigureAwait(false); + if (!success) throw new GarnetException("Main store checkpoint stream failed!"); - if (!ClusterProvider.serverOptions.DisableObjects) - { - // Iterate through object store - var objectStoreCheckpointTask = ClusterProvider.storeWrapper.objectStore. - TakeFullCheckpointAsync(CheckpointType.StreamingSnapshot, cancellationToken: cts.Token, streamingSnapshotIteratorFunctions: manager.objectStoreSnapshotIterator); - result = await WaitOrDieAsync(checkpointTask: objectStoreCheckpointTask, iteratorManager: manager); - if (!result.success) - throw new GarnetException("Object store checkpoint stream failed!"); - } - // Note: We do not truncate the AOF here as this was just a "virtual" checkpoint // WaitOrDie is needed here to check if streaming checkpoint is making progress. // We cannot use a timeout on the cancellationToken because we don't know in total how long the streaming checkpoint will take diff --git a/libs/cluster/Server/Replication/PrimaryOps/PrimarySync.cs b/libs/cluster/Server/Replication/PrimaryOps/PrimarySync.cs index c5352435477..bb77eca60b9 100644 --- a/libs/cluster/Server/Replication/PrimaryOps/PrimarySync.cs +++ b/libs/cluster/Server/Replication/PrimaryOps/PrimarySync.cs @@ -4,6 +4,11 @@ using System; using System.Text; using System.Threading.Tasks; + +#if DEBUG +using Garnet.common; +#endif +using Garnet.server; using Microsoft.Extensions.Logging; namespace Garnet.cluster @@ -17,7 +22,7 @@ internal sealed partial class ReplicationManager : IDisposable /// /// /// - public async Task<(bool Success, ReadOnlyMemory ErrorMessage)> TryAttachSyncAsync(SyncMetadata replicaSyncMetadata) + public async Task<(bool Success, ReadOnlyMemory ErrorMessage)> TryBeginDisklessSyncAsync(SyncMetadata replicaSyncMetadata) { ReadOnlyMemory errorMessage = default; if (clusterProvider.serverOptions.ReplicaDisklessSync) @@ -29,6 +34,10 @@ internal sealed partial class ReplicationManager : IDisposable logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage.Span)); } +#if DEBUG + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync).ConfigureAwait(false); +#endif + var status = await replicationSyncManager.ReplicationSyncDriverAsync(replicaSyncSession).ConfigureAwait(false); if (status.syncStatus == SyncStatus.FAILED) errorMessage = Encoding.ASCII.GetBytes(status.error); @@ -37,25 +46,6 @@ internal sealed partial class ReplicationManager : IDisposable return (true, errorMessage); } - /// - /// Start sync of remote replica from this primary - /// - /// - /// - /// - /// - /// - /// - public Task<(bool Success, ReadOnlyMemory ErrorMessage)> TryBeginPrimarySyncAsync( - string replicaNodeId, - string replicaAssignedPrimaryId, - CheckpointEntry replicaCheckpointEntry, - long replicaAofBeginAddress, - long replicaAofTailAddress) - { - return TryBeginDiskSyncAsync(replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, replicaAofBeginAddress, replicaAofTailAddress); - } - /// /// Begin background replica sync session /// @@ -65,18 +55,20 @@ internal sealed partial class ReplicationManager : IDisposable /// AOF begin address at replica /// AOF tail address at replica /// - public Task<(bool Success, ReadOnlyMemory ErrorMessage)> TryBeginDiskSyncAsync( + public Task<(bool Success, ReadOnlyMemory ErrorMessage)> TryBeginDiskbasedSyncAsync( string replicaNodeId, string replicaAssignedPrimaryId, CheckpointEntry replicaCheckpointEntry, - long replicaAofBeginAddress, - long replicaAofTailAddress) + AofAddress replicaAofBeginAddress, + AofAddress replicaAofTailAddress) { + ReadOnlyMemory errorMessage = default; + if (!replicaSyncSessionTaskStore.TryAddReplicaSyncSession(replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, replicaAofBeginAddress, replicaAofTailAddress)) { - var errorMessage = CmdStrings.RESP_ERR_CREATE_SYNC_SESSION_ERROR.ToArray(); - logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage)); - return Task.FromResult((false, (ReadOnlyMemory)errorMessage.AsMemory())); + errorMessage = CmdStrings.RESP_ERR_CREATE_SYNC_SESSION_ERROR.ToArray(); + logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage.Span)); + return Task.FromResult((false, errorMessage)); } return ReplicaSyncSessionBackgroundTaskAsync(replicaNodeId); @@ -85,24 +77,32 @@ internal sealed partial class ReplicationManager : IDisposable { try { + ReadOnlyMemory errorMessage = default; + if (!replicaSyncSessionTaskStore.TryGetSession(replicaId, out var session)) { - var errorMessage = CmdStrings.RESP_ERR_RETRIEVE_SYNC_SESSION_ERROR.ToArray(); - logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage)); - return (false, (ReadOnlyMemory)errorMessage.AsMemory()); + errorMessage = CmdStrings.RESP_ERR_RETRIEVE_SYNC_SESSION_ERROR.ToArray(); + logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage.Span)); + return (false, errorMessage); } +#if DEBUG + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync).ConfigureAwait(false); +#endif + if (!await session.SendCheckpointAsync().ConfigureAwait(false)) { - var errorMessage = Encoding.ASCII.GetBytes(session.errorMsg); - return (false, (ReadOnlyMemory)errorMessage.AsMemory()); + errorMessage = Encoding.ASCII.GetBytes(session.errorMsg); + return (false, errorMessage); } - return (true, default); + errorMessage = CmdStrings.RESP_OK.ToArray(); + return (true, errorMessage); } finally { - _ = replicaSyncSessionTaskStore.TryRemove(replicaId); + if (!replicaSyncSessionTaskStore.TryRemove(replicaId)) + logger?.LogError("Unable to remove replica sync session for remote node {replicaId}", replicaId); } } } diff --git a/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSession.cs b/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSession.cs deleted file mode 100644 index ac4bb013f5f..00000000000 --- a/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSession.cs +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Net; -using System.Threading; -using System.Threading.Tasks; -using Garnet.client; -using Garnet.common; -using Garnet.server; -using Microsoft.Extensions.Logging; -using Tsavorite.core; - -namespace Garnet.cluster -{ - internal sealed partial class ReplicaSyncSession( - StoreWrapper storeWrapper, - ClusterProvider clusterProvider, - SyncMetadata replicaSyncMetadata = null, - CancellationToken token = default, - string replicaNodeId = null, - string replicaAssignedPrimaryId = null, - CheckpointEntry replicaCheckpointEntry = null, - long replicaAofBeginAddress = 0, - long replicaAofTailAddress = 0, - ILogger logger = null) : IDisposable - { - readonly StoreWrapper storeWrapper = storeWrapper; - readonly ClusterProvider clusterProvider = clusterProvider; - public readonly SyncMetadata replicaSyncMetadata = replicaSyncMetadata; - readonly CancellationToken token = token; - readonly CancellationTokenSource cts = new(); - SectorAlignedBufferPool bufferPool = null; - readonly SemaphoreSlim signalCompletion = new(0); - - public readonly string replicaNodeId = replicaNodeId; - public readonly string replicaAssignedPrimaryId = replicaAssignedPrimaryId; - private readonly long replicaAofBeginAddress = replicaAofBeginAddress; - private readonly long replicaAofTailAddress = replicaAofTailAddress; - - private readonly CheckpointEntry replicaCheckpointEntry = replicaCheckpointEntry; - - private readonly ILogger logger = logger; - - public string errorMsg = default; - - const int validateMetadataMaxRetryCount = 10; - - public void Dispose() - { - AofSyncTask?.garnetClient?.Dispose(); - AofSyncTask = null; - cts.Cancel(); - cts.Dispose(); - signalCompletion?.Dispose(); - bufferPool?.Free(); - } - - public bool ValidateMetadata( - CheckpointEntry localEntry, - out long index_size, - out LogFileInfo hlog_size, - out long obj_index_size, - out LogFileInfo obj_hlog_size, - out bool skipLocalMainStoreCheckpoint, - out bool skipLocalObjectStoreCheckpoint) - { - hlog_size = default; - obj_hlog_size = default; - index_size = -1L; - obj_index_size = -1L; - - // Local and remote checkpoints are of same history if both of the following hold - // 1. There is a checkpoint available at remote node - // 2. Remote and local checkpoints contain the same PrimaryReplId - var sameMainStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.storePrimaryReplId) && replicaCheckpointEntry.metadata.storePrimaryReplId.Equals(localEntry.metadata.storePrimaryReplId); - var sameObjectStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.objectStorePrimaryReplId) && replicaCheckpointEntry.metadata.objectStorePrimaryReplId.Equals(localEntry.metadata.objectStorePrimaryReplId); - // We will not send the latest local checkpoint if any of the following hold - // 1. Local node does not have any checkpoints - // 2. Local checkpoint is of same version and history as the remote checkpoint - skipLocalMainStoreCheckpoint = localEntry.metadata.storeHlogToken == default || (sameMainStoreCheckpointHistory && localEntry.metadata.storeVersion == replicaCheckpointEntry.metadata.storeVersion); - skipLocalObjectStoreCheckpoint = clusterProvider.serverOptions.DisableObjects || localEntry.metadata.objectStoreHlogToken == default || (sameObjectStoreCheckpointHistory && localEntry.metadata.objectStoreVersion == replicaCheckpointEntry.metadata.objectStoreVersion); - - // Acquire metadata for main store - // If failed then this checkpoint is not usable because it is corrupted - if (!skipLocalMainStoreCheckpoint && !clusterProvider.replicationManager.TryAcquireSettledMetadataForMainStore(localEntry, out hlog_size, out index_size)) - return false; - - // Acquire metadata for object store - // If failed then this checkpoint is not usable because it is corrupted - if (!skipLocalObjectStoreCheckpoint && !clusterProvider.replicationManager.TryAcquireSettledMetadataForObjectStore(localEntry, out obj_hlog_size, out obj_index_size)) - return false; - - return true; - } - - /// - /// Start sending the latest checkpoint to replica - /// - public async Task SendCheckpointAsync() - { - errorMsg = default; - var storeCkptManager = clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main); - var objectStoreCkptManager = clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object); - var current = clusterProvider.clusterManager.CurrentConfig; - var (address, port) = current.GetWorkerAddressFromNodeId(replicaNodeId); - - if (address == null || port == -1) - { - errorMsg = $"PRIMARY-ERR don't know about replicaId: {replicaNodeId}"; - logger?.LogError("{errorMsg}", errorMsg); - return false; - } - - GarnetClientSession gcs = new( - new IPEndPoint(IPAddress.Parse(address), port), - clusterProvider.replicationManager.GetRSSNetworkBufferSettings, - clusterProvider.replicationManager.GetNetworkPool, - tlsOptions: clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, - authUsername: clusterProvider.ClusterUsername, - authPassword: clusterProvider.ClusterPassword, - logger: logger); - CheckpointEntry localEntry = default; - AofSyncTaskInfo aofSyncTaskInfo = null; - - try - { - logger?.LogInformation("Replica replicaId:{replicaId} requesting checkpoint replicaStoreVersion:{replicaStoreVersion} replicaObjectStoreVersion:{replicaObjectStoreVersion}", - replicaNodeId, replicaCheckpointEntry.metadata.storeVersion, replicaCheckpointEntry.metadata.objectStoreVersion); - - logger?.LogInformation("Attempting to acquire checkpoint"); - (localEntry, aofSyncTaskInfo) = await AcquireCheckpointEntryAsync().ConfigureAwait(false); - logger?.LogInformation("Checkpoint search completed"); - - await gcs.ConnectAsync((int)storeWrapper.serverOptions.ReplicaSyncTimeout.TotalMilliseconds).ConfigureAwait(false); - - long index_size = -1; - long obj_index_size = -1; - var hlog_size = default(LogFileInfo); - var obj_hlog_size = default(LogFileInfo); - var skipLocalMainStoreCheckpoint = false; - var skipLocalObjectStoreCheckpoint = false; - var retryCount = validateMetadataMaxRetryCount; - while (!ValidateMetadata(localEntry, out index_size, out hlog_size, out obj_index_size, out obj_hlog_size, out skipLocalMainStoreCheckpoint, out skipLocalObjectStoreCheckpoint)) - { - logger?.LogError("Failed to validate metadata. Retrying...."); - await Task.Yield(); - if (retryCount-- <= 0) - throw new GarnetException("Failed to validate metadata!"); - } - - #region sendStoresSnapshotData - if (!skipLocalMainStoreCheckpoint) - { - logger?.LogInformation("Sending main store checkpoint {version} {storeHlogToken} {storeIndexToken} to replica", localEntry.metadata.storeVersion, localEntry.metadata.storeHlogToken, localEntry.metadata.storeIndexToken); - - // 1. send hlog file segments - if (clusterProvider.serverOptions.EnableStorageTier && hlog_size.hybridLogFileEndAddress > 64) - await SendFileSegmentsAsync(gcs, localEntry.metadata.storeHlogToken, CheckpointFileType.STORE_HLOG, hlog_size.hybridLogFileStartAddress, hlog_size.hybridLogFileEndAddress).ConfigureAwait(false); - - // 2.Send index file segments - //var index_size = storeWrapper.store.GetIndexFileSize(localEntry.storeIndexToken); - await SendFileSegmentsAsync(gcs, localEntry.metadata.storeIndexToken, CheckpointFileType.STORE_INDEX, 0, index_size).ConfigureAwait(false); - - // 3. Send snapshot file segments - await SendFileSegmentsAsync(gcs, localEntry.metadata.storeHlogToken, CheckpointFileType.STORE_SNAPSHOT, 0, hlog_size.snapshotFileEndAddress).ConfigureAwait(false); - - // 4. Send delta log segments - var dlog_size = hlog_size.deltaLogTailAddress; - await SendFileSegmentsAsync(gcs, localEntry.metadata.storeHlogToken, CheckpointFileType.STORE_DLOG, 0, dlog_size).ConfigureAwait(false); - - // 5.Send index metadata - await SendCheckpointMetadataAsync(gcs, storeCkptManager, CheckpointFileType.STORE_INDEX, localEntry.metadata.storeIndexToken).ConfigureAwait(false); - - // 6. Send snapshot metadata - await SendCheckpointMetadataAsync(gcs, storeCkptManager, CheckpointFileType.STORE_SNAPSHOT, localEntry.metadata.storeHlogToken).ConfigureAwait(false); - } - - if (!skipLocalObjectStoreCheckpoint) - { - logger?.LogInformation("Sending object store checkpoint {version} {objectStoreHlogToken} {objectStoreIndexToken} to replica", localEntry.metadata.objectStoreVersion, localEntry.metadata.objectStoreHlogToken, localEntry.metadata.objectStoreIndexToken); - - // 1. send hlog file segments - if (clusterProvider.serverOptions.EnableStorageTier && obj_hlog_size.hybridLogFileEndAddress > 24) - { - //send object hlog file segments - await SendFileSegmentsAsync(gcs, localEntry.metadata.objectStoreHlogToken, CheckpointFileType.OBJ_STORE_HLOG, obj_hlog_size.hybridLogFileStartAddress, obj_hlog_size.hybridLogFileEndAddress).ConfigureAwait(false); - - var hlogSegmentCount = ((obj_hlog_size.hybridLogFileEndAddress - obj_hlog_size.hybridLogFileStartAddress) >> clusterProvider.serverOptions.ObjectStoreSegmentSizeBits()) + 1; - await SendObjectFilesAsync(gcs, localEntry.metadata.objectStoreHlogToken, CheckpointFileType.OBJ_STORE_HLOG_OBJ, (int)hlogSegmentCount).ConfigureAwait(false); - } - - // 2. Send object store snapshot files - if (obj_hlog_size.snapshotFileEndAddress > 24) - { - //send snapshot file segments - await SendFileSegmentsAsync(gcs, localEntry.metadata.objectStoreHlogToken, CheckpointFileType.OBJ_STORE_SNAPSHOT, 0, obj_hlog_size.snapshotFileEndAddress).ConfigureAwait(false); - - //send snapshot.obj file segments - var snapshotSegmentCount = (obj_hlog_size.snapshotFileEndAddress >> clusterProvider.serverOptions.ObjectStoreSegmentSizeBits()) + 1; - await SendObjectFilesAsync(gcs, localEntry.metadata.objectStoreHlogToken, CheckpointFileType.OBJ_STORE_SNAPSHOT_OBJ, (int)snapshotSegmentCount).ConfigureAwait(false); - } - - // 3. Send object store index file segments - if (obj_index_size > 0) - await SendFileSegmentsAsync(gcs, localEntry.metadata.objectStoreIndexToken, CheckpointFileType.OBJ_STORE_INDEX, 0, obj_index_size).ConfigureAwait(false); - - // 4. Send object store delta file segments - var obj_dlog_size = obj_hlog_size.deltaLogTailAddress; - if (obj_dlog_size > 0) - await SendFileSegmentsAsync(gcs, localEntry.metadata.objectStoreHlogToken, CheckpointFileType.OBJ_STORE_DLOG, 0, obj_dlog_size).ConfigureAwait(false); - - // 5. Send object store index metadata - await SendCheckpointMetadataAsync(gcs, objectStoreCkptManager, CheckpointFileType.OBJ_STORE_INDEX, localEntry.metadata.objectStoreIndexToken).ConfigureAwait(false); - - // 6. Send object store snapshot metadata - await SendCheckpointMetadataAsync(gcs, objectStoreCkptManager, CheckpointFileType.OBJ_STORE_SNAPSHOT, localEntry.metadata.objectStoreHlogToken).ConfigureAwait(false); - } - #endregion - - #region startAofSync - var recoverFromRemote = !skipLocalMainStoreCheckpoint || !skipLocalObjectStoreCheckpoint; - var replayAOF = false; - var checkpointAofBeginAddress = localEntry.GetMinAofCoveredAddress(); - var beginAddress = checkpointAofBeginAddress; - if (!recoverFromRemote) - { - // If replica is ahead of this primary it will force itself to forget and start syncing from RecoveredReplicationOffset - if (replicaAofBeginAddress > ReplicationManager.kFirstValidAofAddress && replicaAofBeginAddress > checkpointAofBeginAddress) - { - logger?.LogInformation( - "ReplicaSyncSession: replicaAofBeginAddress {replicaAofBeginAddress} > PrimaryCheckpointRecoveredReplicationOffset {RecoveredReplicationOffset}, cannot use remote AOF", - replicaAofBeginAddress, checkpointAofBeginAddress); - } - else - { - // Tail address cannot be behind the recovered address since above we checked replicaAofBeginAddress and it appears after RecoveredReplicationOffset - // unless we are performing MainMemoryReplication - // TODO: shouldn't we use the remote cEntry's tail address here since replica will recover to that? - if (replicaAofTailAddress < checkpointAofBeginAddress && !clusterProvider.serverOptions.FastAofTruncate) - { - logger?.LogCritical("ReplicaSyncSession replicaAofTail {replicaAofTailAddress} < canServeFromAofAddress {RecoveredReplicationOffset}", replicaAofTailAddress, checkpointAofBeginAddress); - throw new Exception($"ReplicaSyncSession replicaAofTail {replicaAofTailAddress} < canServeFromAofAddress {checkpointAofBeginAddress}"); - } - - // If we are behind this primary we need to decide until where to replay - var replayUntilAddress = replicaAofTailAddress; - // Replica tail is further ahead than committed address of primary - if (storeWrapper.appendOnlyFile.CommittedUntilAddress < replayUntilAddress) - { - replayUntilAddress = storeWrapper.appendOnlyFile.CommittedUntilAddress; - } - - // Replay only if records not included in checkpoint - if (replayUntilAddress > checkpointAofBeginAddress) - { - logger?.LogInformation("ReplicaSyncSession: have to replay remote AOF from {beginAddress} until {untilAddress}", beginAddress, replayUntilAddress); - replayAOF = true; - // Bound replayUntilAddress to ReplicationOffset2 to avoid replaying divergent history only if connecting replica was attached to old primary - if (!string.IsNullOrEmpty(clusterProvider.replicationManager.PrimaryReplId2) && - clusterProvider.replicationManager.PrimaryReplId2.Equals(replicaAssignedPrimaryId) && - replayUntilAddress > clusterProvider.replicationManager.ReplicationOffset2) - replayUntilAddress = clusterProvider.replicationManager.ReplicationOffset2; - checkpointAofBeginAddress = replayUntilAddress; - } - - var sameMainStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.storePrimaryReplId) && replicaCheckpointEntry.metadata.storePrimaryReplId.Equals(localEntry.metadata.storePrimaryReplId); - var sameObjectStoreCheckpointHistory = !string.IsNullOrEmpty(replicaCheckpointEntry.metadata.objectStorePrimaryReplId) && replicaCheckpointEntry.metadata.objectStorePrimaryReplId.Equals(localEntry.metadata.objectStorePrimaryReplId); - if (!sameMainStoreCheckpointHistory || !sameObjectStoreCheckpointHistory) - { - // If we are not in the same checkpoint history, we need to stream the AOF from the primary's beginning address - checkpointAofBeginAddress = beginAddress; - replayAOF = false; - logger?.LogInformation("ReplicaSyncSession: not in same checkpoint history, will replay from beginning address {checkpointAofBeginAddress}", checkpointAofBeginAddress); - } - } - } - - // Signal replica to recover from local/remote checkpoint - // Make replica replayAOF if needed and replay from provided beginAddress to RecoveredReplication Address - var resp = await gcs.ExecuteBeginReplicaRecover( - !skipLocalMainStoreCheckpoint, - !skipLocalObjectStoreCheckpoint, - replayAOF, - clusterProvider.replicationManager.PrimaryReplId, - localEntry.ToByteArray(), - beginAddress, - checkpointAofBeginAddress).WaitAsync(storeWrapper.serverOptions.ReplicaSyncTimeout, cts.Token).ConfigureAwait(false); - var syncFromAofAddress = long.Parse(resp); - - // Assert that AOF address the replica will be requesting can be served, except in case of: - // Possible AOF data loss: { using null AOF device } OR { main memory replication AND no on-demand checkpoints } - var possibleAofDataLoss = clusterProvider.serverOptions.UseAofNullDevice || - (clusterProvider.serverOptions.FastAofTruncate && !clusterProvider.serverOptions.OnDemandCheckpoint); - - if (!possibleAofDataLoss) - { - if (syncFromAofAddress < storeWrapper.appendOnlyFile.BeginAddress) - { - logger?.LogError("syncFromAofAddress: {syncFromAofAddress} < beginAofAddress: {storeWrapper.appendOnlyFile.BeginAddress}", syncFromAofAddress, storeWrapper.appendOnlyFile.BeginAddress); - logger?.LogCheckpointEntry(LogLevel.Error, "Requested replay address truncated", localEntry); - throw new Exception("Failed syncing because replica requested truncated AOF address"); - } - } - else // possible AOF data loss - { - if (syncFromAofAddress < storeWrapper.appendOnlyFile.BeginAddress) - { - logger?.LogWarning("AOF truncated, unsafe attach: syncFromAofAddress: {syncFromAofAddress} < beginAofAddress: {storeWrapper.appendOnlyFile.BeginAddress}", syncFromAofAddress, storeWrapper.appendOnlyFile.BeginAddress); - logger?.LogCheckpointEntry(LogLevel.Warning, "Unsafe replay due to truncated AOF address", localEntry); - } - } - - // Check what happens if we fail after recovery and start AOF stream - ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Replication_Fail_Before_Background_AOF_Stream_Task_Start); - - // We have already added the iterator for the covered address above but replica might request an address - // that is ahead of the covered address so we should start streaming from that address in order not to - // introduce duplicate insertions. - if (!clusterProvider.replicationManager.TryAddReplicationTask(replicaNodeId, syncFromAofAddress, out aofSyncTaskInfo)) - throw new GarnetException("Failed trying to try update replication task"); - if (!clusterProvider.replicationManager.TryConnectToReplica(replicaNodeId, syncFromAofAddress, aofSyncTaskInfo, out _)) - throw new GarnetException("Failed connecting to replica for aofSync"); - #endregion - } - catch (Exception ex) - { - if (localEntry != null) - logger?.LogCheckpointEntry(LogLevel.Error, "Error at attaching", localEntry); - else - logger?.LogError("Error at attaching: {ex}", ex.Message); - - if (aofSyncTaskInfo != null) _ = clusterProvider.replicationManager.TryRemoveReplicationTask(aofSyncTaskInfo); - errorMsg = ex.Message;// this is error sent to remote client - return false; - } - finally - { - // At this point the replica has received the most recent checkpoint data - // and recovered from it so primary can release and delete it safely - localEntry?.RemoveReader(); - gcs.Dispose(); - } - return true; - } - - public async Task<(CheckpointEntry, AofSyncTaskInfo)> AcquireCheckpointEntryAsync() - { - AofSyncTaskInfo aofSyncTaskInfo; - CheckpointEntry cEntry; - - // This loop tries to provide the following two guarantees - // 1. Retrieve latest checkpoint and lock it to prevent deletion before it is send to the replica - // 2. Guard against truncation of AOF in between the retrieval of the checkpoint metadata and start of the aofSyncTask - var iteration = 0; - var numOdcAttempts = 0; - const int maxOdcAttempts = 2; - while (true) - { - logger?.LogInformation("AcquireCheckpointEntry iteration {iteration}", iteration); - iteration++; - - aofSyncTaskInfo = null; - cEntry = default; - - // Acquire startSaveTime to identify if an external task might have taken the checkpoint for us - // This is only useful for MainMemoryReplication where we might have multiple replicas attaching - // We want to share the on-demand checkpoint and ensure that only one replica should succeed when calling TakeOnDemandCheckpoint - var lastSaveTime = storeWrapper.lastSaveTime; - - var exceptionInjected = ExceptionInjectionHelper.TriggerCondition(ExceptionInjectionType.Replication_Acquire_Checkpoint_Entry_Fail_Condition); - - // Retrieve latest checkpoint and lock it from deletion operations - var addedReader = !exceptionInjected && clusterProvider.replicationManager.TryGetLatestCheckpointEntryFromMemory(out cEntry); - - if (!addedReader) - { - // Fail to acquire lock, could mean that a writer might be trying to delete - logger?.LogWarning("Could not acquire lock for existing checkpoint, retrying."); - - // Go back to re-acquire the latest checkpoint - await Task.Yield(); - continue; - } - -#if DEBUG - // Only on Debug mode - await ExceptionInjectionHelper.WaitOnSetAsync(ExceptionInjectionType.Replication_Wait_After_Checkpoint_Acquisition).ConfigureAwait(false); -#endif - - // Calculate the minimum start address covered by this checkpoint - var startAofAddress = cEntry.GetMinAofCoveredAddress(); - - // If there is possible AOF data loss and we need to take an on-demand checkpoint, - // then we should take the checkpoint before we register the sync task, because - // TryAddReplicationTask is guaranteed to return true in this scenario. - var validMetadata = ValidateMetadata(cEntry, out _, out _, out _, out _, out _, out _); - if (clusterProvider.serverOptions.OnDemandCheckpoint && - (startAofAddress < clusterProvider.replicationManager.AofTruncatedUntil || !validMetadata)) - { - if (numOdcAttempts >= maxOdcAttempts && clusterProvider.AllowDataLoss) - { - logger?.LogWarning("Failed to acquire checkpoint after {numOdcAttempts} on-demand checkpoint attempts. Possible data loss, startAofAddress:{startAofAddress} < truncatedUntil:{truncatedUntil}.", numOdcAttempts, startAofAddress, clusterProvider.replicationManager.AofTruncatedUntil); - } - else - { - cEntry.RemoveReader(); - numOdcAttempts++; - logger?.LogInformation("Taking on-demand checkpoint, attempt {numOdcAttempts}.", numOdcAttempts); - await storeWrapper.TakeOnDemandCheckpointAsync(lastSaveTime); - await Task.Yield(); - continue; - } - } - - // Enqueue AOF sync task with startAofAddress to prevent future AOF truncations - // and check if truncation has happened in between retrieving the latest checkpoint and enqueuing the aofSyncTask - if (clusterProvider.replicationManager.TryAddReplicationTask(replicaNodeId, startAofAddress, out aofSyncTaskInfo)) - break; - - // Unlock last checkpoint because associated startAofAddress is no longer available - cEntry.RemoveReader(); - - // Go back to re-acquire checkpoint - await Task.Yield(); - } - - return (cEntry, aofSyncTaskInfo); - } - - private async Task SendCheckpointMetadataAsync(GarnetClientSession gcs, GarnetClusterCheckpointManager ckptManager, CheckpointFileType fileType, Guid fileToken) - { - var retryCount = validateMetadataMaxRetryCount; - while (true) - { - try - { - logger?.LogInformation("(); - if (fileToken != default) - { - switch (fileType) - { - case CheckpointFileType.STORE_SNAPSHOT: - case CheckpointFileType.OBJ_STORE_SNAPSHOT: - checkpointMetadata = ckptManager.GetLogCheckpointMetadata(fileToken, null, true, -1); - break; - case CheckpointFileType.STORE_INDEX: - case CheckpointFileType.OBJ_STORE_INDEX: - checkpointMetadata = ckptManager.GetIndexCheckpointMetadata(fileToken); - break; - } - } - - var resp = await gcs.ExecuteSendCkptMetadata(fileToken.ToByteArray(), (int)fileType, checkpointMetadata).WaitAsync(storeWrapper.serverOptions.ReplicaSyncTimeout, cts.Token).ConfigureAwait(false); - if (!resp.Equals("OK")) - { - logger?.LogError("Primary error at SendCheckpointMetadata {resp}", resp); - throw new Exception($"Primary error at SendCheckpointMetadata {resp}"); - } - - logger?.LogInformation(" - /// Note: will read potentially more data (based on sector alignment) - /// - /// - /// - /// - /// - private async Task<(SectorAlignedMemory, int)> ReadIntoAsync(IDevice device, ulong address, int size, int segmentId = -1) - { - bufferPool ??= new SectorAlignedBufferPool(1, (int)device.SectorSize); - - long numBytesToRead = size; - numBytesToRead = ((numBytesToRead + (device.SectorSize - 1)) & ~(device.SectorSize - 1)); - - var pbuffer = bufferPool.Get((int)numBytesToRead); - unsafe - { - if (segmentId == -1) - device.ReadAsync(address, (IntPtr)pbuffer.aligned_pointer, (uint)numBytesToRead, IOCallback, null); - else - device.ReadAsync(segmentId, address, (IntPtr)pbuffer.aligned_pointer, (uint)numBytesToRead, IOCallback, null); - } - await signalCompletion.WaitAsync(storeWrapper.serverOptions.ReplicaSyncTimeout, cts.Token).ConfigureAwait(false); - return (pbuffer, (int)numBytesToRead); - } - - private unsafe void IOCallback(uint errorCode, uint numBytes, object context) - { - if (errorCode != 0) - { - var errorMessage = Tsavorite.core.Utility.GetCallbackErrorMessage(errorCode, numBytes, context); - logger?.LogError("[ReplicaSyncSession] OverlappedStream GetQueuedCompletionStatus error: {errorCode} msg: {errorMessage}", errorCode, errorMessage); - } - signalCompletion.Release(); - } - } - - internal static unsafe class SectorAlignedMemoryExtensions - { - public static Span GetSlice(this SectorAlignedMemory pbuffer, int length) - { - return new Span(pbuffer.aligned_pointer, length); - } - } -} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSessionTaskStore.cs b/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSessionTaskStore.cs index d92e12d7b1c..60fdaebebc0 100644 --- a/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSessionTaskStore.cs +++ b/libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSessionTaskStore.cs @@ -118,9 +118,9 @@ public bool TryAddReplicaSyncSession(ReplicaSyncSession session) /// /// /// - public bool TryAddReplicaSyncSession(string replicaNodeId, string replicaAssignedPrimaryId, CheckpointEntry replicaCheckpointEntry, long replicaAofBeginAddress, long replicaAofTailAddress) + public bool TryAddReplicaSyncSession(string replicaNodeId, string replicaAssignedPrimaryId, CheckpointEntry replicaCheckpointEntry, AofAddress replicaAofBeginAddress, AofAddress replicaAofTailAddress) { - var retSession = new ReplicaSyncSession(storeWrapper, clusterProvider, replicaSyncMetadata: null, token: default, replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, replicaAofBeginAddress, replicaAofTailAddress, logger); + var retSession = new ReplicaSyncSession(storeWrapper, clusterProvider, replicaAofBeginAddress, replicaAofTailAddress, replicaSyncMetadata: null, token: default, replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, logger); var success = false; try { @@ -160,7 +160,7 @@ public bool TryRemove(string remoteNodeId) { _lock.WriteLock(); if (_disposed) return true; - for (int i = 0; i < numSessions; i++) + for (var i = 0; i < numSessions; i++) { var s = sessions[i]; if (s.replicaNodeId == remoteNodeId) diff --git a/libs/cluster/Server/Replication/PrimaryOps/ReplicationPrimaryAofSync.cs b/libs/cluster/Server/Replication/PrimaryOps/ReplicationPrimaryAofSync.cs index a2a2d011152..7c5143bf3e1 100644 --- a/libs/cluster/Server/Replication/PrimaryOps/ReplicationPrimaryAofSync.cs +++ b/libs/cluster/Server/Replication/PrimaryOps/ReplicationPrimaryAofSync.cs @@ -12,47 +12,30 @@ namespace Garnet.cluster { internal sealed partial class ReplicationManager : IDisposable { - // Must be the same as the TsavoriteLog start address of allocator + // Must be the same as the TsavoriteAof start address of allocator public static readonly long kFirstValidAofAddress = 64; - readonly AofTaskStore aofTaskStore; + readonly AofSyncDriverStore aofSyncDriverStore; - public int ConnectedReplicasCount => aofTaskStore.CountConnectedReplicas(); + public AofSyncDriverStore AofSyncDriverStore => aofSyncDriverStore; - public List GetReplicaInfo() => aofTaskStore.GetReplicaInfo(ReplicationOffset); + public int ConnectedReplicasCount => aofSyncDriverStore.CountConnectedReplicas(); - public bool TryAddReplicationTask(string nodeid, long startAddress, out AofSyncTaskInfo aofSyncTaskInfo) - => aofTaskStore.TryAddReplicationTask(nodeid, startAddress, out aofSyncTaskInfo); - - public bool TryAddReplicationTasks(ReplicaSyncSession[] replicaSyncSessions, long startAddress) - => aofTaskStore.TryAddReplicationTasks(replicaSyncSessions, startAddress); - - public long AofTruncatedUntil => aofTaskStore.AofTruncatedUntil; - - public bool TryRemoveReplicationTask(AofSyncTaskInfo aofSyncTaskInfo) - => aofTaskStore.TryRemove(aofSyncTaskInfo); - - /// - /// Safely truncate iterator - /// - /// - /// - public long SafeTruncateAof(long CheckpointCoveredAofAddress) - => aofTaskStore.SafeTruncateAof(CheckpointCoveredAofAddress); + public List GetReplicaInfo() => aofSyncDriverStore.GetReplicaInfo(ReplicationOffset); /// /// Try to initiate connection from primary to replica in order to stream aof. /// /// /// - /// + /// /// The ASCII encoded error message if the method returned ; otherwise /// - public bool TryConnectToReplica(string nodeid, long startAddress, AofSyncTaskInfo aofSyncTaskInfo, out ReadOnlySpan errorMessage) + public bool TryConnectToReplica(string nodeid, ref AofAddress startAddress, AofSyncDriver aofSyncDriver, out ReadOnlySpan errorMessage) { errorMessage = default; if (_disposed) { - aofTaskStore.TryRemove(aofSyncTaskInfo); + aofSyncDriverStore.TryRemove(aofSyncDriver); errorMessage = "ERR Replication Manager Disposed"u8; logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage)); @@ -64,15 +47,15 @@ public bool TryConnectToReplica(string nodeid, long startAddress, AofSyncTaskInf var (address, port) = clusterProvider.clusterManager.CurrentConfig.GetWorkerAddressFromNodeId(nodeid); if (address == null) { - aofTaskStore.TryRemove(aofSyncTaskInfo); + aofSyncDriverStore.TryRemove(aofSyncDriver); errorMessage = Encoding.ASCII.GetBytes($"ERR unknown endpoint for {nodeid}"); logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage)); return false; } - var tailAddress = storeWrapper.appendOnlyFile.TailAddress; + var tailAddress = storeWrapper.appendOnlyFile.Log.TailAddress; // Check if requested AOF address goes beyond the maximum available AOF address of this primary - if (startAddress > storeWrapper.appendOnlyFile.TailAddress) + if (startAddress.AnyGreater(tailAddress)) { if (clusterProvider.serverOptions.FastAofTruncate) { @@ -80,14 +63,14 @@ public bool TryConnectToReplica(string nodeid, long startAddress, AofSyncTaskInf } else { - aofTaskStore.TryRemove(aofSyncTaskInfo); + aofSyncDriverStore.TryRemove(aofSyncDriver); logger?.LogError("AOF sync task failed to start. Requested address {startAddress} unavailable. Local primary tail address {tailAddress}", startAddress, tailAddress); errorMessage = Encoding.ASCII.GetBytes($"ERR requested AOF address: {startAddress} goes beyond, primary tail address: {tailAddress}"); return false; } } - _ = Task.Run(aofSyncTaskInfo.ReplicaSyncTaskAsync); + _ = Task.Run(() => aofSyncDriver.RunAsync()); return true; } } diff --git a/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriver.cs b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriver.cs new file mode 100644 index 00000000000..3f04bf1409f --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriver.cs @@ -0,0 +1,335 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Garnet.common; +using Garnet.networking; +using Garnet.server; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Replica main replay driver + /// + internal sealed class ReplicaReplayDriver : IBulkLogEntryConsumer, IDisposable + { + internal readonly int physicalSublogIdx; + readonly GarnetServerOptions serverOptions; + readonly GarnetAppendOnlyFile appendOnlyFile; + readonly ReplicationManager replicationManager; + readonly CancellationTokenSource cts; + readonly INetworkSender respSessionNetworkSender; + readonly ILogger logger; + TsavoriteLogScanSingleIterator replayIterator; + + readonly ActiveWorkerMonitor activeWorkerMonitor; + internal readonly ReplayBatchContext replayBatchContext; + readonly ReplicaReplayTask[] replayTasks; + readonly TsavoriteLog physicalSublog; + readonly bool useChannels = false; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool ResumeReplay() => activeWorkerMonitor.TryEnter(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SuspendReplay() => _ = activeWorkerMonitor.Exit(); + + /// + /// Initializes a new instance of the ReplicaReplayDriver class, setting up replay tasks for a specific physical + /// sublog. + /// + /// The index of the physical sublog to be replayed. + /// The cluster provider containing server options and storage components. + /// The network sender used for response sessions. + /// The cancellation token source for managing task cancellation. + /// The logger instance for logging operations, or null to disable logging. + public ReplicaReplayDriver(int physicalSublogIdx, ClusterProvider clusterProvider, INetworkSender respSessionNetworkSender, CancellationTokenSource cts, ILogger logger = null) + { + this.physicalSublogIdx = physicalSublogIdx; + this.respSessionNetworkSender = respSessionNetworkSender; + serverOptions = clusterProvider.serverOptions; + appendOnlyFile = clusterProvider.storeWrapper.appendOnlyFile; + replicationManager = clusterProvider.replicationManager; + replayIterator = null; + activeWorkerMonitor = new(); + physicalSublog = appendOnlyFile.Log.GetSubLog(physicalSublogIdx); + this.cts = cts; + this.logger = logger; + + // Initialize background replay tasks for this sublog replay driver + var replayTaskCount = serverOptions.AofReplayTaskCount; + if (replayTaskCount > 1) + { + replayBatchContext = new ReplayBatchContext(replayTaskCount); + replayTasks = [.. Enumerable.Range(0, replayTaskCount).Select(i => new ReplicaReplayTask(i, this, clusterProvider, cts, logger))]; + foreach (var replayTask in replayTasks) + { + if (!useChannels) + { + _ = Task.Run(() => replayTask.FullPageBasedBackgroundReplayAsync()); + } + else + { + _ = Task.Run(() => replayTask.ChannelBasedBackgroundReplayAsync()); + } + } + } + } + + public void Dispose() + { + activeWorkerMonitor.Dispose(); + replayIterator?.Dispose(); + respSessionNetworkSender?.Dispose(); + } + + #region IBulkLogEntryConsumer + + /// + /// Main bulk consume implementation. + /// + /// Pointer to the log entry record to consume. + /// Length of the log entry record in bytes. + /// Current address of the log entry in the log. + /// Next address in the log after the current record. + /// Indicates whether the log entry is protected. + /// Thrown if the background replay operation times out. + public unsafe void Consume(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) + { + if (serverOptions.AofReplayTaskCount == 1) + { + ConsumeDirect(record, recordLength, currentAddress, nextAddress, isProtected); + } + else + { + if (!useChannels) + { + ConsumeSchedulePage(record, recordLength, currentAddress, nextAddress, isProtected); + } + else + { + ConsumeScheduleChannel(record, recordLength, currentAddress, nextAddress, isProtected); + } + } + } + + private unsafe void ConsumeSchedulePage(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) + { + replayBatchContext.Record = record; + replayBatchContext.RecordLength = recordLength; + replayBatchContext.CurrentAddress = currentAddress; + replayBatchContext.NextAddress = nextAddress; + replayBatchContext.IsProtected = isProtected; + replayBatchContext.LeaderFollowerBarrier.SignalWorkReady(); + + // Set replication offset currentAddress + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, currentAddress); + + // Wait for replay to complete. + if (!replayBatchContext.LeaderFollowerBarrier.WaitCompleted(serverOptions.ReplicaSyncTimeout, cts.Token)) + ExceptionUtils.ThrowException(new GarnetException("Timed out waiting for parallel replay tasks to complete", LogLevel.Warning, clientResponse: false)); + // Release participants for next cycle + replayBatchContext.LeaderFollowerBarrier.Release(); + + // Before updating replication offset, we must wait for any pending Vector Set ops to complete + replicationManager.AofProcessor.WaitForVectorOperationsToComplete(); + + // Advertise new replicaton offset after replay completes + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, nextAddress); + } + + private unsafe void ConsumeScheduleChannel(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) + { + ValidateSublogIndex(physicalSublogIdx); + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, currentAddress); + var replicationOffset = currentAddress; + var ptr = record; + + // logger?.LogError("[{physicalSublogIdx}] = {currentAddress} -> {nextAddress}", physicalSublogIdx, currentAddress, nextAddress); + while (ptr < record + recordLength) + { + cts.Token.ThrowIfCancellationRequested(); + var entryLength = appendOnlyFile.HeaderSize; + var payloadLength = physicalSublog.UnsafeGetLength(ptr); + if (payloadLength > 0) + { + var entryPtr = ptr + entryLength; + var logAddressSequenceNumber = currentAddress + (ptr - record); + var replayTaskIdx = replicationManager.AofProcessor.GetReplayTaskIdx(entryPtr); + replayTasks[replayTaskIdx].AddRecord(new ReplayRecord() + { + entryPtr = entryPtr, + payloadLength = payloadLength, + logAddressSequenceNumber = logAddressSequenceNumber + }); + entryLength += TsavoriteLog.UnsafeAlign(payloadLength); + } + else if (payloadLength < 0) + { + TsavoriteLogRecoveryInfo info = new(); + info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); + physicalSublog.UnsafeCommitMetadataOnly(info, isProtected); + entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); + } + ptr += entryLength; + replicationOffset += entryLength; + } + + // Wait for every task to drain its ring and signal on the barrier. + if (!replayBatchContext.LeaderFollowerBarrier.WaitCompleted(serverOptions.ReplicaSyncTimeout, cts.Token)) + throw new GarnetException("Timed out draining replay batch", LogLevel.Warning, clientResponse: false); + // Release participants for next cycle + replayBatchContext.LeaderFollowerBarrier.Release(); + + // Before updating replication offset, we must wait for any pending Vector Set ops to complete + replicationManager.AofProcessor.WaitForVectorOperationsToComplete(); + + // Set replication offset after replay completes + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, replicationOffset); + // logger?.LogError("[{physicalSublogIdx}] = {currentAddress} -> {nextAddress}", physicalSublogIdx, currentAddress, nextAddress); + + if (replicationManager.GetSublogReplicationOffset(physicalSublogIdx) != nextAddress) + { + logger?.LogError("ReplicaReplayTask.Consume NextAddress Mismatch sublogIdx: {sublogIdx}; recordLength:{recordLength}; currentAddress:{currentAddress}; nextAddress:{nextAddress}; replicationOffset:{ReplicationOffset}", physicalSublogIdx, recordLength, currentAddress, nextAddress, replicationManager.GetReplicationOffset(physicalSublogIdx)); + throw new GarnetException("Failed validating integrity of replay", LogLevel.Warning, clientResponse: false); + } + } + + /// + /// Processes record on a single replay task directly. + /// + /// Pointer to the start of the log record to process. + /// Length in bytes of the log record. + /// Current address in the log for replication. + /// Expected next address in the log after processing. + /// Indicates whether the operation should be performed in protected mode. + /// Thrown if fast commit is not enabled when a fast commit request is received, or if log integrity validation + /// fails. + private unsafe void ConsumeDirect(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) + { + ValidateSublogIndex(physicalSublogIdx); + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, currentAddress); + var replicationOffset = currentAddress; + // logger?.LogError("[{physicalSublogIdx}] = {currentAddress} -> {nextAddress}", physicalSublogIdx, currentAddress, nextAddress); + + try + { + var ptr = record; + while (ptr < record + recordLength) + { + cts.Token.ThrowIfCancellationRequested(); + var entryLength = appendOnlyFile.HeaderSize; + var payloadLength = physicalSublog.UnsafeGetLength(ptr); + if (payloadLength > 0) + { + var logAddressSequenceNumber = currentAddress + (ptr - record); + replicationManager.AofProcessor.ProcessAofRecordInternal(physicalSublogIdx, ptr + entryLength, payloadLength, true, out var isCheckpointStart, logAddressSequenceNumber); + // Encountered checkpoint start marker, log the ReplicationCheckpointStartOffset so we know the correct AOF truncation + // point when we take a checkpoint at the checkpoint end marker + if (isCheckpointStart) + { + // This is safe to be updated in parallel given that each sublog replay taks will update its own slot with corresponding address of the checkpoint marker + replicationManager.ReplicationCheckpointStartOffset[physicalSublogIdx] = replicationOffset; + } + entryLength += TsavoriteLog.UnsafeAlign(payloadLength); + } + else if (payloadLength < 0) + { + TsavoriteLogRecoveryInfo info = new(); + info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); + physicalSublog.UnsafeCommitMetadataOnly(info, isProtected); + entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); + } + ptr += entryLength; + replicationOffset += entryLength; + } + // logger?.LogError("[{physicalSublogIdx}] = {currentAddress} -> {nextAddress}", physicalSublogIdx, currentAddress, nextAddress); + } + catch + { + // If an exception occurrs, be sure to advance ReplicationOffset by the amount of successful work that transpired before the error + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, replicationOffset); + throw; + } + + // Before updating replication offset, we must wait for any pending Vector Set ops to complete + replicationManager.AofProcessor.WaitForVectorOperationsToComplete(); + + replicationManager.SetSublogReplicationOffset(physicalSublogIdx, replicationOffset); + + if (replicationOffset != nextAddress) + { + logger?.LogError("ReplicaReplayTask.Consume NextAddress Mismatch sublogIdx: {sublogIdx}; recordLength:{recordLength}; currentAddress:{currentAddress}; nextAddress:{nextAddress}; replicationOffset:{ReplicationOffset}", physicalSublogIdx, recordLength, currentAddress, nextAddress, replicationManager.GetReplicationOffset(physicalSublogIdx)); + throw new GarnetException("Failed validating integrity of replay", LogLevel.Warning, clientResponse: false); + } + } + + public void Throttle() { } + #endregion + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ValidateSublogIndex(int physicalSublogIdx) + { + if (physicalSublogIdx != this.physicalSublogIdx) + throw new GarnetException($"PhysicalSublogIdx mismatch; expected:{this.physicalSublogIdx} - received:{physicalSublogIdx}"); + } + + /// + /// Method to create a background replay task that iterates and consume this replicas physical log + /// + /// + public void InitialiazeBackgroundReplayTask(long startAddress) + { + if (replayIterator == null) + { + replayIterator = appendOnlyFile.Log.ScanSingle(physicalSublogIdx, startAddress, long.MaxValue, scanUncommitted: true, recover: false, logger: logger); + _ = BackgroundReplayTaskAsync(); + } + + async Task BackgroundReplayTaskAsync() + { + // Force async + await Task.Yield(); + + var readLock = ResumeReplay(); + try + { + if (!readLock) + throw new GarnetException("Failed to acquire replayLock"); + + await replayIterator.BulkConsumeAllAsync( + this, + serverOptions.ReplicaSyncDelayMs, + maxChunkSize: 1 << 20, + cts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "An exception occurred at ReplicationManager.ReplicaReplayTask - terminating"); + } + finally + { + if (readLock) + SuspendReplay(); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ThrottlePrimary() + { + while (serverOptions.ReplicationOffsetMaxLag != -1 && replayIterator != null && + appendOnlyFile.Log.GetTailAddress(physicalSublogIdx) - replicationManager.GetReplicationOffset(physicalSublogIdx) > serverOptions.ReplicationOffsetMaxLag) + { + cts.Token.ThrowIfCancellationRequested(); + Thread.Yield(); + } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriverStore.cs b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriverStore.cs new file mode 100644 index 00000000000..7a07259a9c4 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayDriverStore.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Threading; +using Garnet.common; +using Garnet.networking; +using Microsoft.Extensions.Logging; + +namespace Garnet.cluster +{ + internal class ReplicaReplayDriverStore(ClusterProvider clusterProvider, ILogger logger) + { + readonly ClusterProvider clusterProvider = clusterProvider; + readonly ILogger logger = logger; + + /// + /// Get replay driver for given sublogIdx + /// + /// + /// + public ReplicaReplayDriver GetReplayDriver(int physicalSublogIdx) + => replicaReplayDrivers[physicalSublogIdx]; + + /// + /// Replay task instances per sublog (used with ShardedLog) + /// + readonly ReplicaReplayDriver[] replicaReplayDrivers = new ReplicaReplayDriver[clusterProvider.serverOptions.AofPhysicalSublogCount]; + + /// + /// Replay barrier used to coordinate connection of replay tasks + /// + readonly Barrier barrier = clusterProvider.serverOptions.AofPhysicalSublogCount > 1 ? new(clusterProvider.serverOptions.AofPhysicalSublogCount) : null; + + /// + /// Disposed lock + /// + private SingleWriterMultiReaderLock _lock = new(); + + /// + /// Disposed flag + /// + private bool disposed = false; + + /// + /// Cancellation token source for replay task group + /// + readonly CancellationTokenSource cts = new(); + + /// + /// Add replica replay driver to this store + /// + /// + /// + public void AddReplicaReplayDriver(int physicalSublogIdx, INetworkSender networkSender) + { + try + { + _lock.ReadLock(); + if (disposed) + return; + replicaReplayDrivers[physicalSublogIdx] = new ReplicaReplayDriver(physicalSublogIdx, clusterProvider, networkSender, cts, logger); + _ = barrier?.SignalAndWait(clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token); + } + finally + { + _lock.ReadUnlock(); + } + } + + /// + /// Dispose replica replay task group + /// + public void Dispose() + { + try + { + _lock.WriteLock(); + if (disposed) + return; + disposed = true; + } + finally + { + _lock.WriteUnlock(); + } + + cts.Cancel(); + var replicaReplayTasks = replicaReplayDrivers; + if (replicaReplayTasks != null) + { + for (var i = 0; i < replicaReplayTasks.Length; i++) + replicaReplayTasks[i]?.Dispose(); + } + cts.Dispose(); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayManager.cs b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayManager.cs new file mode 100644 index 00000000000..27584710ce5 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayManager.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Garnet.networking; + +namespace Garnet.cluster +{ + internal sealed partial class ReplicationManager : IDisposable + { + /// + /// Replica replay driver store instance + /// + public ReplicaReplayDriverStore ReplicaReplayDriverStore; + + /// + /// Initialize replica replay driver + /// + /// + /// + /// True if re + public bool InitializeReplicaReplayDriver(int physicalSublogIdx, INetworkSender networkSender) + { + if (ReplicaReplayDriverStore.GetReplayDriver(physicalSublogIdx) != null) + return false; + + ReplicaReplayDriverStore.AddReplicaReplayDriver(physicalSublogIdx, networkSender); + return true; + } + + /// + /// Resets the state of the replica replay driver store + /// + public void ResetReplicaReplayDriverStore() + { + ReplicaReplayDriverStore?.Dispose(); + ReplicaReplayDriverStore = new ReplicaReplayDriverStore(clusterProvider, logger); + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/ReplicationReplicaAofSync.cs b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplaySession.cs similarity index 64% rename from libs/cluster/Server/Replication/ReplicaOps/ReplicationReplicaAofSync.cs rename to libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplaySession.cs index 6227f8939d8..bfd0f486f33 100644 --- a/libs/cluster/Server/Replication/ReplicaOps/ReplicationReplicaAofSync.cs +++ b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplaySession.cs @@ -1,35 +1,29 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; -using System.Runtime.CompilerServices; -using System.Threading; using Garnet.common; +using Garnet.server; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.cluster { - internal sealed partial class ReplicationManager : IDisposable + internal sealed unsafe partial class ClusterSession : IClusterSession { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - void ThrottlePrimary() - { - while (storeWrapper.serverOptions.ReplicationOffsetMaxLag != -1 && replayIterator != null && storeWrapper.appendOnlyFile.TailAddress - ReplicationOffset > storeWrapper.serverOptions.ReplicationOffsetMaxLag) - { - replicaReplayTaskCts.Token.ThrowIfCancellationRequested(); - Thread.Yield(); - } - } + ReplicaReplayDriverStore replicaReplayDriverStore = null; + TsavoriteLog physicalSublog = null; /// /// Apply primary AOF records. /// + /// /// /// /// /// /// - public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long previousAddress, long currentAddress, long nextAddress) + public void ProcessPrimaryStream(int physicalSublogIdx, byte* record, int recordLength, long previousAddress, long currentAddress, long nextAddress) { // logger?.LogInformation("Processing {recordLength} bytes; previousAddress {previousAddress}, currentAddress {currentAddress}, nextAddress {nextAddress}, current AOF tail {tail}", recordLength, previousAddress, currentAddress, nextAddress, storeWrapper.appendOnlyFile.TailAddress); var currentConfig = clusterProvider.clusterManager.CurrentConfig; @@ -38,11 +32,11 @@ public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long pre // Need to ensure that this replay task is allowed to complete before the replicaReplayGroup is disposed // NOTE: this should not be expensive because every replay task has its own lock copy // Cache invalidation happens only on dispose which is rare operation - var failReplay = syncReplay && !activeReplay.TryReadLock(); + var failReplay = syncReplay && !replicaReplayDriverStore.GetReplayDriver(physicalSublogIdx).ResumeReplay(); try { if (failReplay) - throw new GarnetException($"Failed to acquire activeReplay lock!", LogLevel.Warning, clientResponse: false); + throw new GarnetException($"[{physicalSublogIdx}] Failed to acquire activeReplay lock!", LogLevel.Warning, clientResponse: false); if (clusterProvider.replicationManager.CannotStreamAOF) { @@ -63,17 +57,17 @@ public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long pre if (currentAddress > previousAddress) { if ( - (currentAddress % (1 << pageSizeBits) != 0) || // the skip was to a non-page-boundary + (currentAddress % (1 << clusterProvider.replicationManager.PageSizeBits) != 0) || // the skip was to a non-page-boundary (currentAddress >= previousAddress + recordLength) // the skip will not be auto-handled by the AOF enqueue ) { - logger?.LogWarning("MainMemoryReplication: Skipping from {ReplicaReplicationOffset} to {currentAddress}", ReplicationOffset, currentAddress); - storeWrapper.appendOnlyFile.SafeInitialize(currentAddress, currentAddress); + logger?.LogWarning("MainMemoryReplication: Skipping from {ReplicaReplicationOffset} to {currentAddress}", clusterProvider.replicationManager.GetSublogReplicationOffset(physicalSublogIdx), currentAddress); + clusterProvider.storeWrapper.appendOnlyFile.Log.SafeInitialize(physicalSublogIdx, currentAddress, currentAddress); // If any Vector Set ops in progress, we must wait for them before we advertise a new eplication offset - storeWrapper.DefaultDatabase?.VectorManager.WaitForVectorOperationsToComplete(); + clusterProvider.storeWrapper.DefaultDatabase?.VectorManager.WaitForVectorOperationsToComplete(); - ReplicationOffset = currentAddress; + clusterProvider.replicationManager.SetSublogReplicationOffset(physicalSublogIdx, currentAddress); } } } @@ -81,8 +75,8 @@ public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long pre // Injection for a "something went wrong with THIS Replica's AOF file" ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.Divergent_AOF_Stream); - var tail = storeWrapper.appendOnlyFile.TailAddress; - var nextPageBeginAddress = ((tail >> pageSizeBits) + 1) << pageSizeBits; + var tail = clusterProvider.storeWrapper.appendOnlyFile.Log.GetTailAddress(physicalSublogIdx); + var nextPageBeginAddress = ((tail >> clusterProvider.replicationManager.PageSizeBits) + 1) << clusterProvider.replicationManager.PageSizeBits; // Check to ensure: // 1. if record fits in current page tailAddress of this local node (replica) should be equal to the incoming currentAddress (address of chunk send from primary node) // 2. if record does not fit in current page start address of the next page matches incoming currentAddress (address of chunk send from primary node) @@ -95,39 +89,31 @@ public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long pre } // Address check only if synchronous replication is enabled - if (storeWrapper.serverOptions.ReplicationOffsetMaxLag == 0 && ReplicationOffset != storeWrapper.appendOnlyFile.TailAddress) + if (clusterProvider.storeWrapper.serverOptions.ReplicationOffsetMaxLag == 0 && clusterProvider.replicationManager.GetSublogReplicationOffset(physicalSublogIdx) != tail) { - logger?.LogInformation("Processing {recordLength} bytes; previousAddress {previousAddress}, currentAddress {currentAddress}, nextAddress {nextAddress}, current AOF tail {tail}", recordLength, previousAddress, currentAddress, nextAddress, storeWrapper.appendOnlyFile.TailAddress); - logger?.LogError("Before ProcessPrimaryStream: Replication offset mismatch: ReplicaReplicationOffset {ReplicaReplicationOffset}, aof.TailAddress {tailAddress}", ReplicationOffset, storeWrapper.appendOnlyFile.TailAddress); - throw new GarnetException($"Before ProcessPrimaryStream: Replication offset mismatch: ReplicaReplicationOffset {ReplicationOffset}, aof.TailAddress {storeWrapper.appendOnlyFile.TailAddress}", LogLevel.Warning, clientResponse: false); + logger?.LogInformation("Processing {recordLength} bytes; previousAddress {previousAddress}, currentAddress {currentAddress}, nextAddress {nextAddress}, current AOF tail {tail}", recordLength, previousAddress, currentAddress, nextAddress, tail); + logger?.LogError("Before ProcessPrimaryStream: Replication offset mismatch: ReplicaReplicationOffset {ReplicaReplicationOffset}, aof.TailAddress {tailAddress}", clusterProvider.replicationManager.GetSublogReplicationOffset(physicalSublogIdx), tail); + throw new GarnetException($"Before ProcessPrimaryStream: Replication offset mismatch: ReplicaReplicationOffset {clusterProvider.replicationManager.GetSublogReplicationOffset(physicalSublogIdx)}, aof.TailAddress {tail}", LogLevel.Warning, clientResponse: false); } + // Initialize sublog ref if first time + physicalSublog ??= clusterProvider.storeWrapper.appendOnlyFile.Log.GetSubLog(physicalSublogIdx); + // Enqueue to AOF - _ = clusterProvider.storeWrapper.appendOnlyFile?.UnsafeEnqueueRaw(new Span(record, recordLength), noCommit: clusterProvider.serverOptions.EnableFastCommit); + _ = physicalSublog.UnsafeEnqueueRaw(new Span(record, recordLength), noCommit: true); - if (storeWrapper.serverOptions.ReplicationOffsetMaxLag == 0) + if (clusterProvider.storeWrapper.serverOptions.ReplicationOffsetMaxLag == 0) { // Synchronous replay - Consume(record, recordLength, currentAddress, nextAddress, isProtected: false); + replicaReplayDriverStore.GetReplayDriver(physicalSublogIdx).Consume(record, recordLength, currentAddress, nextAddress, isProtected: false); } else { - // Throttle to give the opportunity to the background replay task to catch up - ThrottlePrimary(); + // Initialize iterator and run background task once + replicaReplayDriverStore.GetReplayDriver(physicalSublogIdx).InitialiazeBackgroundReplayTask(previousAddress); - // If background task has not been initialized - // initialize it here and start background replay task - if (replayIterator == null) - { - replayIterator = clusterProvider.storeWrapper.appendOnlyFile.ScanSingle( - previousAddress, - long.MaxValue, - scanUncommitted: true, - recover: false, - logger: logger); - - _ = System.Threading.Tasks.Task.Run(ReplicaReplayTaskAsync); - } + // Throttle to give the opportunity to the background replay task to catch up + replicaReplayDriverStore.GetReplayDriver(physicalSublogIdx).ThrottlePrimary(); } } catch (Exception ex) @@ -138,7 +124,7 @@ public unsafe void ProcessPrimaryStream(byte* record, int recordLength, long pre finally { if (syncReplay && !failReplay) - activeReplay.ReadUnlock(); + replicaReplayDriverStore.GetReplayDriver(physicalSublogIdx).SuspendReplay(); } } } diff --git a/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayTask.cs b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayTask.cs new file mode 100644 index 00000000000..7822952c3b2 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/AOFReplay/ReplicaReplayTask.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; +using Garnet.common; +using Garnet.server; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + [StructLayout(LayoutKind.Sequential, Size = 12)] + internal unsafe struct ReplayRecord + { + public byte* entryPtr; + public int payloadLength; + public long logAddressSequenceNumber; + } + + internal sealed class ReplicaReplayTask( + int replayIdx, + ReplicaReplayDriver replayDriver, + ClusterProvider clusterProvider, + CancellationTokenSource cts, + ILogger logger = null) + { + readonly int replayTaskIdx = replayIdx; + readonly ReplicaReplayDriver replayDriver = replayDriver; + readonly ReplicationManager replicationManager = clusterProvider.replicationManager; + readonly GarnetAppendOnlyFile appendOnlyFile = clusterProvider.storeWrapper.appendOnlyFile; + readonly ReplayBatchContext replayBatchContext = replayDriver.replayBatchContext; + readonly CancellationTokenSource cts = cts; + readonly TsavoriteLog replaySublog = clusterProvider.storeWrapper.appendOnlyFile.Log.GetSubLog(replayDriver.physicalSublogIdx); + readonly ActiveWorkerMonitor activeWorkerMonitor = new(); + private readonly Channel replayChannel = Channel.CreateUnbounded( + new() { SingleWriter = true, SingleReader = true, AllowSynchronousContinuations = false }); + readonly ILogger logger = logger; + + /// + /// Add record for replay + /// + /// + public void AddRecord(ReplayRecord replayRecord) + => replayChannel.Writer.TryWrite(replayRecord); + + /// + /// Asynchronously replays log entries using SemaphoreSlim coordination, processing and applying them for replication + /// and consistency across sublogs. + /// + /// A task representing the asynchronous replay operation. + internal async Task FullPageBasedBackgroundReplayAsync() + { + var physicalSublogIdx = replayDriver.physicalSublogIdx; + var virtualSublogIdx = appendOnlyFile.GetVirtualSublogIdx(physicalSublogIdx, replayTaskIdx); + + while (!cts.Token.IsCancellationRequested) + { + try + { + await replayBatchContext.LeaderFollowerBarrier.WaitReadyWorkAsync(cancellationToken: cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) when (cts.Token.IsCancellationRequested) + { + // Suppress the exception if the task was cancelled because of store wrapper disposal + } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed at WaitAsync", nameof(FullPageBasedBackgroundReplayAsync)); + await cts.CancelAsync().ConfigureAwait(false); + break; + } + + // Guard: if cancellation happened during WaitReadyWorkAsync, exit cleanly + // without falling through to the processing block (which would issue a spurious SignalCompleted) + if (cts.Token.IsCancellationRequested) + break; + + try + { + unsafe + { + var record = replayBatchContext.Record; + var recordLength = replayBatchContext.RecordLength; + var currentAddress = replayBatchContext.CurrentAddress; + var nextAddress = replayBatchContext.NextAddress; + var isProtected = replayBatchContext.IsProtected; + var ptr = record; + + while (ptr < record + recordLength) + { + cts.Token.ThrowIfCancellationRequested(); + var entryLength = appendOnlyFile.HeaderSize; + var payloadLength = replaySublog.UnsafeGetLength(ptr); + if (payloadLength > 0) + { + var entryPtr = ptr + entryLength; + var logAddressSequenceNumber = currentAddress + (ptr - record); + if (replicationManager.AofProcessor.CanReplay(entryPtr, replayTaskIdx, logAddressSequenceNumber, out _)) + { + replicationManager.AofProcessor.ProcessAofRecordInternal(virtualSublogIdx, entryPtr, payloadLength, true, out var isCheckpointStart, logAddressSequenceNumber); + // Encountered checkpoint start marker, log the ReplicationCheckpointStartOffset so we know the correct AOF truncation + // point when we take a checkpoint at the checkpoint end marker + if (isCheckpointStart) + { + replicationManager.ReplicationCheckpointStartOffset[physicalSublogIdx] = replicationManager.GetSublogReplicationOffset(physicalSublogIdx); + } + } + entryLength += TsavoriteLog.UnsafeAlign(payloadLength); + } + else if (payloadLength < 0) + { + // Only a single thread should commit metadata + if (replayTaskIdx == 0) + { + TsavoriteLogRecoveryInfo info = new(); + info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); + replaySublog.UnsafeCommitMetadataOnly(info, isProtected); + } + entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); + } + ptr += entryLength; + } + + // Advance frontier to nextAddress (past all entries in this page). + // This ensures the read consistency protocol (which waits for frontier > sessionSeq) + // can proceed once all writes in the page are complete. + appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(virtualSublogIdx, nextAddress); + } + } + catch (TaskCanceledException) when (cts.Token.IsCancellationRequested) + { + // Suppress the exception if the task was cancelled because of store wrapper disposal + } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed at replaying", nameof(FullPageBasedBackgroundReplayAsync)); + await cts.CancelAsync().ConfigureAwait(false); + break; + } + finally + { + // Signal work completion after processing + replayBatchContext.LeaderFollowerBarrier.SignalCompleted(); + } + } + } + + /// + /// Asynchronously processes records from the replay channel in the background. + /// + /// A task that represents the asynchronous replay operation. + internal async Task ChannelBasedBackgroundReplayAsync() + { + var physicalSublogIdx = replayDriver.physicalSublogIdx; + var virtualSublogIdx = appendOnlyFile.GetVirtualSublogIdx(physicalSublogIdx, replayTaskIdx); + var reader = replayChannel.Reader; + + while (await reader.WaitToReadAsync(cts.Token).ConfigureAwait(false)) + { + while (reader.TryRead(out var record)) + { + unsafe + { + replicationManager.AofProcessor.ProcessAofRecordInternal(virtualSublogIdx, record.entryPtr, record.payloadLength, true, out var isCheckpointStart, record.logAddressSequenceNumber); + + // Encountered checkpoint start marker, log the ReplicationCheckpointStartOffset so we know the correct AOF truncation + // point when we take a checkpoint at the checkpoint end marker + if (isCheckpointStart) + { + // logger?.LogError("[{sublogIdx}] CheckpointStart {address}", sublogIdx, clusterProvider.replicationManager.GetSublogReplicationOffset(sublogIdx)); + replicationManager.ReplicationCheckpointStartOffset[physicalSublogIdx] = replicationManager.GetSublogReplicationOffset(physicalSublogIdx); + } + } + } + + // Signal work completion after processing + replayBatchContext.LeaderFollowerBarrier.SignalCompleted(); + } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/FileDataSink.cs b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/FileDataSink.cs new file mode 100644 index 00000000000..9ec59bb8d8f --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/FileDataSink.cs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + /// + /// Device-backed implementation of that writes checkpoint + /// file segments to an using sector-aligned buffered I/O. + /// + internal sealed class FileDataSink : ISnapshotDataSink + { + private readonly IDevice device; + private readonly SectorAlignedBufferPool bufferPool; + private readonly SemaphoreSlim writeSemaphore; + private readonly TimeSpan timeout; + private readonly CancellationToken cancellationToken; + private readonly ILogger logger; + private readonly IOCallbackContext ioContext = new(); + private volatile uint lastIOErrorCode; + + public CheckpointFileType Type { get; } + public Guid Token { get; } + + /// + /// Creates a new FileDataSink. + /// + /// The checkpoint file type. + /// The checkpoint token. + /// The initialized device to write to. + /// Shared sector-aligned buffer pool for write operations. + /// Shared semaphore for async I/O completion signaling. + /// Timeout for async write operations. + /// Cancellation token. + /// Optional logger. + public FileDataSink( + CheckpointFileType type, + Guid token, + IDevice device, + SectorAlignedBufferPool bufferPool, + SemaphoreSlim writeSemaphore, + TimeSpan timeout, + CancellationToken cancellationToken, + ILogger logger = null) + { + Type = type; + Token = token; + this.device = device; + this.bufferPool = bufferPool; + this.writeSemaphore = writeSemaphore; + this.timeout = timeout; + this.cancellationToken = cancellationToken; + this.logger = logger; + } + + /// + public unsafe void WriteChunk(long startAddress, ReadOnlySpan data) + { + if ((startAddress & (device.SectorSize - 1)) != 0) + throw new ArgumentException($"startAddress {startAddress} is not aligned to device sector size {device.SectorSize}", nameof(startAddress)); + + long numBytesToWrite = data.Length; + numBytesToWrite = (numBytesToWrite + (device.SectorSize - 1)) & ~(device.SectorSize - 1); + + var pbuffer = bufferPool.Get((int)numBytesToWrite); + ioContext.Buffer = pbuffer; + + fixed (byte* bufferRaw = data) + Buffer.MemoryCopy(bufferRaw, pbuffer.aligned_pointer, data.Length, data.Length); + + device.WriteAsync((IntPtr)pbuffer.aligned_pointer, (ulong)startAddress, (uint)numBytesToWrite, IOCallback, ioContext); + + // The IOCallbackContext roots the buffer for GC safety while the IO is in-flight. + // On timeout or cancellation the buffer is intentionally abandoned (not returned to + // the pool) — the exception aborts the replication session, so the stale semaphore + // count left by the callback is harmless. + if (!writeSemaphore.Wait(timeout, cancellationToken)) + { + ExceptionUtils.ThrowException(new GarnetException( + $"Timed out writing {Type} checkpoint file at address {startAddress} (requested {numBytesToWrite} bytes)")); + } + + var errorCode = lastIOErrorCode; + Debug.Assert(errorCode == 0, $"I/O error {errorCode} writing {Type} checkpoint file at address {startAddress}"); + if (errorCode != 0) + { + ExceptionUtils.ThrowException(new GarnetException( + $"I/O error {errorCode} writing {Type} checkpoint file at address {startAddress} (requested {numBytesToWrite} bytes)")); + } + + // IO completed successfully — return buffer to pool for reuse. + pbuffer.Return(); + } + + /// + public void Complete() + { + device?.Dispose(); + } + + /// + public void Dispose() + { + device?.Dispose(); + } + + private void IOCallback(uint errorCode, uint numBytes, object context) + { + lastIOErrorCode = errorCode; + if (errorCode != 0) + { + var errorMessage = Utility.GetCallbackErrorMessage(errorCode, numBytes, context); + logger?.LogError("[FileDataSink] WriteAsync error: {errorCode} msg: {errorMessage}", errorCode, errorMessage); + } + + try + { + _ = writeSemaphore.Release(); + } + catch (ObjectDisposedException) { } + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ISnapshotDataSink.cs b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ISnapshotDataSink.cs new file mode 100644 index 00000000000..b8089b69d95 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ISnapshotDataSink.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Garnet.cluster +{ + /// + /// Interface for a checkpoint data sink that receives chunk-based writes. + /// This is the write-side counterpart to . + /// + internal interface ISnapshotDataSink : IDisposable + { + /// + /// The type of checkpoint file this sink represents. + /// + CheckpointFileType Type { get; } + + /// + /// The token identifying the checkpoint. + /// + Guid Token { get; } + + /// + /// Writes a chunk of data to the sink. + /// + /// The start address for this chunk. + /// The data to write. + void WriteChunk(long startAddress, ReadOnlySpan data); + + /// + /// Signals that all data has been written and the sink should finalize. + /// + void Complete(); + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/MetadataDataSink.cs b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/MetadataDataSink.cs new file mode 100644 index 00000000000..d06a5dabfa9 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/MetadataDataSink.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Garnet.cluster +{ + /// + /// In-memory implementation of that commits checkpoint + /// metadata bytes to the checkpoint manager. + /// + internal sealed class MetadataDataSink : ISnapshotDataSink + { + private readonly ClusterProvider clusterProvider; + + public CheckpointFileType Type { get; } + public Guid Token { get; } + + /// + /// Creates a new MetadataDataSink. + /// + /// The checkpoint file type (STORE_INDEX or STORE_SNAPSHOT). + /// The checkpoint token. + /// The cluster provider for accessing checkpoint managers. + public MetadataDataSink(CheckpointFileType type, Guid token, ClusterProvider clusterProvider) + { + Type = type; + Token = token; + this.clusterProvider = clusterProvider; + } + + /// + public void WriteChunk(long startAddress, ReadOnlySpan data) + { + var checkpointMetadata = data.ToArray(); + var ckptManager = Type switch + { + CheckpointFileType.STORE_SNAPSHOT or + CheckpointFileType.STORE_INDEX => clusterProvider.ReplicationLogCheckpointManager, + _ => throw new Exception($"Invalid checkpoint filetype {Type}"), + }; + + switch (Type) + { + case CheckpointFileType.STORE_SNAPSHOT: + ckptManager.CommitLogCheckpointSendFromPrimary(Token, checkpointMetadata); + break; + case CheckpointFileType.STORE_INDEX: + ckptManager.CommitIndexCheckpoint(Token, checkpointMetadata); + break; + default: + throw new Exception($"Invalid checkpoint filetype {Type}"); + } + } + + /// + public void Complete() + { + // No finalization needed for metadata commits + } + + /// + public void Dispose() + { + // Nothing to dispose + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ReceiveCheckpointHandler.cs b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ReceiveCheckpointHandler.cs new file mode 100644 index 00000000000..830f467e693 --- /dev/null +++ b/libs/cluster/Server/Replication/ReplicaOps/DiskbasedReplication/ReceiveCheckpointHandler.cs @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.cluster +{ + internal sealed class ReceiveCheckpointHandler : IDisposable + { + readonly ClusterProvider clusterProvider; + readonly CancellationTokenSource cts; + readonly ILogger logger; + + // Shared resources across FileDataSink instances + SectorAlignedBufferPool bufferPool; + readonly SemaphoreSlim writeSemaphore = new(0); + + // Active sink (one at a time, matching the sequential protocol) + ISnapshotDataSink activeSink; + + public ReceiveCheckpointHandler(ClusterProvider clusterProvider, ILogger logger = null) + { + this.clusterProvider = clusterProvider; + this.logger = logger; + cts = new(); + } + + public void Dispose() + { + cts.Cancel(); + cts.Dispose(); + activeSink?.Dispose(); + activeSink = null; + writeSemaphore?.Dispose(); + bufferPool?.Free(); + bufferPool = null; + } + + /// + /// Process file segments sent from primary. + /// An empty data span signals end-of-stream for the current file. + /// + /// The checkpoint token. + /// The checkpoint file type. + /// The start address for this chunk. + /// The data to write. Empty signals end-of-stream. + public void ProcessFileSegment(Guid token, CheckpointFileType type, long startAddress, ReadOnlySpan data) + { + clusterProvider.replicationManager.UpdateLastPrimarySyncTime(); + + if (data.Length == 0) + { + activeSink?.Complete(); + activeSink = null; + return; + } + + if (activeSink == null) + { + // On retry, this may reopen an existing file from a previous failed attempt. + // This is safe because chunks are streamed from the start, overwriting any partial data. + var device = clusterProvider.replicationManager.CreateCheckpointDevice(token, type); + bufferPool ??= new SectorAlignedBufferPool(1, (int)device.SectorSize); + activeSink = new FileDataSink(type, token, device, bufferPool, writeSemaphore, clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token, logger); + } + + activeSink.WriteChunk(startAddress, data); + +#if DEBUG + ExceptionInjectionHelper.WaitOnClear(ExceptionInjectionType.Replication_Timeout_On_Receive_Checkpoint); +#endif + } + + /// + /// Process checkpoint metadata transmitted from primary during replica synchronization. + /// + /// Checkpoint metadata token. + /// Checkpoint metadata filetype. + /// Raw bytes of checkpoint metadata. + public void ProcessMetadata(Guid token, CheckpointFileType type, ReadOnlySpan checkpointMetadata) + { + clusterProvider.replicationManager.UpdateLastPrimarySyncTime(); + using var sink = new MetadataDataSink(type, token, clusterProvider); + sink.WriteChunk(0, checkpointMetadata); + sink.Complete(); + } + + /// + /// Unified entry point for receiving snapshot data from primary. + /// Handles file segments and single-message payloads (metadata). + /// + /// Convention: A of -1 indicates a single-message payload + /// that fits in one message (e.g., checkpoint metadata committed directly). + /// Any other startAddress indicates a streamed file segment. Empty data signals end-of-stream. + /// + /// + /// The checkpoint token. + /// The checkpoint file type. + /// The start address for this chunk, or -1 for single-message payloads. + /// The data to write. Empty signals end-of-stream for streamed file segments. + public void ProcessSnapshotData(Guid token, CheckpointFileType type, long startAddress, ReadOnlySpan data) + { + clusterProvider.replicationManager.UpdateLastPrimarySyncTime(); + + // Single-message payload (startAddress == -1) + // NOTE: Use for single write metadata or to configure initialization parameters for shipping multi-segments files. + if (startAddress == -1) + { + switch (type) + { + case CheckpointFileType.STORE_INDEX: + case CheckpointFileType.STORE_SNAPSHOT: + var sink = new MetadataDataSink(type, token, clusterProvider); + try + { + sink.WriteChunk(0, data); + sink.Complete(); + return; + } + finally + { + sink.Dispose(); + } + default: + ExceptionUtils.ThrowException(new GarnetException($"{nameof(ProcessSnapshotData)} invalid startAddress for checkpoint type: {type}!")); + return; + } + } + + // File segment handling: empty data signals end-of-stream + if (data.Length == 0) + { + activeSink?.Complete(); + activeSink = null; + return; + } + + // Initialize sink for multi-segment file transmission + if (activeSink == null) + { + switch (type) + { + case CheckpointFileType.STORE_HLOG: + case CheckpointFileType.STORE_HLOG_OBJ: + case CheckpointFileType.STORE_SNAPSHOT: + case CheckpointFileType.STORE_SNAPSHOT_OBJ: + case CheckpointFileType.STORE_INDEX: + // On retry, this may reopen an existing file from a previous failed attempt. + // This is safe because chunks are streamed from the start, overwriting any partial data. + var device = clusterProvider.replicationManager.CreateCheckpointDevice(token, type); + bufferPool ??= new SectorAlignedBufferPool(1, (int)device.SectorSize); + activeSink = new FileDataSink(type, token, device, bufferPool, writeSemaphore, clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token, logger); + break; + default: + ExceptionUtils.ThrowException(new GarnetException($"{nameof(ProcessSnapshotData)} invalid startAddress for checkpoint type: {type}!")); + return; + } + } + + activeSink.WriteChunk(startAddress, data); + +#if DEBUG + ExceptionInjectionHelper.WaitOnClear(ExceptionInjectionType.Replication_Timeout_On_Receive_Checkpoint); +#endif + } + } +} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/ReceiveCheckpointHandler.cs b/libs/cluster/Server/Replication/ReplicaOps/ReceiveCheckpointHandler.cs deleted file mode 100644 index 4f6270d76a1..00000000000 --- a/libs/cluster/Server/Replication/ReplicaOps/ReceiveCheckpointHandler.cs +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Threading; -#if DEBUG -using Garnet.common; -#endif -using Microsoft.Extensions.Logging; -using Tsavorite.core; - -namespace Garnet.cluster -{ - internal sealed unsafe class ReceiveCheckpointHandler - { - readonly ClusterProvider clusterProvider; - readonly CancellationTokenSource cts; - IDevice writeIntoCkptDevice = null; - private SemaphoreSlim writeCheckpointSemaphore = null; - private SectorAlignedBufferPool writeCheckpointBufferPool = null; - - readonly ILogger logger; - - public ReceiveCheckpointHandler(ClusterProvider clusterProvider, ILogger logger = null) - { - this.clusterProvider = clusterProvider; - this.logger = logger; - cts = new(); - } - - public void Dispose() - { - cts.Cancel(); - cts.Dispose(); - writeCheckpointSemaphore?.Dispose(); - writeCheckpointBufferPool?.Free(); - writeCheckpointBufferPool = null; - CloseDevice(); - } - - public void CloseDevice() - { - writeIntoCkptDevice?.Dispose(); - writeIntoCkptDevice = null; - } - - /// - /// Process file segments send from primary - /// - /// - /// - /// - /// - /// - public void ProcessFileSegments(int segmentId, Guid token, CheckpointFileType type, long startAddress, ReadOnlySpan data) - { - clusterProvider.replicationManager.UpdateLastPrimarySyncTime(); - if (writeIntoCkptDevice == null) - { - Debug.Assert(writeIntoCkptDevice == null); - writeIntoCkptDevice = clusterProvider.replicationManager.GetInitializedSegmentFileDevice(token, type); - } - - if (data.Length == 0) - { - Debug.Assert(writeIntoCkptDevice != null); - CloseDevice(); - return; - } - - Debug.Assert(writeIntoCkptDevice != null); - WriteInto(writeIntoCkptDevice, (ulong)startAddress, data, data.Length, segmentId); - -#if DEBUG - ExceptionInjectionHelper.WaitOnClear(ExceptionInjectionType.Replication_Timeout_On_Receive_Checkpoint); -#endif - } - - /// - /// Note: pads the bytes with zeros to achieve sector alignment - /// - /// - /// - /// - /// - /// - private unsafe void WriteInto(IDevice device, ulong address, ReadOnlySpan buffer, int size, int segmentId = -1) - { - writeCheckpointBufferPool ??= new SectorAlignedBufferPool(1, (int)device.SectorSize); - - long numBytesToWrite = size; - numBytesToWrite = ((numBytesToWrite + (device.SectorSize - 1)) & ~(device.SectorSize - 1)); - - var pbuffer = writeCheckpointBufferPool.Get((int)numBytesToWrite); - try - { - fixed (byte* bufferRaw = buffer) - { - Buffer.MemoryCopy(bufferRaw, pbuffer.aligned_pointer, size, size); - } - - writeCheckpointSemaphore ??= new(0); - - if (segmentId == -1) - device.WriteAsync((IntPtr)pbuffer.aligned_pointer, address, (uint)numBytesToWrite, IOCallback, null); - else - device.WriteAsync((IntPtr)pbuffer.aligned_pointer, segmentId, address, (uint)numBytesToWrite, IOCallback, null); - - _ = writeCheckpointSemaphore.Wait(clusterProvider.serverOptions.ReplicaSyncTimeout, cts.Token); - } - finally - { - pbuffer.Return(); - } - } - - private unsafe void IOCallback(uint errorCode, uint numBytes, object context) - { - if (errorCode != 0) - { - var errorMessage = Utility.GetCallbackErrorMessage(errorCode, numBytes, context); - logger?.LogError("[ReceiveCheckpointHandler] OverlappedStream GetQueuedCompletionStatus error: {errorCode} msg: {errorMessage}", errorCode, errorMessage); - } - - try - { - _ = writeCheckpointSemaphore.Release(); - } - catch (Exception ex) - { - logger?.LogError(ex, $"{nameof(ReceiveCheckpointHandler)}.IOCallback"); - } - } - - [DllImport("libc")] - private static extern IntPtr strerror(int errnum); - } -} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicaOps/ReplicaReceiveCheckpoint.cs b/libs/cluster/Server/Replication/ReplicaOps/ReplicaDiskbasedSync.cs similarity index 60% rename from libs/cluster/Server/Replication/ReplicaOps/ReplicaReceiveCheckpoint.cs rename to libs/cluster/Server/Replication/ReplicaOps/ReplicaDiskbasedSync.cs index bb50e596218..db03bcd7348 100644 --- a/libs/cluster/Server/Replication/ReplicaOps/ReplicaReceiveCheckpoint.cs +++ b/libs/cluster/Server/Replication/ReplicaOps/ReplicaDiskbasedSync.cs @@ -46,6 +46,9 @@ internal sealed partial class ReplicationManager : IDisposable } } + // Create or update timestamp manager for sharded log if needed + storeWrapper.appendOnlyFile.CreateOrUpdateKeySequenceManager(); + // Wait for threads to agree if (session != null) { @@ -100,45 +103,60 @@ async Task ReplicaSyncAttachTaskAsync(bool downgradeLock, bool forceAsyn logger?.LogError("{msg}", errorMsg); return errorMsg; } + + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token, resetHandler.Token); gcs = new( new IPEndPoint(IPAddress.Parse(address), port), clusterProvider.replicationManager.GetIRSNetworkBufferSettings, clusterProvider.replicationManager.GetNetworkPool, tlsOptions: clusterProvider.serverOptions.TlsOptions?.TlsClientOptions, authUsername: clusterProvider.ClusterUsername, - authPassword: clusterProvider.ClusterPassword); - await gcs.ConnectAsync().ConfigureAwait(false); + authPassword: clusterProvider.ClusterPassword, + clientName: nameof(TryReplicateDiskbasedSyncAsync)); + await gcs.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, linkedCts.Token).ConfigureAwait(false); - // Resetting here to decide later when to sync from - clusterProvider.replicationManager.ReplicationOffset = 0; + // Reset background replay iterator if this node was a replica + clusterProvider.replicationManager.ResetReplicaReplayDriverStore(); - // The caller should have stopped accepting AOF records from old primary at this point - // (TryREPLICAOF -> TryAddReplica -> UnsafeWaitForConfigTransition) + // Remove aofSync tasks if this node was a primary + aofSyncDriverStore.Reset(); - // TODO: ensure we have quiesced reads (no writes on replica) - - // Wait for Commit of AOF (data received from old primary) if FastCommit is not enabled - // If FastCommit is enabled, we commit during AOF stream processing - if (!clusterProvider.serverOptions.EnableFastCommit && storeWrapper.appendOnlyFile != null) + // Reset replication offset + replicationOffset.SetValue(0); + + // Reset the database in preparation for connecting to primary. + // Pause VectorManager's background cleanup task first — Reset's + // post-Phase-2 Initialize() rewinds HeadAddress / BeginAddress / + // TailPageOffset and reallocates pages. Tsavorite's iterator path is + // safe (Initializing flag), but the cleanup task's POST-iterate RMWs + // on metadata records (ClearDeleteInProgress / UpdateContextMetadata) + // are NOT — they can dereference freed pagePointers and AVE. The pause + // serializes the entire cleanup-iteration (iterate + RMWs) with Reset + // by holding cleanupGate, restoring Reset's "store is quiesced" contract. + // + // Pass linkedCts.Token so a slow cleanup-iteration over a large keyspace + // doesn't block re-attach indefinitely if the broader replication is + // cancelled (ctsRepManager / resetHandler). If PauseCleanupAsync throws + // OCE, the try block isn't entered and ResumeCleanup is correctly skipped. + var vectorManager = storeWrapper.DefaultDatabase.VectorManager; + if (vectorManager != null) + await vectorManager.PauseCleanupAsync(linkedCts.Token).ConfigureAwait(false); + try { - await storeWrapper.appendOnlyFile.CommitAsync().ConfigureAwait(false); - - // TODO: Is this still necessary? - await storeWrapper.appendOnlyFile.WaitForCommitAsync().ConfigureAwait(false); + storeWrapper.Reset(); + } + finally + { + vectorManager?.ResumeCleanup(); } - - // Reset background replay iterator - ResetReplayIterator(); - - // Reset replication offset - ReplicationOffset = 0; - - // Reset the database in preparation for connecting to primary - storeWrapper.Reset(); // Suspend background tasks that may interfere with AOF await storeWrapper.SuspendPrimaryOnlyTasksAsync().ConfigureAwait(false); + // Stop advance time task when reconfiguring node to be replica + if (storeWrapper.serverOptions.AofPhysicalSublogCount > 1) + await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false); + // Send request to primary // Primary will initiate background task and start sending checkpoint data // @@ -151,7 +169,10 @@ async Task ReplicaSyncAttachTaskAsync(bool downgradeLock, bool forceAsyn logger?.LogCheckpointEntry(LogLevel.Information, nameof(ReplicaSyncAttachTaskAsync), cEntry); storeWrapper.RecoverAOF(); - logger?.LogInformation("InitiateReplicaSync: AOF BeginAddress:{beginAddress} AOF TailAddress:{tailAddress}", storeWrapper.appendOnlyFile.BeginAddress, storeWrapper.appendOnlyFile.TailAddress); + logger?.LogInformation("InitiateReplicaSync: AOF BeginAddress:{beginAddress} AOF TailAddress:{tailAddress}", storeWrapper.appendOnlyFile.Log.BeginAddress, storeWrapper.appendOnlyFile.Log.TailAddress); + + var beginAddress = storeWrapper.appendOnlyFile.Log.BeginAddress; + var tailAddress = storeWrapper.appendOnlyFile.Log.TailAddress; // 1. Primary will signal checkpoint send complete // 2. Replica will receive signal and recover checkpoint, initialize AOF @@ -159,24 +180,22 @@ async Task ReplicaSyncAttachTaskAsync(bool downgradeLock, bool forceAsyn // 4. Replica responds with aofStartAddress sync // 5. Primary will initiate aof sync task // 6. Primary releases checkpoint - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token, resetHandler.Token); - // Exception injection point for testing cluster reset during disk-based replication - await ExceptionInjectionHelper.WaitOnSetAsync(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); - var resp = await gcs.ExecuteReplicaSync( + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); + var resp = await gcs.ExecuteClusterInitiateReplicaSync( nodeId, PrimaryReplId, cEntry.ToByteArray(), - storeWrapper.appendOnlyFile.BeginAddress, - storeWrapper.appendOnlyFile.TailAddress).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); + beginAddress.Span, + tailAddress.Span).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); } catch (Exception ex) { logger?.LogError(ex, "An error occurred at ReplicationManager.RetrieveStoreCheckpoint"); + if (options.AllowReplicaResetOnFailure) - { clusterProvider.clusterManager.TryResetReplica(); - } + return ex.Message; } finally @@ -201,56 +220,19 @@ async Task ReplicaSyncAttachTaskAsync(bool downgradeLock, bool forceAsyn } } - /// - /// Process checkpoint metadata transmitted from primary during replica synchronization. - /// - /// Checkpoint metadata token. - /// Checkpoint metadata filetype. - /// Raw bytes of checkpoint metadata. - /// Throws invalid type checkpoint metadata. - public void ProcessCheckpointMetadata(Guid fileToken, CheckpointFileType fileType, byte[] checkpointMetadata) - { - UpdateLastPrimarySyncTime(); - var ckptManager = fileType switch - { - CheckpointFileType.STORE_SNAPSHOT or - CheckpointFileType.STORE_INDEX => clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main), - CheckpointFileType.OBJ_STORE_SNAPSHOT or - CheckpointFileType.OBJ_STORE_INDEX => clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object), - _ => throw new Exception($"Invalid checkpoint filetype {fileType}"), - }; - - switch (fileType) - { - case CheckpointFileType.STORE_SNAPSHOT: - case CheckpointFileType.OBJ_STORE_SNAPSHOT: - ckptManager.CommitLogCheckpointSendFromPrimary(fileToken, checkpointMetadata); - break; - case CheckpointFileType.STORE_INDEX: - case CheckpointFileType.OBJ_STORE_INDEX: - ckptManager.CommitIndexCheckpoint(fileToken, checkpointMetadata); - break; - default: - throw new Exception($"Invalid checkpoint filetype {fileType}"); - } - } - /// /// Check if device needs to be initialized with a specifi segment size depending on the checkpoint file type /// - /// - /// - public static bool ShouldInitialize(CheckpointFileType type) + /// Checkpoint type + /// Server options to acquire segment bit counts + /// A tuple indicating whether to initialize and, if so, the segment size bits + public static (bool shouldInitialize, int segmentSizeBits) ShouldInitialize(CheckpointFileType type, GarnetServerOptions serverOptions) { - //TODO: verify that the below checkpoint file types require initialization with segment size given as option return type switch { - CheckpointFileType.STORE_HLOG or - CheckpointFileType.STORE_SNAPSHOT or - CheckpointFileType.OBJ_STORE_HLOG or - CheckpointFileType.OBJ_STORE_SNAPSHOT - => true, - _ => false, + CheckpointFileType.STORE_HLOG or CheckpointFileType.STORE_SNAPSHOT => (true, serverOptions.SegmentSizeBits(isObj: false)), + CheckpointFileType.STORE_HLOG_OBJ or CheckpointFileType.STORE_SNAPSHOT_OBJ => (true, serverOptions.SegmentSizeBits(isObj: true)), + _ => (false, 0) }; } @@ -260,42 +242,30 @@ CheckpointFileType.OBJ_STORE_HLOG or /// /// /// - public IDevice GetInitializedSegmentFileDevice(Guid token, CheckpointFileType type) + public IDevice CreateCheckpointDevice(Guid token, CheckpointFileType type) { var device = type switch { - CheckpointFileType.STORE_HLOG => GetStoreHLogDevice(), - CheckpointFileType.OBJ_STORE_HLOG => GetObjectStoreHLogDevice(false),//TODO: return device for object store hlog - CheckpointFileType.OBJ_STORE_HLOG_OBJ => GetObjectStoreHLogDevice(true), - _ => clusterProvider.GetReplicationLogCheckpointManager(type.ToStoreType()).GetDevice(type, token), + CheckpointFileType.STORE_HLOG => GetStoreHLogDevice(isObj: false), + CheckpointFileType.STORE_HLOG_OBJ => GetStoreHLogDevice(isObj: true), + _ => clusterProvider.ReplicationLogCheckpointManager.GetDevice(type, token), }; - if (ShouldInitialize(type)) - device.Initialize(segmentSize: 1L << clusterProvider.serverOptions.SegmentSizeBits()); + var (shouldInitialize, segmentSizeBits) = ShouldInitialize(type, clusterProvider.serverOptions); + if (shouldInitialize) + device.Initialize(segmentSize: 1L << segmentSizeBits); return device; - IDevice GetStoreHLogDevice() + IDevice GetStoreHLogDevice(bool isObj) { var opts = clusterProvider.serverOptions; if (opts.EnableStorageTier) { - var LogDir = opts.LogDir; - if (LogDir is null or "") LogDir = Directory.GetCurrentDirectory(); + var LogDir = !string.IsNullOrEmpty(opts.LogDir) ? opts.LogDir : Directory.GetCurrentDirectory(); var logFactory = opts.GetInitializedDeviceFactory(LogDir); - return logFactory.Get(new FileDescriptor("Store", "hlog")); - } - return null; - } - IDevice GetObjectStoreHLogDevice(bool obj) - { - var opts = clusterProvider.serverOptions; - if (opts.EnableStorageTier) - { - var LogDir = opts.LogDir; - if (LogDir is null or "") LogDir = Directory.GetCurrentDirectory(); - var logFactory = opts.GetInitializedDeviceFactory(LogDir); - return obj ? logFactory.Get(new FileDescriptor("ObjectStore", "hlog.obj")) : logFactory.Get(new FileDescriptor("ObjectStore", "hlog")); + // These must match GarnetServerOptions.GetSettings, EnableStorageTier + return logFactory.Get(new FileDescriptor("Store", isObj ? "hlog_objs" : "hlog")); } return null; } @@ -304,22 +274,21 @@ IDevice GetObjectStoreHLogDevice(bool obj) /// /// Process request from primary to start recovery process from the retrieved checkpoint. /// - /// - /// - /// + /// + /// /// /// /// /// + /// /// - public long BeginReplicaRecover( - bool recoverMainStoreFromToken, - bool recoverObjectStoreFromToken, - bool replayAOF, + public AofAddress TryReplicaDiskbasedRecovery( + bool recoverStoreFromToken, + ulong replayAOFMap, string primaryReplicationId, CheckpointEntry remoteCheckpoint, - long beginAddress, - long recoveredReplicationOffset, + in AofAddress beginAddress, + ref AofAddress recoveredReplicationOffset, out ReadOnlySpan errorMessage) { try @@ -327,56 +296,51 @@ public long BeginReplicaRecover( errorMessage = []; UpdateLastPrimarySyncTime(); - logger?.LogInformation("Replica Recover MainStore: {storeVersion}>[{sIndexToken} {sHlogToken}]" + - "\nObjectStore: {objectStoreVersion}>[{oIndexToken} {oHlogToken}]", + logger?.LogInformation("Replica Recover Store: {storeVersion}>[{sIndexToken} {sHlogToken}]", remoteCheckpoint.metadata.storeVersion, remoteCheckpoint.metadata.storeIndexToken, - remoteCheckpoint.metadata.storeHlogToken, - remoteCheckpoint.metadata.objectStoreVersion, - remoteCheckpoint.metadata.objectStoreIndexToken, - remoteCheckpoint.metadata.objectStoreHlogToken); + remoteCheckpoint.metadata.storeHlogToken); storeWrapper.RecoverCheckpoint( replicaRecover: true, - recoverMainStoreFromToken, - recoverObjectStoreFromToken, + recoverStoreFromToken, remoteCheckpoint.metadata); - if (replayAOF) + if (replayAOFMap > 0) { - logger?.LogInformation("ReplicaRecover: replay local AOF from {beginAddress} until {recoveredReplicationOffset}", beginAddress, recoveredReplicationOffset); - recoveredReplicationOffset = storeWrapper.ReplayAOF(recoveredReplicationOffset); + logger?.LogError("ReplicaRecover: replay local AOF from {beginAddress} until {recoveredReplicationOffset}", beginAddress, recoveredReplicationOffset); + var replayUntil = recoveredReplicationOffset; + for (var sublogIdx = 0; sublogIdx < recoveredReplicationOffset.Length; sublogIdx++) + replayUntil[sublogIdx] = (((1UL) << sublogIdx) > 0) ? recoveredReplicationOffset[sublogIdx] : beginAddress[sublogIdx]; + recoveredReplicationOffset = storeWrapper.ReplayAOF(replayUntil); } logger?.LogInformation("Initializing AOF"); - storeWrapper.appendOnlyFile.Initialize(beginAddress, recoveredReplicationOffset); + storeWrapper.appendOnlyFile.Log.Initialize(beginAddress, recoveredReplicationOffset); + + // Before we can use the replication offset, we must wait for queued Vector Set ops to complete + storeWrapper.DefaultDatabase.VectorManager?.WaitForVectorOperationsToComplete(); // Before we can use the replication offset, we must wait for queued Vector Set ops to complete storeWrapper.DefaultDatabase.VectorManager?.WaitForVectorOperationsToComplete(); // Finally, advertise that we are caught up to the replication offset - ReplicationOffset = recoveredReplicationOffset; - logger?.LogInformation("ReplicaRecover: ReplicaReplicationOffset = {ReplicaReplicationOffset}", ReplicationOffset); + replicationOffset = recoveredReplicationOffset; + logger?.LogInformation("ReplicaRecover: ReplicaReplicationOffset = {ReplicaReplicationOffset}", replicationOffset); // If checkpoint for main store was send add its token here in preparation for purge later on - if (recoverMainStoreFromToken) + if (recoverStoreFromToken) { cEntry.metadata.storeIndexToken = remoteCheckpoint.metadata.storeIndexToken; cEntry.metadata.storeHlogToken = remoteCheckpoint.metadata.storeHlogToken; } - // If checkpoint for object store was send add its token here in preparation for purge later on - if (recoverObjectStoreFromToken) - { - cEntry.metadata.objectStoreIndexToken = remoteCheckpoint.metadata.objectStoreIndexToken; - cEntry.metadata.objectStoreHlogToken = remoteCheckpoint.metadata.objectStoreHlogToken; - } checkpointStore.PurgeAllCheckpointsExceptEntry(cEntry); // Initialize in-memory checkpoint store and delete outdated checkpoint entries logger?.LogInformation("Initializing CheckpointStore"); if (!InitializeCheckpointStore()) - logger?.LogWarning("Failed acquiring latest memory checkpoint metadata at {method}", nameof(BeginReplicaRecover)); + logger?.LogWarning("Failed acquiring latest memory checkpoint metadata at {method}", nameof(TryReplicaDiskbasedRecovery)); // Update replicationId to mark any subsequent checkpoints as part of this history logger?.LogInformation("Updating ReplicationId"); @@ -386,13 +350,16 @@ public long BeginReplicaRecover( // This is necessary to ensure that the stored procedure can perform write operations if needed clusterProvider.replicationManager.aofProcessor.SetReadWriteSession(); - return ReplicationOffset; + // Start advance time signal processing background task + clusterProvider.replicationManager.StartAdvanceTimeBackgroundTask(); + + return this.replicationOffset; } catch (Exception ex) { - logger?.LogError(ex, $"{nameof(BeginReplicaRecover)}"); + logger?.LogError(ex, $"{nameof(TryReplicaDiskbasedRecovery)}"); errorMessage = Encoding.ASCII.GetBytes(ex.Message); - return -1; + return AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, -1); } finally { diff --git a/libs/cluster/Server/Replication/ReplicaOps/ReplicaDisklessSync.cs b/libs/cluster/Server/Replication/ReplicaOps/ReplicaDisklessSync.cs index 972af7a17f3..73a32635aff 100644 --- a/libs/cluster/Server/Replication/ReplicaOps/ReplicaDisklessSync.cs +++ b/libs/cluster/Server/Replication/ReplicaOps/ReplicaDisklessSync.cs @@ -9,6 +9,7 @@ using Garnet.client; using Garnet.cluster.Server.Replication; using Garnet.common; +using Garnet.server; using Microsoft.Extensions.Logging; namespace Garnet.cluster @@ -39,6 +40,9 @@ internal sealed partial class ReplicationManager : IDisposable } } + // Create or update timestamp manager for sharded log if needed + storeWrapper.appendOnlyFile.CreateOrUpdateKeySequenceManager(); + // Wait for threads to agree configuration change of this node if (session != null) { @@ -73,31 +77,56 @@ async Task TryBeginReplicaSyncAsync(bool downgradeLock, bool forceAsync) } var disklessSync = clusterProvider.serverOptions.ReplicaDisklessSync; - var disableObjects = clusterProvider.serverOptions.DisableObjects; GarnetClientSession gcs = null; resetHandler ??= new CancellationTokenSource(); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token, resetHandler.Token); try { - if (!clusterProvider.serverOptions.EnableFastCommit && storeWrapper.appendOnlyFile != null) - { - await storeWrapper.appendOnlyFile.CommitAsync().ConfigureAwait(false); - // TODO: Is this wait necessary? - await storeWrapper.appendOnlyFile.WaitForCommitAsync().ConfigureAwait(false); - } + // Reset background replay tasks if this node was a replica + clusterProvider.replicationManager.ResetReplicaReplayDriverStore(); - // Reset background replay iterator - ResetReplayIterator(); + // Remove aofSync tasks if this node was a primary + aofSyncDriverStore.Reset(); // Reset the database in preparation for connecting to primary // only if we expect to have disk checkpoint to recover from, - // otherwise the replica will receive a reset message from primary if needed + // otherwise the replica will receive a reset message from primary if needed. + // Pause VectorManager's background cleanup task first — Reset's + // post-Phase-2 Initialize() rewinds HeadAddress / BeginAddress / + // TailPageOffset and reallocates pages. Tsavorite's iterator path is + // safe (Initializing flag), but the cleanup task's POST-iterate RMWs + // on metadata records (ClearDeleteInProgress / UpdateContextMetadata) + // are NOT — they can dereference freed pagePointers and AVE. The pause + // serializes the entire cleanup-iteration (iterate + RMWs) with Reset + // by holding cleanupGate, restoring Reset's "store is quiesced" contract. + // + // Pass linkedCts.Token so a slow cleanup-iteration over a large keyspace + // doesn't block re-attach indefinitely if the broader replication is + // cancelled (ctsRepManager / resetHandler). If PauseCleanupAsync throws + // OCE, the try block isn't entered and ResumeCleanup is correctly skipped. if (!disklessSync) - storeWrapper.Reset(); + { + var vectorManager = storeWrapper.DefaultDatabase.VectorManager; + if (vectorManager != null) + await vectorManager.PauseCleanupAsync(linkedCts.Token).ConfigureAwait(false); + try + { + storeWrapper.Reset(); + } + finally + { + vectorManager?.ResumeCleanup(); + } + } // Suspend background tasks that may interfere with AOF await storeWrapper.SuspendPrimaryOnlyTasksAsync().ConfigureAwait(false); + // Stop advance time task when reconfiguring node to be replica + if (storeWrapper.serverOptions.AofPhysicalSublogCount > 1) + await clusterProvider.storeWrapper.TaskManager.CancelAsync(TaskType.AdvanceTimeReplicaTask).ConfigureAwait(false); + // Send request to primary // Primary will initiate background task and start sending checkpoint data // @@ -128,7 +157,8 @@ async Task TryBeginReplicaSyncAsync(bool downgradeLock, bool forceAsync) // Used only for disk-based replication if (!disklessSync) recvCheckpointHandler = new ReceiveCheckpointHandler(clusterProvider, logger); - await gcs.ConnectAsync().ConfigureAwait(false); + + await gcs.ConnectAsync((int)clusterProvider.serverOptions.ReplicaSyncTimeout.TotalMilliseconds, linkedCts.Token).ConfigureAwait(false); SyncMetadata syncMetadata = new( fullSync: false, @@ -136,27 +166,23 @@ async Task TryBeginReplicaSyncAsync(bool downgradeLock, bool forceAsync) originNodeId: current.LocalNodeId, currentPrimaryReplId: PrimaryReplId, currentStoreVersion: storeWrapper.store.CurrentVersion, - currentObjectStoreVersion: disableObjects ? -1 : storeWrapper.objectStore.CurrentVersion, - currentAofBeginAddress: storeWrapper.appendOnlyFile.BeginAddress, - currentAofTailAddress: storeWrapper.appendOnlyFile.TailAddress, + currentAofBeginAddress: storeWrapper.appendOnlyFile.Log.BeginAddress, + currentAofTailAddress: storeWrapper.appendOnlyFile.Log.TailAddress, currentReplicationOffset: ReplicationOffset, checkpointEntry: checkpointEntry); - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token, resetHandler.Token); - // Exception injection point for testing cluster reset during diskless replication - await ExceptionInjectionHelper.WaitOnSetAsync(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); + await ExceptionInjectionHelper.ResetAndWaitAsync(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); - var resp = await gcs.ExecuteAttachSync(syncMetadata.ToByteArray()).WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); + var resp = await gcs.ExecuteClusterAttachSync(syncMetadata.ToByteArray()). + WaitAsync(storeWrapper.serverOptions.ReplicaAttachTimeout, linkedCts.Token).ConfigureAwait(false); } catch (Exception ex) { logger?.LogError(ex, $"{nameof(TryBeginReplicaSyncAsync)}"); if (options.AllowReplicaResetOnFailure) - { clusterProvider.clusterManager.TryResetReplica(); - } return ex.Message; } @@ -182,30 +208,28 @@ async Task TryBeginReplicaSyncAsync(bool downgradeLock, bool forceAsync) } } - public long ReplicaRecoverDiskless(SyncMetadata primarySyncMetadata, out ReadOnlySpan errorMessage) + public AofAddress TryReplicaDisklessRecovery(SyncMetadata primarySyncMetadata, out ReadOnlySpan errorMessage) { try { errorMessage = []; - logger?.LogSyncMetadata(LogLevel.Trace, nameof(ReplicaRecoverDiskless), primarySyncMetadata); + logger?.LogSyncMetadata(LogLevel.Trace, nameof(TryReplicaDisklessRecovery), primarySyncMetadata); var aofBeginAddress = primarySyncMetadata.currentAofBeginAddress; var aofTailAddress = aofBeginAddress; - var replicationOffset = aofBeginAddress; + var _replicationOffset = aofBeginAddress; if (!primarySyncMetadata.fullSync) { // For diskless replication if we are performing a partial sync need to start streaming from replicationOffset // hence our tail needs to be reset to that point - aofTailAddress = replicationOffset = ReplicationOffset; + aofTailAddress = _replicationOffset = this.replicationOffset; } - storeWrapper.appendOnlyFile.Initialize(aofBeginAddress, aofTailAddress); + storeWrapper.appendOnlyFile.Log.Initialize(aofBeginAddress, aofTailAddress); // Set DB version storeWrapper.store.SetVersion(primarySyncMetadata.currentStoreVersion); - if (!clusterProvider.serverOptions.DisableObjects) - storeWrapper.objectStore.SetVersion(primarySyncMetadata.currentObjectStoreVersion); // Update replicationId to mark any subsequent checkpoints as part of this history logger?.LogInformation("Updating ReplicationId"); @@ -214,19 +238,22 @@ public long ReplicaRecoverDiskless(SyncMetadata primarySyncMetadata, out ReadOnl // Before advertising updated replication offset, wait for Vector Set ops to finish storeWrapper.DefaultDatabase.VectorManager?.WaitForVectorOperationsToComplete(); - ReplicationOffset = replicationOffset; + this.replicationOffset = _replicationOffset; // Mark this txn run as a read-write session if we are replaying as a replica // This is necessary to ensure that the stored procedure can perform write operations if needed clusterProvider.replicationManager.aofProcessor.SetReadWriteSession(); - return ReplicationOffset; + // Start advance time signal processing background task + clusterProvider.replicationManager.StartAdvanceTimeBackgroundTask(); + + return this.replicationOffset; } catch (Exception ex) { - logger?.LogError(ex, $"{nameof(ReplicaRecoverDiskless)}"); + logger?.LogError(ex, $"{nameof(TryReplicaDisklessRecovery)}"); errorMessage = Encoding.ASCII.GetBytes(ex.Message); - return -1; + return AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, -1); } finally { diff --git a/libs/cluster/Server/Replication/ReplicaOps/ReplicaReplayTask.cs b/libs/cluster/Server/Replication/ReplicaOps/ReplicaReplayTask.cs deleted file mode 100644 index 6925cc9a59f..00000000000 --- a/libs/cluster/Server/Replication/ReplicaOps/ReplicaReplayTask.cs +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Garnet.common; -using Microsoft.Extensions.Logging; -using Tsavorite.core; - -namespace Garnet.cluster -{ - internal sealed partial class ReplicationManager : IBulkLogEntryConsumer, IDisposable - { - TsavoriteLogScanSingleIterator replayIterator = null; - CancellationTokenSource replicaReplayTaskCts; - SingleWriterMultiReaderLock activeReplay; - - /// - /// Reset background replay iterator - /// - public void ResetReplayIterator() - { - ResetReplayCts(); - replayIterator?.Dispose(); - replayIterator = null; - - void ResetReplayCts() - { - if (replicaReplayTaskCts == null) - { - replicaReplayTaskCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token); - } - else - { - replicaReplayTaskCts.Cancel(); - try - { - activeReplay.WriteLock(); - - replicaReplayTaskCts.Dispose(); - replicaReplayTaskCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token); - } - finally - { - activeReplay.WriteUnlock(); - } - } - } - } - - public void Throttle() { } - - public unsafe void Consume(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) - { - ReplicationOffset = currentAddress; - - var offsetUpdate = 0L; - - try - { - var ptr = record; - while (ptr < record + recordLength) - { - replicaReplayTaskCts.Token.ThrowIfCancellationRequested(); - var entryLength = storeWrapper.appendOnlyFile.HeaderSize; - var payloadLength = storeWrapper.appendOnlyFile.UnsafeGetLength(ptr); - if (payloadLength > 0) - { - aofProcessor.ProcessAofRecordInternal(ptr + entryLength, payloadLength, true, out var isCheckpointStart); - // Encountered checkpoint start marker, log the ReplicationCheckpointStartOffset so we know the correct AOF truncation - // point when we take a checkpoint at the checkpoint end marker - if (isCheckpointStart) - ReplicationCheckpointStartOffset = ReplicationOffset + offsetUpdate; - entryLength += TsavoriteLog.UnsafeAlign(payloadLength); - } - else if (payloadLength < 0) - { - if (!clusterProvider.serverOptions.EnableFastCommit) - { - throw new GarnetException("Received FastCommit request at replica AOF processor, but FastCommit is not enabled", clientResponse: false); - } - TsavoriteLogRecoveryInfo info = new(); - info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); - storeWrapper.appendOnlyFile?.UnsafeCommitMetadataOnly(info, isProtected); - entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); - } - ptr += entryLength; - - offsetUpdate += entryLength; - } - } - catch - { - // If an exception occurrs, be sure to advance ReplicationOffset by the amount of successful work that transpired before the error - - ReplicationOffset += offsetUpdate; - throw; - } - - // Before updating replication offset, we must wait for any pending Vector Set ops to complete - aofProcessor.WaitForVectorOperationsToComplete(); - - // Do the final offset update - we defer until here so Vector Set operations can proceed without waiting after each record is applied - ReplicationOffset += offsetUpdate; - - if (ReplicationOffset != nextAddress) - { - logger?.LogError("ReplicaReplayTask.Consume NextAddress Mismatch recordLength:{recordLength}; currentAddress:{currentAddress}; nextAddress:{nextAddress}; replicationOffset:{ReplicationOffset}", recordLength, currentAddress, nextAddress, ReplicationOffset); - throw new GarnetException($"ReplicaReplayTask.Consume NextAddress Mismatch recordeLength:{recordLength}; currentAddress:{currentAddress}; nextAddress:{nextAddress}; replicationOffset:{ReplicationOffset}", LogLevel.Warning, clientResponse: false); - } - } - - public async Task ReplicaReplayTaskAsync() - { - try - { - activeReplay.ReadLock(); - - await replayIterator.BulkConsumeAllAsync( - this, - clusterProvider.serverOptions.ReplicaSyncDelayMs, - maxChunkSize: 1 << 20, - replicaReplayTaskCts.Token).ConfigureAwait(false); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "An exception occurred at ReplicationManager.ReplicaReplayTask - terminating"); - } - finally - { - activeReplay.ReadUnlock(); - } - } - } -} \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicationCheckpointManagement.cs b/libs/cluster/Server/Replication/ReplicationCheckpointManagement.cs index e22db3ad2b1..d7555b0dbaa 100644 --- a/libs/cluster/Server/Replication/ReplicationCheckpointManagement.cs +++ b/libs/cluster/Server/Replication/ReplicationCheckpointManagement.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using Garnet.server; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -15,7 +16,7 @@ public bool InitializeCheckpointStore() checkpointStore.Initialize(); if (checkpointStore.TryGetLatestCheckpointEntryFromMemory(out var cEntry)) { - aofTaskStore.UpdateTruncatedUntil(cEntry.GetMinAofCoveredAddress()); + aofSyncDriverStore.UpdateTruncatedUntil(cEntry.GetMinAofCoveredAddress()); cEntry.RemoveReader(); return true; } @@ -31,7 +32,7 @@ public bool InitializeCheckpointStore() public bool TryAcquireSettledMetadataForMainStore(CheckpointEntry entry, out LogFileInfo hlog_size, out long index_size) { hlog_size = default; - index_size = -1; + index_size = -1L; try { hlog_size = storeWrapper.store.GetLogFileSize(entry.metadata.storeHlogToken); @@ -45,29 +46,6 @@ public bool TryAcquireSettledMetadataForMainStore(CheckpointEntry entry, out Log } } - /// - /// Keep trying to acquire object store metadata until it settles - /// - /// CheckpointEntry to retrieve metadata for - /// LogFileInfo to return - /// Index size in bytes to return - public bool TryAcquireSettledMetadataForObjectStore(CheckpointEntry entry, out LogFileInfo hlog_size, out long index_size) - { - hlog_size = default; - index_size = -1; - try - { - hlog_size = storeWrapper.objectStore.GetLogFileSize(entry.metadata.objectStoreHlogToken); - index_size = storeWrapper.objectStore.GetIndexFileSize(entry.metadata.objectStoreIndexToken); - return true; - } - catch (Exception ex) - { - logger?.LogError(ex, "Waiting for object store metadata to settle"); - return false; - } - } - /// /// Add new checkpoint entry to the in-memory store /// @@ -89,22 +67,18 @@ public string GetLatestCheckpointFromDiskInfo() => checkpointStore.GetLatestCheckpointFromDiskInfo(); #endregion - public long StoreCurrentSafeAofAddress => clusterProvider.storeWrapper.StoreCheckpointManager.CurrentSafeAofAddress; - public long ObjectStoreCurrentSafeAofAddress => clusterProvider.serverOptions.DisableObjects ? -1 : clusterProvider.storeWrapper.ObjectStoreCheckpointManager.CurrentSafeAofAddress; + public AofAddress StoreCurrentSafeAofAddress => clusterProvider.storeWrapper.StoreCheckpointManager.CurrentSafeAofAddress; - public long StoreRecoveredSafeAofTailAddress => clusterProvider.storeWrapper.StoreCheckpointManager.RecoveredSafeAofAddress; - public long ObjectStoreRecoveredSafeAofTailAddress => clusterProvider.serverOptions.DisableObjects ? -1 : clusterProvider.storeWrapper.ObjectStoreCheckpointManager.RecoveredSafeAofAddress; + public AofAddress StoreRecoveredSafeAofTailAddress => clusterProvider.storeWrapper.StoreCheckpointManager.RecoveredSafeAofAddress; /// /// Update current aof address for pending commit. /// This is necessary to recover safe aof address along with the commit information. /// /// - public void UpdateCommitSafeAofAddress(long safeAofTailAddress) + public void UpdateCommitSafeAofAddress(ref AofAddress safeAofTailAddress) { - clusterProvider.storeWrapper.StoreCheckpointManager.CurrentSafeAofAddress = safeAofTailAddress; - if (!clusterProvider.serverOptions.DisableObjects) - clusterProvider.storeWrapper.ObjectStoreCheckpointManager.CurrentSafeAofAddress = safeAofTailAddress; + clusterProvider.storeWrapper.StoreCheckpointManager.SetCurrentSafeAofAddress(ref safeAofTailAddress); } /// @@ -114,8 +88,6 @@ public void UpdateCommitSafeAofAddress(long safeAofTailAddress) public void SetPrimaryReplicationId() { clusterProvider.storeWrapper.StoreCheckpointManager.CurrentHistoryId = PrimaryReplId; - if (!clusterProvider.serverOptions.DisableObjects) - clusterProvider.storeWrapper.ObjectStoreCheckpointManager.CurrentHistoryId = PrimaryReplId; } } } \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicationHistoryManager.cs b/libs/cluster/Server/Replication/ReplicationHistoryManager.cs index 9fca3999fe5..3778ccf7c1e 100644 --- a/libs/cluster/Server/Replication/ReplicationHistoryManager.cs +++ b/libs/cluster/Server/Replication/ReplicationHistoryManager.cs @@ -3,9 +3,9 @@ using System; using System.IO; -using System.Text; using System.Threading; using Garnet.common; +using Garnet.server; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -13,22 +13,30 @@ namespace Garnet.cluster { internal sealed class ReplicationHistory { - public string primary_replid; + /// + /// Version of the replication history serialization format. + /// Increment when the binary layout of / changes. + /// + public const byte ReplicationHistoryVersion = 1; + + public string PrimaryReplId => primary_replid; + string primary_replid; + public string PrimaryReplId2 => primary_replid2; public string primary_replid2; - public long replicationOffset; - public long replicationOffset2; + AofAddress replicationOffset; + public AofAddress replicationOffset2; - public ReplicationHistory() + public ReplicationHistory(int aofPhysicalSublogCount) { primary_replid = Generator.CreateHexId(); - primary_replid2 = String.Empty; - replicationOffset = 0; - replicationOffset2 = int.MaxValue; + primary_replid2 = string.Empty; + replicationOffset = AofAddress.Create(aofPhysicalSublogCount, 0); + replicationOffset2 = AofAddress.Create(aofPhysicalSublogCount, long.MaxValue); } public ReplicationHistory Copy() { - return new ReplicationHistory() + return new ReplicationHistory(replicationOffset.Length) { primary_replid = primary_replid, primary_replid2 = primary_replid2, @@ -39,33 +47,38 @@ public ReplicationHistory Copy() public byte[] ToByteArray() { - var ms = new MemoryStream(); - var writer = new BinaryWriter(ms, Encoding.ASCII); + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms); + writer.Write(ReplicationHistoryVersion); writer.Write(primary_replid); writer.Write(primary_replid2); - writer.Write(replicationOffset); - writer.Write(replicationOffset2); + replicationOffset.Serialize(writer); + replicationOffset2.Serialize(writer); - byte[] byteArray = ms.ToArray(); - writer.Dispose(); - ms.Dispose(); + var byteArray = ms.ToArray(); return byteArray; } public static ReplicationHistory FromByteArray(byte[] data) { - var ms = new MemoryStream(data); - var reader = new BinaryReader(ms); + using var ms = new MemoryStream(data); + using var reader = new BinaryReader(ms); + + // Read and validate serialization format version + if (data.Length < 1) + throw new InvalidDataException("Invalid ReplicationHistory payload: too short to contain a version"); + + var version = reader.ReadByte(); + if (version != ReplicationHistoryVersion) + throw new InvalidDataException($"Incompatible ReplicationHistory version: expected {ReplicationHistoryVersion}, got {version}"); var primary_replid = reader.ReadString(); var primary_replid2 = reader.ReadString(); - var replicationOffset = reader.ReadInt64(); - var replicationOffset2 = reader.ReadInt64(); + var replicationOffset = AofAddress.Deserialize(reader); + var replicationOffset2 = AofAddress.Deserialize(reader); - reader.Dispose(); - ms.Dispose(); - return new ReplicationHistory() + return new ReplicationHistory(replicationOffset.Length) { primary_replid = primary_replid, primary_replid2 = primary_replid2, @@ -81,7 +94,7 @@ public ReplicationHistory UpdateReplicationId(string primary_replid) return newConfig; } - public ReplicationHistory FailoverUpdate(long replicationOffset2) + public ReplicationHistory FailoverUpdate(AofAddress replicationOffset2) { var newConfig = this.Copy(); newConfig.primary_replid2 = primary_replid; @@ -97,22 +110,24 @@ internal sealed partial class ReplicationManager : IDisposable readonly IDevice replicationConfigDevice; readonly SectorAlignedBufferPool replicationConfigDevicePool; - private void InitializeReplicationHistory() + private void InitializeReplicationHistory(int aofPhysicalSublogCount) { - currentReplicationConfig = new ReplicationHistory(); + currentReplicationConfig = new ReplicationHistory(aofPhysicalSublogCount); FlushConfig(); } private void RecoverReplicationHistory() { var replConfig = ClusterUtils.ReadDevice(replicationConfigDevice, replicationConfigDevicePool, logger); - currentReplicationConfig = ReplicationHistory.FromByteArray(replConfig); - //TODO: handle scenario where replica crashed before became a primary and it has two replication ids - //var current = storeWrapper.clusterManager.CurrentConfig; - //if(current.GetLocalNodeRole() == NodeRole.REPLICA && !primary_replid2.Equals(Generator.DefaultHexId())) - //{ - - //} + try + { + currentReplicationConfig = ReplicationHistory.FromByteArray(replConfig); + } + catch (Exception ex) when (ex is InvalidDataException or EndOfStreamException or IOException) + { + logger?.LogWarning(ex, "Corrupt or incompatible replication history on disk, reinitializing fresh state"); + InitializeReplicationHistory(storeWrapper.serverOptions.AofPhysicalSublogCount); + } } private void TryUpdateMyPrimaryReplId(string primaryReplicationId) @@ -132,14 +147,9 @@ private void TryUpdateMyPrimaryReplId(string primaryReplicationId) /// public void TryUpdateForFailover() { - if (!clusterProvider.serverOptions.EnableFastCommit) - { - storeWrapper.appendOnlyFile?.Commit(); - storeWrapper.appendOnlyFile?.WaitForCommit(); - } while (true) { - var replicationOffset2 = storeWrapper.appendOnlyFile.CommittedUntilAddress; + var replicationOffset2 = storeWrapper.appendOnlyFile.Log.CommittedUntilAddress; var current = currentReplicationConfig; var newConfig = current.FailoverUpdate(replicationOffset2); if (Interlocked.CompareExchange(ref currentReplicationConfig, newConfig, current) == current) diff --git a/libs/cluster/Server/Replication/ReplicationManager.cs b/libs/cluster/Server/Replication/ReplicationManager.cs index c80405fd77e..d42384fcbad 100644 --- a/libs/cluster/Server/Replication/ReplicationManager.cs +++ b/libs/cluster/Server/Replication/ReplicationManager.cs @@ -2,10 +2,10 @@ // Licensed under the MIT license. using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Runtime.CompilerServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -21,6 +21,7 @@ internal sealed partial class ReplicationManager : IDisposable { readonly ClusterProvider clusterProvider; readonly StoreWrapper storeWrapper; + public AofProcessor AofProcessor => aofProcessor; readonly AofProcessor aofProcessor; readonly CheckpointStore checkpointStore; readonly ReplicationSyncManager replicationSyncManager; @@ -28,6 +29,7 @@ internal sealed partial class ReplicationManager : IDisposable CancellationTokenSource resetHandler = new(); readonly int pageSizeBits; + public int PageSizeBits => pageSizeBits; readonly ILogger logger; bool _disposed; @@ -47,68 +49,76 @@ internal sealed partial class ReplicationManager : IDisposable public bool CannotStreamAOF => IsRecovering && currentRecoveryStatus != RecoveryStatus.CheckpointRecoveredAtReplica; - private long replicationOffset; + private AofAddress replicationOffset; - public long ReplicationOffset + public AofAddress ReplicationOffset { get { + if (!storeWrapper.serverOptions.EnableAOF) + return replicationOffset; + // Primary tracks replicationOffset indirectly through AOF tailAddress // Replica will adjust replication offset as it receives data from primary (TODO: since AOFs are synced this might obsolete) var role = clusterProvider.clusterManager.CurrentConfig.LocalNodeRole; - return role == NodeRole.PRIMARY ? - (clusterProvider.serverOptions.EnableAOF && storeWrapper.appendOnlyFile.TailAddress > kFirstValidAofAddress ? storeWrapper.appendOnlyFile.TailAddress : kFirstValidAofAddress) : - replicationOffset; + if (role == NodeRole.PRIMARY) + return storeWrapper.appendOnlyFile.Log.TailAddress; + return replicationOffset; } + } + + /// + /// Return the replication offset for a specific sublog without copying the full AofAddress struct. + /// + /// Index of the physical sublog. + /// The replication offset of the specified sublog. + public long GetReplicationOffset(int sublogIdx) + { + if (!storeWrapper.serverOptions.EnableAOF) + return replicationOffset[sublogIdx]; - set { replicationOffset = value; } + var role = clusterProvider.clusterManager.CurrentConfig.LocalNodeRole; + if (role == NodeRole.PRIMARY) + return storeWrapper.appendOnlyFile.Log.GetTailAddress(sublogIdx); + return replicationOffset[sublogIdx]; } + public void SetSublogReplicationOffset(int sublogIdx, long offset) + => replicationOffset[sublogIdx] = offset; + public long GetSublogReplicationOffset(int sublogIdx) + => replicationOffset[sublogIdx]; + /// /// Replication offset corresponding to the checkpoint start marker. We will truncate only to this point after taking a checkpoint (the checkpoint /// is taken only when we encounter a checkpoint end marker). /// - public long ReplicationCheckpointStartOffset; + public AofAddress ReplicationCheckpointStartOffset; /// /// Replication offset until which AOF address is valid for old primary if failover has occurred /// - public long ReplicationOffset2 - { - get { return currentReplicationConfig.replicationOffset2; } - } + public AofAddress ReplicationOffset2 => currentReplicationConfig.replicationOffset2; - public string PrimaryReplId => currentReplicationConfig.primary_replid; - public string PrimaryReplId2 => currentReplicationConfig.primary_replid2; + public string PrimaryReplId => currentReplicationConfig.PrimaryReplId; + public string PrimaryReplId2 => currentReplicationConfig.PrimaryReplId2; /// /// Recovery status /// public RecoveryStatus currentRecoveryStatus; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public GarnetClusterCheckpointManager GetCkptManager(StoreType storeType) - { - return storeType switch - { - StoreType.Main => (GarnetClusterCheckpointManager)storeWrapper.store.CheckpointManager, - StoreType.Object => (GarnetClusterCheckpointManager)storeWrapper.objectStore?.CheckpointManager, - _ => throw new Exception($"GetCkptManager: unexpected state {storeType}") - }; - } + public GarnetClusterCheckpointManager CheckpointManager => (GarnetClusterCheckpointManager)storeWrapper.store.CheckpointManager; - public long GetRecoveredSafeAofAddress() + public AofAddress GetRecoveredSafeAofAddress() { - var storeAofAddress = clusterProvider.replicationManager.GetCkptManager(StoreType.Main).RecoveredSafeAofAddress; - var objectStoreAofAddress = clusterProvider.serverOptions.DisableObjects ? long.MaxValue : clusterProvider.replicationManager.GetCkptManager(StoreType.Object).RecoveredSafeAofAddress; - return Math.Min(storeAofAddress, objectStoreAofAddress); + var storeAofAddress = clusterProvider.replicationManager.CheckpointManager.RecoveredSafeAofAddress; + return storeAofAddress; } - public long GetCurrentSafeAofAddress() + public AofAddress GetCurrentSafeAofAddress() { - var storeAofAddress = clusterProvider.replicationManager.GetCkptManager(StoreType.Main).CurrentSafeAofAddress; - var objectStoreAofAddress = clusterProvider.serverOptions.DisableObjects ? long.MaxValue : clusterProvider.replicationManager.GetCkptManager(StoreType.Object).CurrentSafeAofAddress; - return Math.Min(storeAofAddress, objectStoreAofAddress); + var storeAofAddress = clusterProvider.replicationManager.CheckpointManager.CurrentSafeAofAddress; + return storeAofAddress; } public ReplicationManager(ClusterProvider clusterProvider, ILogger logger = null) @@ -117,33 +127,29 @@ public ReplicationManager(ClusterProvider clusterProvider, ILogger logger = null this.logger = logger; this.clusterProvider = clusterProvider; this.storeWrapper = clusterProvider.storeWrapper; - this.pageSizeBits = storeWrapper.appendOnlyFile == null ? 0 : storeWrapper.appendOnlyFile.UnsafeGetLogPageSizeBits(); + this.pageSizeBits = storeWrapper.appendOnlyFile == null ? 0 : storeWrapper.appendOnlyFile.Log.UnsafeGetLogPageSizeBits(); networkBufferSettings.Log(logger, nameof(ReplicationManager)); - this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); + this.networkPool = networkBufferSettings.CreateBufferPool(ownerType: PoolOwnerType.Replication, logger: logger); ValidateNetworkBufferSettings(); aofProcessor = new AofProcessor(storeWrapper, recordToAof: false, clusterProvider: clusterProvider, logger: logger); replicaSyncSessionTaskStore = new ReplicaSyncSessionTaskStore(storeWrapper, clusterProvider, logger); replicationSyncManager = new ReplicationSyncManager(clusterProvider, logger); - ReplicationOffset = 0; + replicationOffset = AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, kFirstValidAofAddress); + ReplicationCheckpointStartOffset = AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, kFirstValidAofAddress); // Set the appendOnlyFile field for all stores - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main).checkpointVersionShiftStart = CheckpointVersionShiftStart; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Main).checkpointVersionShiftEnd = CheckpointVersionShiftEnd; - if (storeWrapper.objectStore != null) - { - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object).checkpointVersionShiftStart = CheckpointVersionShiftStart; - clusterProvider.GetReplicationLogCheckpointManager(StoreType.Object).checkpointVersionShiftEnd = CheckpointVersionShiftEnd; - } + clusterProvider.ReplicationLogCheckpointManager.checkpointVersionShiftStart = CheckpointVersionShiftStart; + clusterProvider.ReplicationLogCheckpointManager.checkpointVersionShiftEnd = CheckpointVersionShiftEnd; // If this node starts as replica, it cannot serve requests until it is connected to primary if (clusterProvider.clusterManager.CurrentConfig.LocalNodeRole == NodeRole.REPLICA && clusterProvider.serverOptions.Recover && !BeginRecovery(RecoveryStatus.InitializeRecover, upgradeLock: false)) throw new Exception(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_CANNOT_ACQUIRE_RECOVERY_LOCK)); checkpointStore = new CheckpointStore(storeWrapper, clusterProvider, true, logger); - aofTaskStore = new(clusterProvider, 1, logger); + aofSyncDriverStore = new(clusterProvider, 1, logger); var clusterFolder = "/cluster"; var clusterDataPath = opts.CheckpointDir + clusterFolder; @@ -164,12 +170,11 @@ public ReplicationManager(ClusterProvider clusterProvider, ILogger logger = null logger?.LogTrace("Initializing new in-memory checkpoint registry"); // If recover option is not enabled or replication history is not available // initialize new empty replication history. - InitializeReplicationHistory(); + InitializeReplicationHistory(storeWrapper.serverOptions.AofPhysicalSublogCount); } // After initializing replication history propagate replicationId to ReplicationLogCheckpointManager SetPrimaryReplicationId(); - replicaReplayTaskCts = CancellationTokenSource.CreateLinkedTokenSource(ctsRepManager.Token); } /// @@ -486,7 +491,7 @@ public void ResetRecovery() public void Dispose() { _disposed = true; - + ctsRepManager.Cancel(); replicationConfigDevice?.Dispose(); replicationConfigDevicePool?.Free(); @@ -494,14 +499,9 @@ public void Dispose() checkpointStore.WaitForReplicas(); replicaSyncSessionTaskStore.Dispose(); - replicaReplayTaskCts.Cancel(); - activeReplay.WriteLock(); - replicaReplayTaskCts.Dispose(); - ctsRepManager.Cancel(); + ReplicaReplayDriverStore?.Dispose(); ctsRepManager.Dispose(); - resetHandler.Cancel(); - resetHandler.Dispose(); - aofTaskStore.Dispose(); + aofSyncDriverStore.Dispose(); aofProcessor?.Dispose(); networkPool?.Dispose(); } @@ -543,16 +543,14 @@ private void RecoverCheckpointAndAOF() { // If recovered checkpoint corresponds to an unavailable AOF address, we initialize AOF to that address var recoveredSafeAofAddress = GetRecoveredSafeAofAddress(); - if (storeWrapper.appendOnlyFile.TailAddress < recoveredSafeAofAddress) - storeWrapper.appendOnlyFile.Initialize(recoveredSafeAofAddress, recoveredSafeAofAddress); - logger?.LogInformation("Recovered AOF: begin address = {beginAddress}, tail address = {tailAddress}", storeWrapper.appendOnlyFile.BeginAddress, storeWrapper.appendOnlyFile.TailAddress); - - var replayAofOffset = storeWrapper.ReplayAOF(); + storeWrapper.appendOnlyFile.Log.InitializeIf(ref recoveredSafeAofAddress); + logger?.LogInformation("Recovered AOF: begin address = {beginAddress}, tail address = {tailAddress}", storeWrapper.appendOnlyFile.Log.BeginAddress, storeWrapper.appendOnlyFile.Log.TailAddress); + var replayedUntil = storeWrapper.ReplayAOF(AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, -1)); // Before advertising new replication offset, wait for any queued Vector Set ops to complete storeWrapper.DefaultDatabase.VectorManager?.WaitForVectorOperationsToComplete(); - ReplicationOffset = replayAofOffset; + replicationOffset.SetValue(ref replayedUntil); } // First recover and then load latest checkpoint info in-memory @@ -565,11 +563,11 @@ private void RecoverCheckpointAndAOF() /// /// /// - public async Task WaitForReplicationOffsetAsync(long primaryReplicationOffset) + public async Task WaitForReplicationOffsetAsync(AofAddress primaryReplicationOffset) { - while (ReplicationOffset < primaryReplicationOffset) + while (ReplicationOffset.AnyLesser(primaryReplicationOffset)) { - if (ctsRepManager.IsCancellationRequested) return -1; + if (ctsRepManager.IsCancellationRequested) return AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, -1); await Task.Yield(); } return ReplicationOffset; @@ -611,21 +609,107 @@ public void Start() } else if (localNodeRole == NodeRole.PRIMARY && replicaOfNodeId == null) { - var replicaIds = current.GetLocalNodeReplicaIds(); - foreach (var replicaId in replicaIds) - { - // TODO: Initiate AOF sync task correctly when restarting primary - if (clusterProvider.replicationManager.TryAddReplicationTask(replicaId, 0, out var aofSyncTaskInfo)) - { - if (!TryConnectToReplica(replicaId, 0, aofSyncTaskInfo, out var errorMessage)) - logger?.LogError("{errorMessage}", Encoding.ASCII.GetString(errorMessage)); - } - } + // Restarting as a primary we do nothing. + // The replica will have to initiate the recovery process. } else { logger?.LogWarning("Replication manager starting configuration inconsistent role:{role} replicaOfId:{replicaOfNodeId}", replicaOfNodeId, localNodeRole); } } + + struct AdvanceTimeEvent + { + public long sequenceNumber; + public AofAddress tailAddress; + } + + ConcurrentStack advanceTimeWorkQueue; + SingleWaiterAutoResetEvent onAdvanceTimeSignal; + SingleWaiterAutoResetEvent onAdvanceTimeWorkerStart; + + /// + /// Process message from primary related to observing a specific tail address snapshot at a given sequence number (timestamp). + /// + /// Sequence number associated with observing the given tail address. + /// Tail address snapshot. + /// + /// + public void SignalAdvanceTime(long sequenceNumber, AofAddress tailAddress) + { + advanceTimeWorkQueue?.Push(new() { sequenceNumber = sequenceNumber, tailAddress = tailAddress }); + onAdvanceTimeSignal?.Signal(); + } + + /// + /// Start replica background task to process advance time signals from the primary. + /// + /// + public void StartAdvanceTimeBackgroundTask() + { + // NOTE: At this point the AdvanceTimeReplicaTask should not be running. This applies to both Single and MultiLog cases. + // In SingleLog the task should not be spawned and for multi-log it should have been disposed at the beginning of sync. + Debug.Assert(!storeWrapper.TaskManager.IsRunning(TaskType.AdvanceTimeReplicaTask), "AdvanceTimeReplicaTask should be not running at this stage!"); + onAdvanceTimeWorkerStart = new(); + if (clusterProvider.serverOptions.AofPhysicalSublogCount > 1 && + !clusterProvider.storeWrapper.TaskManager.RegisterAndRun(TaskType.AdvanceTimeReplicaTask, (token) => AdvanceTimeWorkerAsync(token))) + { + logger?.LogError("Failed to register AdvanceTime task at the replica"); + throw new GarnetException("Failed to register AdvanceTime task at the replica"); + } + + _ = onAdvanceTimeWorkerStart?.WaitAsync().AsTask().WaitAsync(storeWrapper.serverOptions.ReplicaSyncTimeout); + + async Task AdvanceTimeWorkerAsync(CancellationToken token) + { + var appendOnlyFile = storeWrapper.appendOnlyFile; + advanceTimeWorkQueue = new(); + onAdvanceTimeSignal = new() { RunContinuationsAsynchronously = true }; + onAdvanceTimeWorkerStart.Signal(); + try + { + while (!token.IsCancellationRequested) + { + var advanceTimeSignalTask = onAdvanceTimeSignal.WaitAsync().AsTask(); + await advanceTimeSignalTask.WaitAsync(token).ConfigureAwait(false); + + while (advanceTimeWorkQueue.TryPop(out var result)) + { + var observationSequenceNumber = result.sequenceNumber; + var observedTailAddress = result.tailAddress; + var converged = false; + while (!converged) + { + converged = true; + for (var i = 0; i < observedTailAddress.Length; i++) + { + // Move logical time forward for sublog if the replay has progressed at least until the tailAddress + if (observedTailAddress[i] <= replicationOffset[i]) + appendOnlyFile.readConsistencyManager.UpdatePhysicalSublogMaxSequenceNumber(i, observationSequenceNumber); + else + converged = false; + } + await Task.Delay(storeWrapper.serverOptions.AofTailWitnessFreqMs, token).ConfigureAwait(false); + } + } + } + } + catch (TaskCanceledException) when (token.IsCancellationRequested) + { + // Suppress the exception if the task was cancelled because of store wrapper disposal + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed at {method}", nameof(AdvanceTimeWorkerAsync)); + } + finally + { + advanceTimeWorkQueue.Clear(); + advanceTimeWorkQueue = null; + onAdvanceTimeWorkerStart = null; + onAdvanceTimeSignal = null; + } + } + } } } \ No newline at end of file diff --git a/libs/cluster/Server/Replication/ReplicationNetworkBufferSettings.cs b/libs/cluster/Server/Replication/ReplicationNetworkBufferSettings.cs index 14ed7d47de5..ab72a13a944 100644 --- a/libs/cluster/Server/Replication/ReplicationNetworkBufferSettings.cs +++ b/libs/cluster/Server/Replication/ReplicationNetworkBufferSettings.cs @@ -12,7 +12,7 @@ internal sealed partial class ReplicationManager : IDisposable /// /// NetworkBufferSettings for the buffer pool maintained by the ReplicationManager /// - NetworkBufferSettings networkBufferSettings => NetworkBufferSettings.GetInclusive([GetRSSNetworkBufferSettings, GetIRSNetworkBufferSettings, GetAofSyncNetworkBufferSettings]); + internal NetworkBufferSettings networkBufferSettings => NetworkBufferSettings.GetInclusive([GetRSSNetworkBufferSettings, GetIRSNetworkBufferSettings, GetAofSyncNetworkBufferSettings]); /// /// Network pool maintained by the ReplicationManager diff --git a/libs/cluster/Server/Replication/SyncMetadata.cs b/libs/cluster/Server/Replication/SyncMetadata.cs index 927f600e123..bad9223436b 100644 --- a/libs/cluster/Server/Replication/SyncMetadata.cs +++ b/libs/cluster/Server/Replication/SyncMetadata.cs @@ -3,6 +3,7 @@ using System.IO; using System.Text; +using Garnet.server; using Microsoft.Extensions.Logging; namespace Garnet.cluster @@ -26,7 +27,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m "originNodeId:{originNodeId}\n" + "currentPrimaryReplId:{currentPrimaryReplId}\n" + "currentStoreVersion:{currentStoreVersion}\n" + - "currentObjectStoreVersion:{currentObjectStoreVersion}\n" + "currentAofBeginAddress:{currentAofBeginAddress}\n" + "currentAofTailAddress:{currentAofTailAddress}\n" + "currentReplicationOffset:{currentReplicationOffset}\n" + @@ -37,7 +37,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m syncMetadata.originNodeId, syncMetadata.currentPrimaryReplId, syncMetadata.currentStoreVersion, - syncMetadata.currentObjectStoreVersion, syncMetadata.currentAofBeginAddress, syncMetadata.currentAofTailAddress, syncMetadata.currentReplicationOffset, @@ -63,7 +62,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m "originNodeId:{originNodeId}\n" + "currentPrimaryReplId:{currentPrimaryReplId}\n" + "currentStoreVersion:{currentStoreVersion}\n" + - "currentObjectStoreVersion:{currentObjectStoreVersion}\n" + "currentAofBeginAddress:{currentAofBeginAddress}\n" + "currentAofTailAddress:{currentAofTailAddress}\n" + "currentReplicationOffset:{currentReplicationOffset}\n" + @@ -74,7 +72,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m "recoverOriginNodeId:{originNodeId}\n" + "recoverCurrentPrimaryReplId:{currentPrimaryReplId}\n" + "recoverCurrentStoreVersion:{currentStoreVersion}\n" + - "recoverCurrentObjectStoreVersion:{currentObjectStoreVersion}\n" + "recoverCurrentAofBeginAddress:{currentAofBeginAddress}\n" + "recoverCurrentAofTailAddress:{currentAofTailAddress}\n" + "recoverCurrentReplicationOffset:{currentReplicationOffset}\n" + @@ -85,7 +82,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m origin.originNodeId, origin.currentPrimaryReplId, origin.currentStoreVersion, - origin.currentObjectStoreVersion, origin.currentAofBeginAddress, origin.currentAofTailAddress, origin.currentReplicationOffset, @@ -95,7 +91,6 @@ public static void LogSyncMetadata(this ILogger log, LogLevel logLevel, string m local.originNodeId, local.currentPrimaryReplId, local.currentStoreVersion, - local.currentObjectStoreVersion, local.currentAofBeginAddress, local.currentAofTailAddress, local.currentReplicationOffset, @@ -109,10 +104,9 @@ internal sealed class SyncMetadata( string originNodeId, string currentPrimaryReplId, long currentStoreVersion, - long currentObjectStoreVersion, - long currentAofBeginAddress, - long currentAofTailAddress, - long currentReplicationOffset, + AofAddress currentAofBeginAddress, + AofAddress currentAofTailAddress, + AofAddress currentReplicationOffset, CheckpointEntry checkpointEntry) { public readonly bool fullSync = fullSync; @@ -120,10 +114,9 @@ internal sealed class SyncMetadata( public readonly string originNodeId = originNodeId; public readonly string currentPrimaryReplId = currentPrimaryReplId; public readonly long currentStoreVersion = currentStoreVersion; - public readonly long currentObjectStoreVersion = currentObjectStoreVersion; - public readonly long currentAofBeginAddress = currentAofBeginAddress; - public readonly long currentAofTailAddress = currentAofTailAddress; - public readonly long currentReplicationOffset = currentReplicationOffset; + public readonly AofAddress currentAofBeginAddress = currentAofBeginAddress; + public readonly AofAddress currentAofTailAddress = currentAofTailAddress; + public readonly AofAddress currentReplicationOffset = currentReplicationOffset; public readonly CheckpointEntry checkpointEntry = checkpointEntry; public byte[] ToByteArray() @@ -137,11 +130,10 @@ public byte[] ToByteArray() writer.Write(currentPrimaryReplId); writer.Write(currentStoreVersion); - writer.Write(currentObjectStoreVersion); - writer.Write(currentAofBeginAddress); - writer.Write(currentAofTailAddress); - writer.Write(currentReplicationOffset); + currentAofBeginAddress.Serialize(writer); + currentAofTailAddress.Serialize(writer); + currentReplicationOffset.Serialize(writer); if (checkpointEntry != null) { @@ -168,10 +160,9 @@ public static SyncMetadata FromByteArray(byte[] serialized) originNodeId: reader.ReadString(), currentPrimaryReplId: reader.ReadString(), currentStoreVersion: reader.ReadInt64(), - currentObjectStoreVersion: reader.ReadInt64(), - currentAofBeginAddress: reader.ReadInt64(), - currentAofTailAddress: reader.ReadInt64(), - currentReplicationOffset: reader.ReadInt64(), + currentAofBeginAddress: AofAddress.Deserialize(reader), + currentAofTailAddress: AofAddress.Deserialize(reader), + currentReplicationOffset: AofAddress.Deserialize(reader), checkpointEntry: CheckpointEntry.FromByteArray(reader.ReadBytes(reader.ReadInt32())) ); return syncMetadata; diff --git a/libs/cluster/Session/ClusterCommands.cs b/libs/cluster/Session/ClusterCommands.cs index d938b710340..877f87ec977 100644 --- a/libs/cluster/Session/ClusterCommands.cs +++ b/libs/cluster/Session/ClusterCommands.cs @@ -15,41 +15,20 @@ internal sealed unsafe partial class ClusterSession : IClusterSession { ClusterConfig lastSentConfig; - private int CountKeysInSessionStore(int slot) + private int CountKeysInSlot(int slot) { - ClusterKeyIterationFunctions.MainStoreCountKeys iterFuncs = new(slot); + ClusterKeyIterationFunctions.CountKeys iterFuncs = new(slot); var cursor = 0L; - _ = basicGarnetApi.IterateMainStore(ref iterFuncs, ref cursor); + _ = basicGarnetApi.IterateStore(ref iterFuncs, ref cursor); return iterFuncs.KeyCount; } - private int CountKeysInObjectStore(int slot) - { - if (!clusterProvider.serverOptions.DisableObjects) - { - ClusterKeyIterationFunctions.ObjectStoreCountKeys iterFuncs = new(slot); - var cursor = 0L; - _ = basicGarnetApi.IterateObjectStore(ref iterFuncs, ref cursor); - return iterFuncs.KeyCount; - } - return 0; - } - - private int CountKeysInSlot(int slot) => CountKeysInSessionStore(slot) + CountKeysInObjectStore(slot); - private List GetKeysInSlot(int slot, int keyCount) { List keys = []; - ClusterKeyIterationFunctions.MainStoreGetKeysInSlot mainIterFuncs = new(keys, slot, keyCount); + ClusterKeyIterationFunctions.GetKeysInSlot iterFuncs = new(keys, slot, keyCount); var cursor = 0L; - _ = basicGarnetApi.IterateMainStore(ref mainIterFuncs, ref cursor); - - if (!clusterProvider.serverOptions.DisableObjects) - { - ClusterKeyIterationFunctions.ObjectStoreGetKeysInSlot objectIterFuncs = new(keys, slot); - var objectCursor = 0L; - _ = basicGarnetApi.IterateObjectStore(ref objectIterFuncs, ref objectCursor); - } + _ = basicGarnetApi.IterateStore(ref iterFuncs, ref cursor); return keys; } @@ -141,7 +120,7 @@ private void ProcessClusterCommands(RespCommand command, VectorManager vectorMan { RespCommand.CLUSTER_ADDSLOTS => NetworkClusterAddSlots(out invalidParameters), RespCommand.CLUSTER_ADDSLOTSRANGE => NetworkClusterAddSlotsRange(out invalidParameters), - RespCommand.CLUSTER_AOFSYNC => NetworkClusterAOFSync(out invalidParameters), + RespCommand.CLUSTER_ADVANCE_TIME => NetworkClusterAdvanceTime(out invalidParameters), RespCommand.CLUSTER_APPENDLOG => NetworkClusterAppendLog(out invalidParameters), RespCommand.CLUSTER_ATTACH_SYNC => NetworkClusterAttachSync(out invalidParameters), RespCommand.CLUSTER_BANLIST => NetworkClusterBanList(out invalidParameters), @@ -183,6 +162,8 @@ private void ProcessClusterCommands(RespCommand command, VectorManager vectorMan RespCommand.CLUSTER_SHARDS => NetworkClusterShards(out invalidParameters), RespCommand.CLUSTER_SLOTS => NetworkClusterSlots(out invalidParameters), RespCommand.CLUSTER_SLOTSTATE => NetworkClusterSlotState(out invalidParameters), + RespCommand.CLUSTER_SNAPSHOT_DATA => NetworkClusterSnapshotData(out invalidParameters), + RespCommand.CLUSTER_MLOG_KEY_TIME => NetworkClusterMlogKeyTime(out invalidParameters), RespCommand.CLUSTER_SYNC => NetworkClusterSync(out invalidParameters), _ => throw new Exception($"Unexpected cluster subcommand: {command}") }; diff --git a/libs/cluster/Session/ClusterKeyIterationFunctions.cs b/libs/cluster/Session/ClusterKeyIterationFunctions.cs index af011f3798c..24291c4e44f 100644 --- a/libs/cluster/Session/ClusterKeyIterationFunctions.cs +++ b/libs/cluster/Session/ClusterKeyIterationFunctions.cs @@ -22,7 +22,7 @@ internal class KeyIterationInfo internal KeyIterationInfo(int slot) => this.slot = slot; } - internal sealed class MainStoreCountKeys : IScanIteratorFunctions + internal sealed class CountKeys : IScanIteratorFunctions { private readonly KeyIterationInfo info; // This must be a class as it is passed through pending IO operations @@ -30,115 +30,44 @@ internal sealed class MainStoreCountKeys : IScanIteratorFunctions info.keyCount; set => info.keyCount = value; } internal int Slot => info.slot; - internal MainStoreCountKeys(int slot) => info = new(slot); + internal CountKeys(int slot) => info = new(slot); - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - // TODO: better way to detect namespace - if (key.MetadataSize == 1) - { - // Namespace means not visible - cursorRecordResult = CursorRecordResult.Skip; - return true; - } - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - if (HashSlotUtils.HashSlot(ref key) == Slot && !Expired(ref value)) + if (HashSlotUtils.HashSlot(srcLogRecord.Key) == Slot && !Expired(in srcLogRecord)) KeyCount++; return true; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public bool OnStart(long beginAddress, long endAddress) => true; - public void OnStop(bool completed, long numberOfRecords) { } - public void OnException(Exception exception, long numberOfRecords) { } - } - - internal sealed class ObjectStoreCountKeys : IScanIteratorFunctions - { - private readonly KeyIterationInfo info; - // This must be a class as it is passed through pending IO operations - internal int KeyCount { get => info.keyCount; set => info.keyCount = value; } - internal int Slot => info.slot; - - internal ObjectStoreCountKeys(int slot) => info = new(slot); - - public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here , out CursorRecordResult cursorRecordResult - fixed (byte* keyPtr = key) - { - if (HashSlotUtils.HashSlot(keyPtr, key.Length) == Slot && !Expired(ref value)) - KeyCount++; - } - return true; - } - public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } } - internal readonly struct MainStoreGetKeysInSlot : IScanIteratorFunctions + internal readonly struct GetKeysInSlot : IScanIteratorFunctions { readonly List keys; readonly int slot, maxKeyCount; - internal MainStoreGetKeysInSlot(List keys, int slot, int maxKeyCount) + internal GetKeysInSlot(List keys, int slot, int maxKeyCount) { this.keys = keys; this.slot = slot; this.maxKeyCount = maxKeyCount; } - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - // TODO: better way to detect namespace - if (key.MetadataSize == 1) - { - // Namespace means not visible - cursorRecordResult = CursorRecordResult.Skip; - return true; - } - cursorRecordResult = CursorRecordResult.Accept; // default; not used here, out CursorRecordResult cursorRecordResult - - if (HashSlotUtils.HashSlot(ref key) == slot && !Expired(ref value)) - keys.Add(key.ToByteArray()); + var key = srcLogRecord.Key; + if (HashSlotUtils.HashSlot(key) == slot && !Expired(in srcLogRecord)) + keys.Add(key.ToArray()); return keys.Count < maxKeyCount; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public bool OnStart(long beginAddress, long endAddress) => true; - public void OnStop(bool completed, long numberOfRecords) { } - public void OnException(Exception exception, long numberOfRecords) { } - } - internal readonly struct ObjectStoreGetKeysInSlot : IScanIteratorFunctions - { - readonly List keys; - readonly int slot; - - internal ObjectStoreGetKeysInSlot(List keys, int slot) - { - this.keys = keys; - this.slot = slot; - } - - public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - fixed (byte* keyPtr = key) - { - if (HashSlotUtils.HashSlot(keyPtr, key.Length) == slot && !Expired(ref value)) - keys.Add(key); - } - return true; - } - public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } diff --git a/libs/cluster/Session/ClusterSession.cs b/libs/cluster/Session/ClusterSession.cs index 09d0dc7cbcd..969b116b1ff 100644 --- a/libs/cluster/Session/ClusterSession.cs +++ b/libs/cluster/Session/ClusterSession.cs @@ -9,26 +9,9 @@ using Garnet.server.ACL; using Garnet.server.Auth; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.cluster { - using BasicContext = BasicContext, - SpanByteAllocator>>; - - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - - using VectorContext = BasicContext, SpanByteAllocator>>; - internal sealed partial class ClusterSession : IClusterSession { readonly ClusterProvider clusterProvider; @@ -67,20 +50,10 @@ internal sealed partial class ClusterSession : IClusterSession /// public IGarnetServer Server { get; set; } - private VectorContext vectorContext; - private BasicContext basicContext; - - public ClusterSession( - ClusterProvider clusterProvider, - TransactionManager txnManager, - IGarnetAuthenticator authenticator, - UserHandle userHandle, - GarnetSessionMetrics sessionMetrics, - BasicGarnetApi basicGarnetApi, - BasicContext basicContext, - VectorContext vectorContext, - INetworkSender networkSender, - ILogger logger = null) + private StringBasicContext stringBasicContext; + private VectorBasicContext vectorBasicContext; + + public ClusterSession(ClusterProvider clusterProvider, TransactionManager txnManager, IGarnetAuthenticator authenticator, UserHandle userHandle, GarnetSessionMetrics sessionMetrics, BasicGarnetApi basicGarnetApi, StringBasicContext stringBasicContext, VectorBasicContext vectorBasicContext, INetworkSender networkSender, ILogger logger = null) { this.clusterProvider = clusterProvider; this.authenticator = authenticator; @@ -88,8 +61,8 @@ public ClusterSession( this.txnManager = txnManager; this.sessionMetrics = sessionMetrics; this.basicGarnetApi = basicGarnetApi; - this.basicContext = basicContext; - this.vectorContext = vectorContext; + this.stringBasicContext = stringBasicContext; + this.vectorBasicContext = vectorBasicContext; this.networkSender = networkSender; this.logger = logger; } @@ -103,13 +76,13 @@ public unsafe void ProcessClusterCommands(RespCommand command, VectorManager vec try { - RespCommandsInfo commandInfo = null; if (command.IsClusterSubCommand()) { - if (RespCommandsInfo.TryGetRespCommandInfo(command, out commandInfo) && commandInfo.KeySpecifications != null) + if (RespCommandsInfo.TryGetSimpleRespCommandInfo(command, out var cmdInfo) && cmdInfo.KeySpecs?.Length > 0) { - csvi.keyNumOffset = -1; - clusterProvider.ExtractKeySpecs(commandInfo, command, ref parseState, ref csvi); + csvi.keySpecs = cmdInfo.KeySpecs; + csvi.isSubCommand = cmdInfo.IsSubCommand; + csvi.readOnly = command.IsReadOnly(); if (NetworkMultiKeySlotVerifyNoResponse(ref parseState, ref csvi, ref this.dcurr, ref this.dend)) return; } @@ -129,7 +102,7 @@ public unsafe void ProcessClusterCommands(RespCommand command, VectorManager vec if (invalidParameters) { - var cmdName = commandInfo?.Name ?? RespCommandsInfo.GetRespCommandName(command); + var cmdName = RespCommandsInfo.GetRespCommandName(command); var errorMessage = string.Format(CmdStrings.GenericErrWrongNumArgs, cmdName.ToLowerInvariant()); while (!RespWriteUtils.TryWriteError(errorMessage, ref this.dcurr, this.dend)) SendAndReset(); @@ -178,7 +151,8 @@ unsafe void Send(byte* d) // Debug.WriteLine("SEND: [" + Encoding.UTF8.GetString(new Span(d, (int)(dcurr - d))).Replace("\n", "|").Replace("\r", "!") + "]"); if (clusterProvider.storeWrapper.appendOnlyFile != null && clusterProvider.storeWrapper.serverOptions.WaitForCommit) { - clusterProvider.storeWrapper.appendOnlyFile.WaitForCommit(); + var task = clusterProvider.storeWrapper.appendOnlyFile.Log.WaitForCommitAsync(); + if (!task.IsCompletedSuccessfully) AsyncUtils.BlockingWait(task); } int sendBytes = (int)(dcurr - d); networkSender.SendResponse((int)(d - networkSender.GetResponseObjectHead()), sendBytes); @@ -206,5 +180,26 @@ public async Task UnsafeBumpAndWaitForEpochTransitionAsync() _ = await clusterProvider.BumpAndWaitForEpochTransitionAsync().ConfigureAwait(false); AcquireCurrentEpoch(); } + + /// + /// NOTE: Unsafe! DO NOT USE, other than benchmarking + /// + /// + public void UnsafeSetConfig(string replicaOf = null) + { + var config = clusterProvider.clusterManager.CurrentConfig; + config = config.MakeReplicaOf(replicaOf); + clusterProvider.clusterManager.UnsafeSetConfig(config); + + if (replicaOf != null) + clusterProvider.replicationManager.ResetReplicaReplayDriverStore(); + } + + public void Dispose() + { + // Call dispose on ref of this session if this session is a replication task + if (IsReplicating) + replicaReplayDriverStore?.Dispose(); + } } } \ No newline at end of file diff --git a/libs/cluster/Session/MigrateCommand.cs b/libs/cluster/Session/MigrateCommand.cs index 9babf5b872e..a43d690d185 100644 --- a/libs/cluster/Session/MigrateCommand.cs +++ b/libs/cluster/Session/MigrateCommand.cs @@ -14,9 +14,9 @@ namespace Garnet.cluster { internal sealed unsafe partial class ClusterSession : IClusterSession { - public static bool Expired(ref SpanByte value) => value.MetadataSize == 8 && value.ExtraMetadata < DateTimeOffset.UtcNow.Ticks; - - public static bool Expired(ref IGarnetObject value) => value.Expiration != 0 && value.Expiration < DateTimeOffset.UtcNow.Ticks; + public static bool Expired(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => logRecord.Info.HasExpiration && logRecord.Expiration < DateTimeOffset.UtcNow.Ticks; internal enum MigrateCmdParseState : byte { @@ -157,7 +157,7 @@ private bool NetworkTryMIGRATE(out bool invalidParameters) { transferOption = TransferOption.KEYS; sketch = new(); - sketch.HashAndStore(ref keySlice); + sketch.HashAndStore(keySlice); } var currTokenIdx = 5; @@ -189,13 +189,12 @@ private bool NetworkTryMIGRATE(out bool invalidParameters) while (currTokenIdx < parseState.Count) { var currKeySlice = parseState.GetArgSliceByRef(currTokenIdx++); - var sbKey = currKeySlice.SpanByte; // Skip if previous error encountered if (pstate != MigrateCmdParseState.SUCCESS) continue; // Check if all keys are local R/W because we migrate keys and need to be able to delete them - var slot = HashSlotUtils.HashSlot(sbKey.ToPointer(), sbKey.Length); + var slot = HashSlotUtils.HashSlot(currKeySlice); if (!current.IsLocal(slot, readWriteSession: false)) { pstate = MigrateCmdParseState.SLOTNOTLOCAL; @@ -217,7 +216,7 @@ private bool NetworkTryMIGRATE(out bool invalidParameters) } // Add key to sketch - sketch.HashAndStore(ref currKeySlice); + sketch.HashAndStore(currKeySlice); _ = slots.Add(slot); } } diff --git a/libs/cluster/Session/ReplicaOfCommand.cs b/libs/cluster/Session/ReplicaOfCommand.cs index d979fee2471..15c3d887b82 100644 --- a/libs/cluster/Session/ReplicaOfCommand.cs +++ b/libs/cluster/Session/ReplicaOfCommand.cs @@ -25,7 +25,7 @@ private bool NetworkTryREPLICAOF(out bool invalidParameters) var addressSpan = parseState.GetArgSliceByRef(0).ReadOnlySpan; var portSpan = parseState.GetArgSliceByRef(1).ReadOnlySpan; - // Turn off replication and make replica into a primary but do not delete data + // Turn of replication and make replica into a primary but do not delete data if (addressSpan.EqualsUpperCaseSpanIgnoringCase("NO"u8) && portSpan.EqualsUpperCaseSpanIgnoringCase("ONE"u8)) { @@ -43,9 +43,8 @@ private bool NetworkTryREPLICAOF(out bool invalidParameters) clusterProvider.clusterManager.TryResetReplica(); clusterProvider.replicationManager.TryUpdateForFailover(); - clusterProvider.replicationManager.ResetReplayIterator(); - // Cannot avoid blocking here we're on the network thread + clusterProvider.replicationManager.ResetReplicaReplayDriverStore(); AsyncUtils.BlockingWait(UnsafeBumpAndWaitForEpochTransitionAsync()); AsyncUtils.BlockingWait(clusterProvider.storeWrapper.SuspendReplicaOnlyTasksAsync()); @@ -53,7 +52,8 @@ private bool NetworkTryREPLICAOF(out bool invalidParameters) } finally { - if (acquiredLock) clusterProvider.replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); + if (acquiredLock) + clusterProvider.replicationManager.EndRecovery(RecoveryStatus.NoRecovery, downgradeLock: false); } } else diff --git a/libs/cluster/Session/RespClusterBasicCommands.cs b/libs/cluster/Session/RespClusterBasicCommands.cs index af13d8d9b0d..58a49320325 100644 --- a/libs/cluster/Session/RespClusterBasicCommands.cs +++ b/libs/cluster/Session/RespClusterBasicCommands.cs @@ -374,12 +374,10 @@ private bool NetworkClusterGossip(out bool invalidParameters) Debug.Assert(withMeetSpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHMEET)); if (withMeetSpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHMEET)) - { gossipWithMeet = true; - } } - var gossipMessage = parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var gossipMessage = parseState.GetArgSliceByRef(currTokenIdx).ToArray(); clusterProvider.clusterManager.gossipStats.UpdateGossipBytesRecv(gossipMessage.Length); var current = clusterProvider.clusterManager.CurrentConfig; @@ -387,29 +385,35 @@ private bool NetworkClusterGossip(out bool invalidParameters) // Try merge if not just a ping message if (gossipMessage.Length > 0) { - var other = ClusterConfig.FromByteArray(gossipMessage); - // Accept gossip message if it is a gossipWithMeet or node from node that is already known and trusted - // GossipWithMeet messages are only send through a call to CLUSTER MEET at the remote node - if (gossipWithMeet || current.IsKnown(other.LocalNodeId)) + // Validate config version before full deserialization + if (!ClusterConfig.TryPeekVersion(gossipMessage, out var version) || version != ClusterConfig.ClusterConfigVersion) { - // NOTE: release the epoch to avoid deadlock with MIGRATE config suspension - ReleaseCurrentEpoch(); - try - { - _ = clusterProvider.clusterManager.TryMerge(other); - } - finally - { - AcquireCurrentEpoch(); - } - - // Remember that this connection is being used for another cluster node to talk to us - Debug.Assert(RemoteNodeId is null || RemoteNodeId == other.LocalNodeId, "Node Id shouldn't change once set for a connection"); - RemoteNodeId = other.LocalNodeId; + logger?.LogWarning("Received gossip with incompatible config version: {version}", version); } else { - logger?.LogWarning("Received gossip from unknown node: {node-id}", other.LocalNodeId); + var other = ClusterConfig.FromByteArray(gossipMessage); + // Accept gossip message if it is a gossipWithMeet or node from node that is already known and trusted + // GossipWithMeet messages are only send through a call to CLUSTER MEET at the remote node + if (gossipWithMeet || current.IsKnown(other.LocalNodeId)) + { + // NOTE: release the epoch to avoid deadlock with MIGRATE config suspension + ReleaseCurrentEpoch(); + try + { + _ = clusterProvider.clusterManager.TryMerge(other); + } + finally + { + AcquireCurrentEpoch(); + } + + // Remember that this connection is being used for another cluster node to talk to us + Debug.Assert(RemoteNodeId is null || RemoteNodeId == other.LocalNodeId, "Node Id shouldn't change once set for a connection"); + RemoteNodeId = other.LocalNodeId; + } + else + logger?.LogWarning("Received gossip from unknown node: {node-id}", other.LocalNodeId); } } @@ -496,6 +500,7 @@ private bool NetworkClusterReset(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterPublish(out bool invalidParameters) { invalidParameters = false; diff --git a/libs/cluster/Session/RespClusterFailoverCommands.cs b/libs/cluster/Session/RespClusterFailoverCommands.cs index 92477feb552..c3bf214b6fe 100644 --- a/libs/cluster/Session/RespClusterFailoverCommands.cs +++ b/libs/cluster/Session/RespClusterFailoverCommands.cs @@ -101,6 +101,7 @@ private bool NetworkClusterFailover(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterFailStopWrites(out bool invalidParameters) { invalidParameters = false; @@ -125,8 +126,7 @@ private bool NetworkClusterFailStopWrites(out bool invalidParameters) // Cannot avoid blocking here we're on the network thread AsyncUtils.BlockingWait(UnsafeBumpAndWaitForEpochTransitionAsync()); - - while (!RespWriteUtils.TryWriteInt64(clusterProvider.replicationManager.ReplicationOffset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteAsciiBulkString(clusterProvider.replicationManager.ReplicationOffset.ToString(), ref dcurr, dend)) SendAndReset(); return true; } @@ -136,6 +136,7 @@ private bool NetworkClusterFailStopWrites(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterFailReplicationOffset(out bool invalidParameters) { invalidParameters = false; @@ -147,16 +148,11 @@ private bool NetworkClusterFailReplicationOffset(out bool invalidParameters) return true; } - if (!parseState.TryGetLong(0, out var primaryReplicationOffset)) - { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) - SendAndReset(); - return true; - } + var primaryReplicationOffset = AofAddress.FromByteArray(parseState.GetArgSliceByRef(0).ToArray()); // Cannot avoid blocking here we're on the network thread var rOffset = AsyncUtils.BlockingWait(clusterProvider.replicationManager.WaitForReplicationOffsetAsync(primaryReplicationOffset)); - while (!RespWriteUtils.TryWriteInt64(rOffset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteAsciiBulkString(rOffset.ToString(), ref dcurr, dend)) SendAndReset(); return true; diff --git a/libs/cluster/Session/RespClusterMigrateCommands.cs b/libs/cluster/Session/RespClusterMigrateCommands.cs index d0a9dd8464f..a5d56e6e742 100644 --- a/libs/cluster/Session/RespClusterMigrateCommands.cs +++ b/libs/cluster/Session/RespClusterMigrateCommands.cs @@ -2,9 +2,10 @@ // Licensed under the MIT license. using System; +using System.Buffers.Binary; using System.Diagnostics; -using System.Text; using System.Threading.Tasks; +using Garnet.client; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -12,16 +13,6 @@ namespace Garnet.cluster { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - internal sealed unsafe partial class ClusterSession : IClusterSession { long lastLog = 0; @@ -31,15 +22,14 @@ internal sealed unsafe partial class ClusterSession : IClusterSession /// Logging of migrate session status /// /// - /// /// - private void TrackImportProgress(int keyCount, bool isMainStore, bool completed = false) + private void TrackImportProgress(int keyCount, bool completed = false) { totalKeyCount += keyCount; var duration = TimeSpan.FromTicks(Stopwatch.GetTimestamp() - lastLog); if (completed || lastLog == 0 || duration >= clusterProvider.storeWrapper.loggingFrequency) { - logger?.LogTrace("[{op}]: isMainStore:({storeType}) totalKeyCount:({totalKeyCount})", completed ? "COMPLETED" : "IMPORTING", isMainStore, totalKeyCount.ToString("N0")); + logger?.LogTrace("[{op}]: totalKeyCount:({totalKeyCount})", completed ? "COMPLETED" : "IMPORTING", totalKeyCount.ToString("N0")); lastLog = Stopwatch.GetTimestamp(); } } @@ -62,21 +52,23 @@ private bool NetworkClusterMigrate(out bool invalidParameters) } var replace = parseState.GetArgSliceByRef(1).ReadOnlySpan; - var storeType = parseState.GetArgSliceByRef(2).ReadOnlySpan; - var payloadStartPtr = parseState.GetArgSliceByRef(3).SpanByte.ToPointer(); - var lastParam = parseState.GetArgSliceByRef(parseState.Count - 1).SpanByte; + var vectorSet = parseState.GetArgSliceByRef(2).ReadOnlySpan; + var payloadStartPtr = parseState.GetArgSliceByRef(3).ToPointer(); + var lastParam = parseState.GetArgSliceByRef(parseState.Count - 1); + var payloadEndPtr = lastParam.ToPointer() + lastParam.Length; + var replaceOption = replace.EqualsUpperCaseSpanIgnoringCase("T"u8); + var vectorSetOption = vectorSet.EqualsUpperCaseSpanIgnoringCase("T"u8); - var storeTypeStr = Encoding.ASCII.GetString(storeType); var buffer = new Span(payloadStartPtr, (int)(payloadEndPtr - payloadStartPtr)).ToArray(); if (clusterProvider.serverOptions.FastMigrate) - _ = Task.Run(() => Process(basicGarnetApi, buffer, storeTypeStr, replaceOption)); + _ = Task.Run(() => Process(basicGarnetApi, buffer, replaceOption, vectorSetOption)); else - Process(basicGarnetApi, buffer, storeTypeStr, replaceOption); + Process(basicGarnetApi, buffer, replaceOption, vectorSetOption); - void Process(BasicGarnetApi basicGarnetApi, byte[] input, string storeTypeSpan, bool replaceOption) + void Process(BasicGarnetApi basicGarnetApi, byte[] input, bool replaceOption, bool vectorSetOption) { var currentConfig = clusterProvider.clusterManager.CurrentConfig; byte migrateState = 0; @@ -85,124 +77,138 @@ void Process(BasicGarnetApi basicGarnetApi, byte[] input, string storeTypeSpan, { var payloadPtr = ptr; var payloadEndPtr = ptr + input.Length; - if (storeTypeSpan.Equals("SSTORE", StringComparison.OrdinalIgnoreCase)) - { - var keyCount = *(int*)payloadPtr; - payloadPtr += 4; - var i = 0; - TrackImportProgress(keyCount, isMainStore: true, keyCount == 0); - while (i < keyCount) - { - ref var key = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += key.TotalSize; - ref var value = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += value.TotalSize; + var keyCount = *(int*)payloadPtr; + payloadPtr += sizeof(int); + var i = 0; - // An error has occurred - if (migrateState > 0) - { - i++; - continue; - } + TrackImportProgress(keyCount, keyCount == 0); + var storeWrapper = clusterProvider.storeWrapper; + var transientObjectIdMap = storeWrapper.store.Log.TransientObjectIdMap; - // TODO: better way to handle namespaces - if (key.MetadataSize == 1) + // Use try/finally instead of "using" because we don't want the boxing that an interface call would entail. Double-Dispose() is OK for DiskLogRecord. + DiskLogRecord diskLogRecord = default; + try + { + if (vectorSetOption) + { + // Vector Sets need special handling + while (i < keyCount) { - // This is a Vector Set namespace key being migrated - it won't necessarily look like it's "in" a hash slot - // because it's dependent on some other key (the index key) being migrated which itself is in a moving hash slot + var kind = (MigrationRecordSpanType)(*payloadPtr); + payloadPtr++; - clusterProvider.storeWrapper.DefaultDatabase.VectorManager.HandleMigratedElementKey(ref basicContext, ref vectorContext, ref key, ref value); - } - else - { - var slot = HashSlotUtils.HashSlot(ref key); - if (!currentConfig.IsImportingSlot(slot)) // Slot is not in importing state + if (!RespReadUtils.GetSerializedRecordSpan(out var payloadRaw, ref payloadPtr, payloadEndPtr)) + return; + + if (kind != MigrationRecordSpanType.VectorSetIndex) + { + throw new InvalidOperationException($"Unexpected {nameof(MigrationRecordSpanType)}: {kind}"); + } + + var payload = payloadRaw.ReadOnlySpan; + + // Vector Set indexes are Key + Value + var keyLen = BinaryPrimitives.ReadInt32LittleEndian(payload); + var keyBytes = payload.Slice(sizeof(int), keyLen); + var valueLen = BinaryPrimitives.ReadInt32LittleEndian(payload[(sizeof(int) + keyBytes.Length)..]); + var valueBytes = payload.Slice(sizeof(int) + keyBytes.Length + sizeof(int), valueLen); + + // An error has occurred + if (migrateState > 0) { - migrateState = 1; i++; continue; } - // Set if key replace flag is set or key does not exist - var keySlice = new ArgSlice(key.ToPointer(), key.Length); - if (replaceOption || !Exists(ref keySlice)) - _ = basicGarnetApi.SET(ref key, ref value); + clusterProvider.storeWrapper.DefaultDatabase.VectorManager.HandleMigratedIndexKey(clusterProvider.storeWrapper.DefaultDatabase, clusterProvider.storeWrapper, keyBytes, valueBytes); + i++; } - - i++; } - } - else if (storeTypeSpan.Equals("OSTORE", StringComparison.OrdinalIgnoreCase)) - { - var keyCount = *(int*)payloadPtr; - payloadPtr += 4; - var i = 0; - TrackImportProgress(keyCount, isMainStore: false, keyCount == 0); - while (i < keyCount) + else { - if (!RespReadUtils.TryReadSerializedData(out var key, out var data, out var expiration, ref payloadPtr, payloadEndPtr)) - return; + while (i < keyCount) + { + var kind = (MigrationRecordSpanType)(*payloadPtr); + payloadPtr++; - // An error has occurred - if (migrateState > 0) - continue; + if (!RespReadUtils.GetSerializedRecordSpan(out var payloadRaw, ref payloadPtr, payloadEndPtr)) + return; - var slot = HashSlotUtils.HashSlot(key); - if (!currentConfig.IsImportingSlot(slot)) // Slot is not in importing state - { - migrateState = 1; - continue; - } + if (kind == MigrationRecordSpanType.VectorSetElement) + { + // This is a Vector Set namespace key being migrated - it won't necessarily look like it's "in" a hash slot + // because it's dependent on some other key (the index key) being migrated which itself is in a moving hash slot - var value = clusterProvider.storeWrapper.GarnetObjectSerializer.Deserialize(data); - value.Expiration = expiration; + // Vector Set elements are Namespace + Key + Value - // Set if key replace flag is set or key does not exist - if (replaceOption || !CheckIfKeyExists(key)) - _ = basicGarnetApi.SET(key, value); + var payload = payloadRaw.ReadOnlySpan; - i++; - } - } - else if (storeTypeSpan.Equals("VSTORE", StringComparison.OrdinalIgnoreCase)) - { - // This is the subset of the main store that holds Vector Set _index_ keys - // - // Namespace'd element keys are handled by the SSTORE path + var namespaceLen = BinaryPrimitives.ReadInt32LittleEndian(payload); + var namespaceBytes = payload.Slice(sizeof(int), namespaceLen); + var keyLen = BinaryPrimitives.ReadInt32LittleEndian(payload[(sizeof(int) + namespaceBytes.Length)..]); + var keyBytes = payload.Slice(sizeof(int) + namespaceLen + sizeof(int), keyLen); + var valueLen = BinaryPrimitives.ReadInt32LittleEndian(payload[(sizeof(int) + namespaceBytes.Length + sizeof(int) + keyBytes.Length)..]); + var valueBytes = payload.Slice(sizeof(int) + namespaceLen + sizeof(int) + keyBytes.Length + sizeof(int), valueLen); - var keyCount = *(int*)payloadPtr; - payloadPtr += 4; - var i = 0; + // An error has occurred + if (migrateState > 0) + { + i++; + continue; + } - TrackImportProgress(keyCount, isMainStore: true, keyCount == 0); - while (i < keyCount) - { - ref var key = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += key.TotalSize; - ref var value = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += value.TotalSize; + clusterProvider.storeWrapper.DefaultDatabase.VectorManager.HandleMigratedElementKey(ref stringBasicContext, ref vectorBasicContext, namespaceBytes, keyBytes, valueBytes); + } + else if (kind == MigrationRecordSpanType.LogRecord) + { + // An error has occurred + if (migrateState > 0) + { + i++; + continue; + } + + diskLogRecord = DiskLogRecord.Deserialize(payloadRaw, storeWrapper.GarnetObjectSerializer, + transientObjectIdMap, storeWrapper.storeFunctions); + + var slot = HashSlotUtils.HashSlot(diskLogRecord.Key); + if (!currentConfig.IsImportingSlot(slot)) // Slot is not in importing state + { + migrateState = 1; + i++; + continue; + } + + // Set if key replace flag is set or key does not exist + var keySlice = PinnedSpanByte.FromPinnedSpan(diskLogRecord.Key); + if (replaceOption || !Exists(keySlice)) + _ = basicGarnetApi.SET(in diskLogRecord); + + storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); + diskLogRecord = default; // prevent double-trigger in finally + } + else + { + throw new InvalidOperationException($"Unexpected {nameof(MigrationRecordSpanType)}: {kind}"); + } - // An error has occurred - if (migrateState > 0) - { i++; - continue; } - - clusterProvider.storeWrapper.DefaultDatabase.VectorManager.HandleMigratedIndexKey(clusterProvider.storeWrapper.DefaultDatabase, clusterProvider.storeWrapper, ref key, ref value); - i++; } } - else + finally { - throw new Exception("CLUSTER MIGRATE STORE TYPE ERROR!"); + if (diskLogRecord.IsSet) + { + storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); + } } } } - var currentConfig = clusterProvider.clusterManager.CurrentConfig; - while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); diff --git a/libs/cluster/Session/RespClusterReplicationCommands.cs b/libs/cluster/Session/RespClusterReplicationCommands.cs index 2d5153753f4..2bb4f4ba59f 100644 --- a/libs/cluster/Session/RespClusterReplicationCommands.cs +++ b/libs/cluster/Session/RespClusterReplicationCommands.cs @@ -2,7 +2,9 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; using System.Text; +using Garnet.client; using Garnet.cluster.Server.Replication; using Garnet.common; using Garnet.server; @@ -153,7 +155,7 @@ private bool NetworkClusterReserve(VectorManager vectorManager, out bool invalid invalidParameters = false; - if (!vectorManager.TryReserveContextsForMigration(ref vectorContext, numVectorSetContexts, out var newContexts)) + if (!vectorManager.TryReserveContextsForMigration(ref vectorBasicContext, numVectorSetContexts, out var newContexts)) { while (!RespWriteUtils.TryWriteError("Insufficients contexts available to reserve"u8, ref dcurr, dend)) SendAndReset(); @@ -173,64 +175,18 @@ private bool NetworkClusterReserve(VectorManager vectorManager, out bool invalid return true; } - /// - /// Implements CLUSTER aofsync command (only for internode use) - /// - /// - /// - private bool NetworkClusterAOFSync(out bool invalidParameters) - { - invalidParameters = false; - - if (parseState.Count != 2) - { - invalidParameters = true; - return true; - } - - var nodeId = parseState.GetString(0); - - if (!parseState.TryGetLong(1, out var nextAddress)) - { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) - SendAndReset(); - return true; - } - - if (clusterProvider.serverOptions.EnableAOF) - { - clusterProvider.replicationManager.TryAddReplicationTask(nodeId, nextAddress, out var aofSyncTaskInfo); - if (!clusterProvider.replicationManager.TryConnectToReplica(nodeId, nextAddress, aofSyncTaskInfo, out var errorMessage)) - { - while (!RespWriteUtils.TryWriteError(errorMessage, ref dcurr, dend)) - SendAndReset(); - } - else - { - while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) - SendAndReset(); - } - } - else - { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_REPLICATION_AOF_TURNEDOFF, ref dcurr, dend)) - SendAndReset(); - } - - return true; - } - /// /// Implements CLUSTER appendlog command (only for internode use) /// /// /// + /// private bool NetworkClusterAppendLog(out bool invalidParameters) { invalidParameters = false; - // Expecting exactly 5 arguments (5-th argument is AOF page parsed later) - if (parseState.Count != 5) + // Expecting exactly 6 arguments (6-th argument is AOF page parsed later) + if (parseState.Count > 6 || parseState.Count < 5) { invalidParameters = true; return true; @@ -238,15 +194,16 @@ private bool NetworkClusterAppendLog(out bool invalidParameters) var nodeId = parseState.GetString(0); - if (!parseState.TryGetLong(1, out var previousAddress) || - !parseState.TryGetLong(2, out var currentAddress) || - !parseState.TryGetLong(3, out var nextAddress)) + if (!parseState.TryGetInt(1, out var physicalSublogIdx) || + !parseState.TryGetLong(2, allowLeadingZeros: true, out var previousAddress) || + !parseState.TryGetLong(3, allowLeadingZeros: true, out var currentAddress) || + !parseState.TryGetLong(4, allowLeadingZeros: true, out var nextAddress)) { logger?.LogError("{str}", Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER)); return true; } - var sbRecord = parseState.GetArgSliceByRef(4).SpanByte; + LogPrimaryStream(physicalSublogIdx, previousAddress, currentAddress, nextAddress, logger); var currentConfig = clusterProvider.clusterManager.CurrentConfig; var localRole = currentConfig.LocalNodeRole; @@ -261,13 +218,42 @@ private bool NetworkClusterAppendLog(out bool invalidParameters) } else { + // Mark this session as the active replication stream so that + // EnsureReplication does not trigger spurious resyncs while the + // AOF stream is idle (no data APPENDLOG to set the flag later). IsReplicating = true; - clusterProvider.replicationManager.ProcessPrimaryStream(sbRecord.ToPointer(), sbRecord.Length, + // This is an initialization message + if (previousAddress == -1 && currentAddress == -1 && nextAddress == -1) + { + if (clusterProvider.replicationManager.InitializeReplicaReplayDriver(physicalSublogIdx, networkSender)) + replicaReplayDriverStore = clusterProvider.replicationManager.ReplicaReplayDriverStore; + else + throw new GarnetException($"Failed to process {nameof(NetworkClusterAppendLog)}: [physicalSublogIdx: {physicalSublogIdx}] Received initialization message but ReplicaReplayDriver is already initialized!", LogLevel.Error, clientResponse: false); + + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var sbRecord = parseState.GetArgSliceByRef(5); + ProcessPrimaryStream(physicalSublogIdx, sbRecord.ToPointer(), sbRecord.Length, previousAddress, currentAddress, nextAddress); } return true; + + [Conditional("DEBUG")] + static void LogPrimaryStream(int physicalSublogIdx, long previousAddress, long currentAddress, long nextAddress, ILogger logger) + { + var state = new GarnetTestLoggingEvent() + { + Type = GarnetTestLoggingEventType.LogPrimaryStreamType, + Message = $"physicalSublogIdx: {physicalSublogIdx}, previousAddress: {previousAddress}, currentAddress: {currentAddress}, nextAddress: {nextAddress}", + }; + + logger?.LogTesting(state); + } } /// @@ -275,6 +261,7 @@ private bool NetworkClusterAppendLog(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterInitiateReplicaSync(out bool invalidParameters) { invalidParameters = false; @@ -288,19 +275,14 @@ private bool NetworkClusterInitiateReplicaSync(out bool invalidParameters) var replicaNodeId = parseState.GetString(0); var replicaAssignedPrimaryId = parseState.GetString(1); - var checkpointEntryBytes = parseState.GetArgSliceByRef(2).SpanByte.ToByteArray(); + var checkpointEntryBytes = parseState.GetArgSliceByRef(2).ToArray(); - if (!parseState.TryGetLong(3, out var replicaAofBeginAddress) || - !parseState.TryGetLong(4, out var replicaAofTailAddress)) - { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) - SendAndReset(); - return true; - } + var replicaAofBeginAddress = AofAddress.FromSpan(parseState.GetArgSliceByRef(3).Span); + var replicaAofTailAddress = AofAddress.FromSpan(parseState.GetArgSliceByRef(4).Span); var replicaCheckpointEntry = CheckpointEntry.FromByteArray(checkpointEntryBytes); - var beginPrimarySyncTask = clusterProvider.replicationManager.TryBeginPrimarySyncAsync(replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, replicaAofBeginAddress, replicaAofTailAddress); + var beginPrimarySyncTask = clusterProvider.replicationManager.TryBeginDiskbasedSyncAsync(replicaNodeId, replicaAssignedPrimaryId, replicaCheckpointEntry, replicaAofBeginAddress, replicaAofTailAddress); // No choice but to block here, we're on the network thread var (success, errorMessage) = AsyncUtils.BlockingWait(beginPrimarySyncTask); @@ -324,6 +306,7 @@ private bool NetworkClusterInitiateReplicaSync(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterSendCheckpointMetadata(out bool invalidParameters) { invalidParameters = false; @@ -344,11 +327,11 @@ private bool NetworkClusterSendCheckpointMetadata(out bool invalidParameters) return true; } - var checkpointMetadata = parseState.GetArgSliceByRef(2).SpanByte.ToByteArray(); + var checkpointMetadata = parseState.GetArgSliceByRef(2).ReadOnlySpan; var fileToken = new Guid(fileTokenBytes); var fileType = (CheckpointFileType)fileTypeInt; - clusterProvider.replicationManager.ProcessCheckpointMetadata(fileToken, fileType, checkpointMetadata); + clusterProvider.replicationManager.recvCheckpointHandler.ProcessMetadata(fileToken, fileType, checkpointMetadata); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -360,6 +343,7 @@ private bool NetworkClusterSendCheckpointMetadata(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterSendCheckpointFileSegment(out bool invalidParameters) { invalidParameters = false; @@ -383,12 +367,16 @@ private bool NetworkClusterSendCheckpointFileSegment(out bool invalidParameters) var data = parseState.GetArgSliceByRef(3).ReadOnlySpan; + // segmentId is validated for backward compatibility but not used; + // disk-based replication now uses the SNAPSHOT_DATA command path instead. + _ = segmentId; + var fileToken = new Guid(fileTokenBytes); var ckptFileType = (CheckpointFileType)ckptFileTypeInt; // Commenting due to high verbosity // logger?.LogTrace("send_ckpt_file_segment {fileToken} {ckptFileType} {startAddress} {dataLength}", fileToken, ckptFileType, startAddress, data.Length); - clusterProvider.replicationManager.recvCheckpointHandler.ProcessFileSegments(segmentId, fileToken, ckptFileType, startAddress, data); + clusterProvider.replicationManager.recvCheckpointHandler.ProcessFileSegment(fileToken, ckptFileType, startAddress, data); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -396,55 +384,88 @@ private bool NetworkClusterSendCheckpointFileSegment(out bool invalidParameters) } /// - /// Implements CLUSTER begin_replica_recover (only for internode use) + /// Implements CLUSTER SNAPSHOT_DATA command (only for internode use). + /// Unified command for receiving both file segments and metadata from a primary. + /// API: CLUSTER SNAPSHOT_DATA token type startAddress data /// /// /// - private bool NetworkClusterBeginReplicaRecover(out bool invalidParameters) + /// + private bool NetworkClusterSnapshotData(out bool invalidParameters) { invalidParameters = false; - // Expecting exactly 7 arguments - if (parseState.Count != 7) + if (parseState.Count != 4) { invalidParameters = true; return true; } - if (!parseState.TryGetBool(0, out var recoverMainStoreFromToken) || - !parseState.TryGetBool(1, out var recoverObjectStoreFromToken) || - !parseState.TryGetBool(2, out var replayAOF)) + var fileTokenBytes = parseState.GetArgSliceByRef(0).ReadOnlySpan; + + if (!parseState.TryGetInt(1, out var ckptFileTypeInt) || + !parseState.TryGetLong(2, out var startAddress)) { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_BOOLEAN, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) SendAndReset(); return true; } - var primaryReplicaId = parseState.GetString(3); - var checkpointEntryBytes = parseState.GetArgSliceByRef(4).SpanByte.ToByteArray(); + var data = parseState.GetArgSliceByRef(3).ReadOnlySpan; + + var fileToken = new Guid(fileTokenBytes); + var ckptFileType = (CheckpointFileType)ckptFileTypeInt; + + clusterProvider.replicationManager.recvCheckpointHandler.ProcessSnapshotData(fileToken, ckptFileType, startAddress, data); + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + + return true; + } + + /// + /// Implements CLUSTER begin_replica_recover (only for internode use) + /// + /// + /// + /// + private bool NetworkClusterBeginReplicaRecover(out bool invalidParameters) + { + invalidParameters = false; - if (!parseState.TryGetLong(5, out var beginAddress) || - !parseState.TryGetLong(6, out var tailAddress)) + // Expecting exactly 6 arguments + if (parseState.Count != 6) { - while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) + invalidParameters = true; + return true; + } + + if (!parseState.TryGetBool(0, out var recoverStoreFromToken) || !parseState.TryGetLong(1, out var replayAOFMap)) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_BOOLEAN, ref dcurr, dend)) SendAndReset(); return true; } + var primaryReplicaId = parseState.GetString(2); + var checkpointEntryBytes = parseState.GetArgSliceByRef(3).ToArray(); + + var beginAddress = AofAddress.FromSpan(parseState.GetArgSliceByRef(4).Span); + var tailAddress = AofAddress.FromSpan(parseState.GetArgSliceByRef(5).Span); + var entry = CheckpointEntry.FromByteArray(checkpointEntryBytes); - var replicationOffset = clusterProvider.replicationManager.BeginReplicaRecover( - recoverMainStoreFromToken, - recoverObjectStoreFromToken, - replayAOF, + var replicationOffset = clusterProvider.replicationManager.TryReplicaDiskbasedRecovery( + recoverStoreFromToken, + (ulong)replayAOFMap, primaryReplicaId, entry, beginAddress, - tailAddress, + ref tailAddress, out var errorMessage); if (errorMessage.IsEmpty) { - while (!RespWriteUtils.TryWriteInt64(replicationOffset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteAsciiBulkString(replicationOffset.ToString(), ref dcurr, dend)) SendAndReset(); } else @@ -461,6 +482,7 @@ private bool NetworkClusterBeginReplicaRecover(out bool invalidParameters) /// /// /// + /// private bool NetworkClusterAttachSync(out bool invalidParameters) { invalidParameters = false; @@ -472,30 +494,30 @@ private bool NetworkClusterAttachSync(out bool invalidParameters) return true; } - var checkpointEntryBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var checkpointEntryBytes = parseState.GetArgSliceByRef(0).ToArray(); var syncMetadata = SyncMetadata.FromByteArray(checkpointEntryBytes); - ReadOnlySpan errorMessage = default; - long replicationOffset = -1; + ReadOnlySpan errorMessage; + var replicationOffset = AofAddress.Create(clusterProvider.serverOptions.AofPhysicalSublogCount, -1); if (syncMetadata.originNodeRole == NodeRole.REPLICA) { - var attachTask = clusterProvider.replicationManager.TryAttachSyncAsync(syncMetadata); + var attachTask = clusterProvider.replicationManager.TryBeginDisklessSyncAsync(syncMetadata); // Must block here because we're on the network thread var (_, err) = AsyncUtils.BlockingWait(attachTask); errorMessage = err.Span; } else - replicationOffset = clusterProvider.replicationManager.ReplicaRecoverDiskless(syncMetadata, out errorMessage); + replicationOffset = clusterProvider.replicationManager.TryReplicaDisklessRecovery(syncMetadata, out errorMessage); - if (!errorMessage.IsEmpty) + if (errorMessage.IsEmpty) { - while (!RespWriteUtils.TryWriteError(errorMessage, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteAsciiBulkString(replicationOffset.ToString(), ref dcurr, dend)) SendAndReset(); } else { - while (!RespWriteUtils.TryWriteInt64(replicationOffset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteError(errorMessage, ref dcurr, dend)) SendAndReset(); } @@ -507,54 +529,70 @@ private bool NetworkClusterAttachSync(out bool invalidParameters) /// /// /// + /// + /// + /// private bool NetworkClusterSync(out bool invalidParameters) { invalidParameters = false; - // Expecting exactly 3 arguments - if (parseState.Count != 3) + // Expecting exactly 2 arguments + if (parseState.Count != 2) { invalidParameters = true; return true; } - var primaryNodeId = parseState.GetString(0); - var storeTypeSpan = parseState.GetArgSliceByRef(1).ReadOnlySpan; - var payload = parseState.GetArgSliceByRef(2).SpanByte; + var payload = parseState.GetArgSliceByRef(1); var payloadPtr = payload.ToPointer(); - var lastParam = parseState.GetArgSliceByRef(parseState.Count - 1).SpanByte; + var lastParam = parseState.GetArgSliceByRef(parseState.Count - 1); var payloadEndPtr = lastParam.ToPointer() + lastParam.Length; - var keyValuePairCount = *(int*)payloadPtr; + var recordCount = *(int*)payloadPtr; var i = 0; payloadPtr += 4; - if (storeTypeSpan.EqualsUpperCaseSpanIgnoringCase("SSTORE"u8)) + + TrackImportProgress(recordCount, recordCount == 0); + var storeWrapper = clusterProvider.storeWrapper; + var transientObjectIdMap = storeWrapper.store.Log.TransientObjectIdMap; + + DiskLogRecord diskLogRecord = default; + try { - TrackImportProgress(keyValuePairCount, isMainStore: true, keyValuePairCount == 0); - while (i < keyValuePairCount) + while (i < recordCount) { - ref var key = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += key.TotalSize; - ref var value = ref SpanByte.Reinterpret(payloadPtr); - payloadPtr += value.TotalSize; + var kind = (MigrationRecordSpanType)(*payloadPtr); + payloadPtr++; + + if (kind == MigrationRecordSpanType.LogRecord) + { + + if (!RespReadUtils.GetSerializedRecordSpan(out var recordSpan, ref payloadPtr, payloadEndPtr)) + return false; + + diskLogRecord = DiskLogRecord.Deserialize(recordSpan, storeWrapper.GarnetObjectSerializer, transientObjectIdMap, storeWrapper.storeFunctions); + _ = basicGarnetApi.SET(in diskLogRecord); + storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); + diskLogRecord = default; // prevent double-trigger in catch + } + else + { + throw new InvalidOperationException($"Unexpected {nameof(MigrationRecordSpanType)}: {kind}"); + } - _ = basicGarnetApi.SET(ref key, ref value); i++; } } - else if (storeTypeSpan.EqualsUpperCaseSpanIgnoringCase("OSTORE"u8)) + catch { - TrackImportProgress(keyValuePairCount, isMainStore: false, keyValuePairCount == 0); - while (i < keyValuePairCount) + // Dispose the diskLogRecord if there was an exception in SET + if (diskLogRecord.IsSet) { - if (!RespReadUtils.TryReadSerializedData(out var key, out var data, out var expiration, ref payloadPtr, payloadEndPtr)) - return false; - - var value = clusterProvider.storeWrapper.GarnetObjectSerializer.Deserialize(data); - value.Expiration = expiration; - _ = basicGarnetApi.SET(key, value); - i++; + storeWrapper.storeFunctions.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); } + throw; } while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) @@ -567,6 +605,7 @@ private bool NetworkClusterSync(out bool invalidParameters) /// Implements CLUSTER FLUSHALL /// /// + /// private bool NetworkClusterFlushAll(out bool invalidParameters) { invalidParameters = false; @@ -585,5 +624,79 @@ private bool NetworkClusterFlushAll(out bool invalidParameters) SendAndReset(); return true; } + + /// + /// Implements CLUSTER_ADVANCE_TIME + /// + /// + /// + /// + private bool NetworkClusterAdvanceTime(out bool invalidParameters) + { + invalidParameters = false; + + // Expecting exactly 2 + if (parseState.Count != 2) + { + invalidParameters = true; + return true; + } + + var sequenceNumber = parseState.GetLong(0); + var tailAddressSpan = parseState.GetArgSliceByRef(1).Span; + clusterProvider.replicationManager.SignalAdvanceTime(sequenceNumber, AofAddress.FromSpan(tailAddressSpan)); + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + return true; + } + + /// + /// Implements CLUSTER MLOG_KEY_TIME command. + /// If node is replica, it returns the sequence number associated with the provided key otherwise the latest sequence number as generated by the sequence number generator. + /// For nodes configured with single log it returns an error. + /// + /// + /// + private bool NetworkClusterMlogKeyTime(out bool invalidParameters) + { + invalidParameters = false; + + // Expecting 1 or 2 arguments (key and optional FRONTIER) + if (parseState.Count < 1 || parseState.Count > 2) + { + invalidParameters = true; + return true; + } + + // Check if multi-log is enabled + if (!clusterProvider.serverOptions.MultiLogEnabled) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_MULTI_LOG_DISABLED, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var sbKey = parseState.GetArgSliceByRef(0).Span; + + long sequenceNumber; + // Return sequence number for the specific key + // Check if this node is a replica + if (!clusterProvider.clusterManager.CurrentConfig.IsPrimary) + { + var getFrontier = parseState.Count == 2 ? parseState.GetBool(1) : false; + // Get sequence number from the replay state + var sn = clusterProvider.storeWrapper.appendOnlyFile.readConsistencyManager?.GetKeySequenceNumber(sbKey, getFrontier); + sequenceNumber = sn.GetValueOrDefault(-1); + } + else + { + // Get sequence number from the primary's sequence number generator + sequenceNumber = clusterProvider.storeWrapper.appendOnlyFile.seqNumGen.GetSequenceNumber(); + } + while (!RespWriteUtils.TryWriteInt64(sequenceNumber, ref dcurr, dend)) + SendAndReset(); + + return true; + } } } \ No newline at end of file diff --git a/libs/cluster/Session/RespClusterSlotManagementCommands.cs b/libs/cluster/Session/RespClusterSlotManagementCommands.cs index afbe57fa217..bc6383e9bce 100644 --- a/libs/cluster/Session/RespClusterSlotManagementCommands.cs +++ b/libs/cluster/Session/RespClusterSlotManagementCommands.cs @@ -288,9 +288,7 @@ private bool NetworkClusterDelKeysInSlot(out bool invalidParameters) } var slots = new HashSet { slot }; - ClusterManager.DeleteKeysInSlotsFromMainStore(basicGarnetApi, slots); - if (!clusterProvider.serverOptions.DisableObjects) - ClusterManager.DeleteKeysInSlotsFromObjectStore(basicGarnetApi, slots); + ClusterManager.DeleteKeysInSlots(basicGarnetApi, slots); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -325,9 +323,7 @@ private bool NetworkClusterDelKeysInSlotRange(out bool invalidParameters) return true; } - ClusterManager.DeleteKeysInSlotsFromMainStore(basicGarnetApi, slots); - if (!clusterProvider.serverOptions.DisableObjects) - ClusterManager.DeleteKeysInSlotsFromObjectStore(basicGarnetApi, slots); + ClusterManager.DeleteKeysInSlots(basicGarnetApi, slots); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -408,7 +404,7 @@ private bool NetworkClusterKeySlot(out bool invalidParameters) return true; } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var sbKey = parseState.GetArgSliceByRef(0); var keyPtr = sbKey.ToPointer(); var keySize = sbKey.Length; diff --git a/libs/cluster/Session/SlotVerification/ClusterSlotVerify.cs b/libs/cluster/Session/SlotVerification/ClusterSlotVerify.cs index 73b274fb168..b4401764c74 100644 --- a/libs/cluster/Session/SlotVerification/ClusterSlotVerify.cs +++ b/libs/cluster/Session/SlotVerification/ClusterSlotVerify.cs @@ -1,41 +1,31 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; +using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet.cluster { internal sealed unsafe partial class ClusterSession : IClusterSession { [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool Exists(ref ArgSlice keySlice) - => basicGarnetApi.EXISTS(keySlice, StoreType.All) == GarnetStatus.OK; + private bool Exists(PinnedSpanByte keySlice) + => basicGarnetApi.EXISTS(keySlice) == GarnetStatus.OK; - private bool CheckIfKeyExists(byte[] key) - { - fixed (byte* keyPtr = key) - { - var keySlice = new ArgSlice(keyPtr, key.Length); - return Exists(ref keySlice); - } - } - - private ClusterSlotVerificationResult SingleKeySlotVerify(ref ClusterConfig config, ref ArgSlice keySlice, bool readOnly, byte SessionAsking, bool waitForStableSlot, int slot = -1) + private ClusterSlotVerificationResult SingleKeySlotVerify(ref ClusterConfig config, ref PinnedSpanByte keySlice, bool readOnly, bool SessionAsking, bool waitForStableSlot, int slot = -1) { Debug.Assert(!waitForStableSlot || (waitForStableSlot && !readOnly), "Shouldn't see Vector Set writes and readonly at same time"); - var ret = readOnly ? SingleKeyReadSlotVerify(ref config, ref keySlice) : SingleKeyReadWriteSlotVerify(waitForStableSlot, ref config, ref keySlice); - - return ret; + return readOnly ? SingleKeyReadSlotVerify(ref config, ref keySlice) : SingleKeyReadWriteSlotVerify(waitForStableSlot, ref config, ref keySlice); [MethodImpl(MethodImplOptions.AggressiveInlining)] - ClusterSlotVerificationResult SingleKeyReadSlotVerify(ref ClusterConfig config, ref ArgSlice keySlice) + ClusterSlotVerificationResult SingleKeyReadSlotVerify(ref ClusterConfig config, ref PinnedSpanByte keySlice) { - var _slot = slot == -1 ? ArgSliceUtils.HashSlot(ref keySlice) : (ushort)slot; + var _slot = slot == -1 ? HashSlotUtils.HashSlot(keySlice) : (ushort)slot; var IsLocal = config.IsLocal(_slot); var state = config.GetState(_slot); @@ -67,16 +57,16 @@ ClusterSlotVerificationResult SingleKeyReadSlotVerify(ref ClusterConfig config, return state switch { SlotState.STABLE => new(SlotVerifiedState.MOVED, _slot), // If local slot in stable state and not local redirect to primary - SlotState.IMPORTING => SessionAsking > 0 ? new(SlotVerifiedState.OK, _slot) : new(SlotVerifiedState.MOVED, _slot), // If it is in importing state serve request only if asking flag is set else redirect + SlotState.IMPORTING => SessionAsking ? new(SlotVerifiedState.OK, _slot) : new(SlotVerifiedState.MOVED, _slot), // If it is in importing state serve request only if asking flag is set else redirect _ => new(SlotVerifiedState.CLUSTERDOWN, _slot) // If not local and any other state respond with CLUSTERDOWN }; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - ClusterSlotVerificationResult SingleKeyReadWriteSlotVerify(bool waitForStableSlot, ref ClusterConfig config, ref ArgSlice keySlice) + ClusterSlotVerificationResult SingleKeyReadWriteSlotVerify(bool waitForStableSlot, ref ClusterConfig config, ref PinnedSpanByte keySlice) { - var _slot = slot == -1 ? ArgSliceUtils.HashSlot(ref keySlice) : (ushort)slot; + var _slot = slot == -1 ? HashSlotUtils.HashSlot(keySlice) : (ushort)slot; tryAgain: var IsLocal = config.IsLocal(_slot, readWriteSession: readWriteSession); @@ -84,7 +74,7 @@ ClusterSlotVerificationResult SingleKeyReadWriteSlotVerify(bool waitForStableSlo if (waitForStableSlot && state is SlotState.IMPORTING or SlotState.MIGRATING) { - WaitForSlotToStabalize(_slot, ref keySlice, ref config); + WaitForSlotToStabalize(_slot, keySlice, ref config); goto tryAgain; } @@ -117,27 +107,28 @@ ClusterSlotVerificationResult SingleKeyReadWriteSlotVerify(bool waitForStableSlo return state switch { SlotState.STABLE => new(SlotVerifiedState.MOVED, _slot), // If local slot in stable state and not local redirect to primary - SlotState.IMPORTING => SessionAsking > 0 ? new(SlotVerifiedState.OK, _slot) : new(SlotVerifiedState.MOVED, _slot), // If it is in importing state serve request only if asking flag is set else redirect + SlotState.IMPORTING => SessionAsking ? new(SlotVerifiedState.OK, _slot) : new(SlotVerifiedState.MOVED, _slot), // If it is in importing state serve request only if asking flag is set else redirect _ => new(SlotVerifiedState.CLUSTERDOWN, _slot) // If not local and any other state respond with CLUSTERDOWN }; } } - bool CanOperateOnKey(ref ArgSlice key, int slot, bool readOnly) + bool CanOperateOnKey(ref PinnedSpanByte key, int slot, bool readOnly) { // For both read and read/write ops we need to ensure that key will not be removed // while we try to operate on it so we will delay the corresponding operation // as long as the key is being actively migrated - while (!clusterProvider.migrationManager.CanAccessKey(ref key, slot, readOnly)) + while (!clusterProvider.migrationManager.CanAccessKey(key, slot, readOnly)) { ReleaseCurrentEpoch(); Thread.Yield(); AcquireCurrentEpoch(); } - return Exists(ref key); + return Exists(key); } - void WaitForSlotToStabalize(ushort slot, ref ArgSlice keySlice, ref ClusterConfig config) + + void WaitForSlotToStabalize(ushort slot, PinnedSpanByte keySlice, ref ClusterConfig config) { // For Vector Set ops specifically, we need a slot to be stable (or faulted, but not migrating) before writes can proceed // @@ -155,54 +146,61 @@ void WaitForSlotToStabalize(ushort slot, ref ArgSlice keySlice, ref ClusterConfi } } - ClusterSlotVerificationResult MultiKeySlotVerify(ClusterConfig config, ref Span keys, bool readOnly, byte sessionAsking, bool waitForStableSlot, int count) + ClusterSlotVerificationResult MultiKeySlotVerify(ClusterConfig config, ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, bool isTxn, bool waitForStableSlot) { - var _end = count < 0 ? keys.Length : count; - var slot = ArgSliceUtils.HashSlot(ref keys[0]); - var verifyResult = SingleKeySlotVerify(ref config, ref keys[0], readOnly, sessionAsking, waitForStableSlot, slot); - - for (var i = 1; i < _end; i++) + // Find the first valid key and initialize slot/result + var specIndex = 0; + // If slot verification is called from transaction manager, parse state contains consecutive keys so we can skip key search + (int firstIdx, int lastIdx, int step) searchArgs = isTxn ? (0, parseState.Count - 1, 1) : default; + while (specIndex < csvi.keySpecs?.Length && + !parseState.TryGetKeySearchArgsFromSimpleKeySpec(csvi.keySpecs[specIndex], csvi.isSubCommand, out searchArgs)) + specIndex++; + + if (specIndex == csvi.keySpecs?.Length && !isTxn) + return default; + + ref var firstKey = ref parseState.GetArgSliceByRef(searchArgs.firstIdx); + var firstSlot = HashSlotUtils.HashSlot(firstKey); + var firstSlotVerifyResult = SingleKeySlotVerify(ref config, ref firstKey, csvi.readOnly, csvi.sessionAsking > 0, waitForStableSlot, firstSlot); + + // Verify remaining keys from the first spec (starting from second key) + var verifyResult = VerifyKeysInRange(ref config, ref parseState, ref csvi, searchArgs.firstIdx + searchArgs.step, + searchArgs.lastIdx, searchArgs.step, firstSlot, waitForStableSlot, ref firstSlotVerifyResult); + if (verifyResult.state != SlotVerifiedState.OK) + return verifyResult; + + // Verify keys from remaining specs + for (specIndex++; specIndex < csvi.keySpecs?.Length; specIndex++) { - var _slot = ArgSliceUtils.HashSlot(ref keys[i]); - var _verifyResult = SingleKeySlotVerify(ref config, ref keys[i], readOnly, sessionAsking, waitForStableSlot, _slot); - - // Check if slot changes between keys - if (_slot != slot) - return new(SlotVerifiedState.CROSSSLOT, slot); + if (!parseState.TryGetKeySearchArgsFromSimpleKeySpec(csvi.keySpecs[specIndex], csvi.isSubCommand, out searchArgs)) + continue; - // Check if state of key changes - if (_verifyResult.state != verifyResult.state) - return new(SlotVerifiedState.TRYAGAIN, slot); + verifyResult = VerifyKeysInRange(ref config, ref parseState, ref csvi, searchArgs.firstIdx, + searchArgs.lastIdx, searchArgs.step, firstSlot, waitForStableSlot, ref firstSlotVerifyResult); + if (verifyResult.state != SlotVerifiedState.OK) + return verifyResult; } - return verifyResult; + return firstSlotVerifyResult; } - ClusterSlotVerificationResult MultiKeySlotVerify(ClusterConfig config, ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + ClusterSlotVerificationResult VerifyKeysInRange(ref ClusterConfig config, ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, + int startIdx, int lastIdx, int step, ushort firstSlot, bool waitForStableSlot, ref ClusterSlotVerificationResult verifyResult) { - ref var key = ref parseState.GetArgSliceByRef(csvi.firstKey); - var slot = ArgSliceUtils.HashSlot(ref key); - var verifyResult = SingleKeySlotVerify(ref config, ref key, csvi.readOnly, csvi.sessionAsking, csvi.waitForStableSlot, slot); - var secondKey = csvi.firstKey + csvi.step; - - for (var i = secondKey; i < csvi.lastKey; i += csvi.step) + for (var i = startIdx; i <= lastIdx; i += step) { - if (csvi.keyNumOffset == i) - continue; - key = ref parseState.GetArgSliceByRef(i); - var _slot = ArgSliceUtils.HashSlot(ref key); - var _verifyResult = SingleKeySlotVerify(ref config, ref key, csvi.readOnly, csvi.sessionAsking, csvi.waitForStableSlot, _slot); - - // Check if slot changes between keys - if (_slot != slot) - return new(SlotVerifiedState.CROSSSLOT, slot); - - // Check if any key might have moved - if (_verifyResult.state != verifyResult.state) - return new(SlotVerifiedState.TRYAGAIN, slot); + ref var key = ref parseState.GetArgSliceByRef(i); + var slot = HashSlotUtils.HashSlot(key); + var result = SingleKeySlotVerify(ref config, ref key, csvi.readOnly, csvi.sessionAsking > 0, waitForStableSlot, slot); + + if (slot != firstSlot) + return new(SlotVerifiedState.CROSSSLOT, firstSlot); + if (result.state != verifyResult.state) + return new(SlotVerifiedState.TRYAGAIN, firstSlot); } - return verifyResult; + return default; } } } \ No newline at end of file diff --git a/libs/cluster/Session/SlotVerification/RespClusterIterativeSlotVerify.cs b/libs/cluster/Session/SlotVerification/RespClusterIterativeSlotVerify.cs index d39ecc654a7..b66605ef0e5 100644 --- a/libs/cluster/Session/SlotVerification/RespClusterIterativeSlotVerify.cs +++ b/libs/cluster/Session/SlotVerification/RespClusterIterativeSlotVerify.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet.cluster { @@ -27,16 +28,15 @@ public void ResetCachedSlotVerificationResult() /// /// /// - /// /// - public bool NetworkIterativeSlotVerify(ArgSlice keySlice, bool readOnly, byte SessionAsking, bool waitForStableSlot) + public bool NetworkIterativeSlotVerify(PinnedSpanByte keySlice, bool readOnly, byte SessionAsking, bool waitForStableSlot) { ClusterSlotVerificationResult verifyResult; // If it is the first verification initialize the result cache if (!initialized) { - verifyResult = SingleKeySlotVerify(ref configSnapshot, ref keySlice, readOnly, SessionAsking, waitForStableSlot); + verifyResult = SingleKeySlotVerify(ref configSnapshot, ref keySlice, readOnly, SessionAsking > 0, waitForStableSlot); cachedVerificationResult = verifyResult; initialized = true; return verifyResult.state == SlotVerifiedState.OK; @@ -46,7 +46,7 @@ public bool NetworkIterativeSlotVerify(ArgSlice keySlice, bool readOnly, byte Se if (cachedVerificationResult.state != SlotVerifiedState.OK) return false; - verifyResult = SingleKeySlotVerify(ref configSnapshot, ref keySlice, readOnly, SessionAsking, waitForStableSlot); + verifyResult = SingleKeySlotVerify(ref configSnapshot, ref keySlice, readOnly, SessionAsking > 0, waitForStableSlot); // Check if slot changes between keys if (verifyResult.slot != cachedVerificationResult.slot) diff --git a/libs/cluster/Session/SlotVerification/RespClusterSlotVerify.cs b/libs/cluster/Session/SlotVerification/RespClusterSlotVerify.cs index fa3efe3dc11..8e1174e2892 100644 --- a/libs/cluster/Session/SlotVerification/RespClusterSlotVerify.cs +++ b/libs/cluster/Session/SlotVerification/RespClusterSlotVerify.cs @@ -86,32 +86,6 @@ private void WriteClusterSlotVerificationMessage(ClusterConfig config, ClusterSl SendAndReset(ref dcurr, ref dend); } - /// - /// Check if read/write is permitted on an array of keys and generate appropriate resp response. - /// - /// - /// - /// - /// - /// - /// - /// - /// - public bool NetworkKeyArraySlotVerify(Span keys, bool readOnly, byte sessionAsking, bool waitForStableSlot, ref byte* dcurr, ref byte* dend, int count = -1) - { - // If cluster is not enabled or a transaction is running skip slot check - if (!clusterProvider.serverOptions.EnableCluster || txnManager.state == TxnState.Running) return false; - - var config = clusterProvider.clusterManager.CurrentConfig; - var vres = MultiKeySlotVerify(config, ref keys, readOnly, sessionAsking, waitForStableSlot, count); - - if (vres.state == SlotVerifiedState.OK) - return false; - else - WriteClusterSlotVerificationMessage(config, vres, ref dcurr, ref dend); - return true; - } - /// /// Verify multi-key slot ownership /// @@ -120,13 +94,13 @@ public bool NetworkKeyArraySlotVerify(Span keys, bool readOnly, byte s /// /// /// - public unsafe bool NetworkMultiKeySlotVerify(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend) + public unsafe bool NetworkMultiKeySlotVerify(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend, bool isTxn = false) { // If cluster is not enabled or a transaction is running skip slot check if (!clusterProvider.serverOptions.EnableCluster || txnManager.state == TxnState.Running) return false; var config = clusterProvider.clusterManager.CurrentConfig; - var vres = MultiKeySlotVerify(config, ref parseState, ref csvi); + var vres = MultiKeySlotVerify(config, ref parseState, ref csvi, isTxn, csvi.waitForStableSlot); if (vres.state == SlotVerifiedState.OK) return false; @@ -142,14 +116,15 @@ public unsafe bool NetworkMultiKeySlotVerify(ref SessionParseState parseState, r /// /// /// + /// /// - public unsafe bool NetworkMultiKeySlotVerifyNoResponse(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend) + public unsafe bool NetworkMultiKeySlotVerifyNoResponse(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend, bool isTxn = false) { // If cluster is not enabled or a transaction is running skip slot check if (!clusterProvider.serverOptions.EnableCluster || txnManager.state == TxnState.Running) return false; var config = clusterProvider.clusterManager.CurrentConfig; - var vres = MultiKeySlotVerify(config, ref parseState, ref csvi); + var vres = MultiKeySlotVerify(config, ref parseState, ref csvi, isTxn, csvi.waitForStableSlot); return vres.state != SlotVerifiedState.OK; } diff --git a/libs/common/BitVector.cs b/libs/common/BitVector.cs new file mode 100644 index 00000000000..a176e241bb3 --- /dev/null +++ b/libs/common/BitVector.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.InteropServices; + +namespace Garnet.common +{ + public struct BitVector(int bytes) + { + readonly byte[] vector = new byte[bytes]; + + void GetOffsets(int index, out int byteIndex, out int bitIndex) + { + byteIndex = index >> 3; + bitIndex = index & 7; + } + + /// + /// Check if bit at index in the bit vector is set + /// + /// + /// + public bool IsSet(int index) + { + GetOffsets(index, out var byteIndex, out var bitIndex); + return (vector[byteIndex] & (byte)(1 << bitIndex)) > 0; + } + + /// + /// Set bit at index + /// + /// + /// True if bit was previously not set, false otherwise + public bool SetBit(int index) + { + GetOffsets(index, out var byteIndex, out var bitIndex); + var wasClear = (vector[byteIndex] & (byte)(1 << bitIndex)) == 0; + vector[byteIndex] |= (byte)(1 << bitIndex); + return wasClear; + } + + /// + /// Clear all bits set in this bit vector + /// + public void Clear() + => Array.Clear(vector); + + /// + /// Copy span to this BitVector + /// + /// + public readonly void CopyTo(Span span) + => vector.CopyTo(span); + + /// + /// Copy from span + /// + /// + /// + public static BitVector CopyFrom(Span span) + { + var bitVector = new BitVector(span.Length); + span.CopyTo(bitVector.vector); + return bitVector; + } + + /// + /// Count bits set in this BitVector + /// + /// + public readonly int PopCount() + { + var count = 0; + ReadOnlySpan ulongs = MemoryMarshal.Cast(vector); + foreach (var value in ulongs) + count += BitOperations.PopCount(value); + + // Handle remaining bytes + var remainder = vector.Length % 8; + for (var i = vector.Length - remainder; i < vector.Length; i++) + count += BitOperations.PopCount(vector[i]); + + return count; + } + } +} \ No newline at end of file diff --git a/libs/common/FixedSpanByteKey.cs b/libs/common/FixedSpanByteKey.cs new file mode 100644 index 00000000000..b69f0d61315 --- /dev/null +++ b/libs/common/FixedSpanByteKey.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +#if !NET9_0_OR_GREATER +using System.Runtime.InteropServices; +#endif +using Tsavorite.core; + +namespace Garnet.common +{ + /// + /// Key type which wraps a . + /// + /// In addition to the span being pinned during, it must also be "fixed" - that is unmoving and not-reused over the whole lifetime of a Tsavorite operation. + /// This is inclusive of asynchronous completions. + /// + public readonly +#if NET9_0_OR_GREATER + ref +#endif + struct FixedSpanByteKey : IKey + { +#if !NET9_0_OR_GREATER + private readonly unsafe void* ptr; + private readonly int len; +#endif + + /// + public readonly bool IsPinned + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => true; + } + + /// + public readonly bool IsEmpty + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => false; + } + + /// + public readonly ReadOnlySpan KeyBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#if NET9_0_OR_GREATER + get; +#else + get + { + unsafe + { + return new(ptr, len); + } + } +#endif + } + + /// + public readonly bool HasNamespace + { + get => false; + } + + /// + public readonly ReadOnlySpan NamespaceBytes + { + get + { + Debug.Fail("Should never be called on FixedSpanByteKey"); + return []; + } + } + + private FixedSpanByteKey(ReadOnlySpan key) + { +#if NET9_0_OR_GREATER + KeyBytes = key; +#else + unsafe + { + ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(key)); + } + len = key.Length; +#endif + } + + /// + public override readonly string ToString() => SpanByte.ToShortString(KeyBytes); + + /// + /// Convert a pinned and "fixed" (data will be unchanged and unmoving until after any async ops complete) to a . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static explicit operator FixedSpanByteKey(ReadOnlySpan key) + => new(key); + + /// + /// Convert a pinned and "fixed" (data will be unchanged and unmoving until after any async ops complete) to a . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static explicit operator FixedSpanByteKey(PinnedSpanByte key) + => new(key.ReadOnlySpan); + } +} \ No newline at end of file diff --git a/libs/common/GarnetException.cs b/libs/common/GarnetException.cs index 379523a71d3..c71773d7c55 100644 --- a/libs/common/GarnetException.cs +++ b/libs/common/GarnetException.cs @@ -3,6 +3,8 @@ using System; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Text; using Microsoft.Extensions.Logging; namespace Garnet.common @@ -51,6 +53,19 @@ public GarnetException(string message, LogLevel logLevel = LogLevel.Trace, bool DisposeSession = disposeSession; } + /// + /// Throw Garnet exception with message. + /// + /// + /// + /// + /// + /// + public GarnetException(ReadOnlySpan messageBytes, LogLevel logLevel = LogLevel.Trace, bool clientResponse = true, bool panic = false, bool disposeSession = true) + : this(Encoding.ASCII.GetString(messageBytes)) + { + } + /// /// Throw Garnet exception with message and inner exception. /// @@ -69,12 +84,13 @@ public GarnetException(string message, Exception innerException, LogLevel logLev } /// - /// Throw helper that throws a GarnetException. + /// Throw helper that throws a GarnetException. We use a method wrapper so that the caller method can execute inlined. /// /// /// /// [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] public static void Throw(string message, LogLevel logLevel = LogLevel.Trace) => throw new GarnetException(message, logLevel); } diff --git a/libs/common/GarnetKeyComparer.cs b/libs/common/GarnetKeyComparer.cs new file mode 100644 index 00000000000..6d5f9c38fb8 --- /dev/null +++ b/libs/common/GarnetKeyComparer.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Tsavorite.core; + +namespace Garnet.common +{ + /// + /// which is aware of the different key types present in Garnet and special cases accordingly. + /// + public readonly struct GarnetKeyComparer : IKeyComparer + { + public static readonly GarnetKeyComparer Instance = new(); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => StaticEquals(k1, k2); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => StaticGetHashCode64(key); + + /// + /// Equality comparison + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool StaticEquals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // Special cases for FixedSpanByteKey + if (typeof(TFirstKey) == typeof(FixedSpanByteKey)) + { + // Guarantee, irrespective of inlining, that we reduce to this + if (typeof(TSecondKey) == typeof(FixedSpanByteKey)) + { + return SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + + if (typeof(TSecondKey) == typeof(VectorElementKey)) + { + // Vector elements always has namespace, never equal + return false; + } + + if (k2.HasNamespace) + { + return false; + } + + return k1.KeyBytes.SequenceEqual(k2.KeyBytes); + } + else if (typeof(TSecondKey) == typeof(FixedSpanByteKey)) + { + if (k1.HasNamespace) + { + return false; + } + + return SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + + // Special cases for VectorElementKey + if (typeof(TFirstKey) == typeof(VectorElementKey)) + { + // Guarantee, irrespective of inlining, that we reduce to this + if (typeof(TSecondKey) == typeof(VectorElementKey)) + { + return SpanByteComparer.StaticEquals(k1.NamespaceBytes, k2.NamespaceBytes) && SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + + if (typeof(TSecondKey) == typeof(FixedSpanByteKey)) + { + // FixedSpanByteKey never has namespace, never equal + return false; + } + + if (!k2.HasNamespace) + { + return false; + } + + return SpanByteComparer.StaticEquals(k1.NamespaceBytes, k2.NamespaceBytes) && SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + else if (typeof(TSecondKey) == typeof(VectorElementKey)) + { + if (!k1.HasNamespace) + { + return false; + } + + return SpanByteComparer.StaticEquals(k1.NamespaceBytes, k2.NamespaceBytes) && SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + + // Generic cases + if (k1.HasNamespace) + { + if (!k2.HasNamespace) + { + return false; + } + + return SpanByteComparer.StaticEquals(k1.NamespaceBytes, k2.NamespaceBytes) && SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + else if (k2.HasNamespace) + { + // Know that k1 has no namespace, bail + return false; + } + + // Known no namespace + return SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); + } + + /// + /// Get 64-bit hash code + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long StaticGetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // Guarantee, irrespective of inlining decisions, that FixedSpanByteKey is special cased + if (typeof(TKey) == typeof(FixedSpanByteKey)) + { + return SpanByteComparer.StaticGetHashCode64(key.KeyBytes); + } + + // Guarantee, irrespective of inlining decisions, that VectorElementKey is special cased + if (typeof(TKey) == typeof(VectorElementKey)) + { + // TODO: Better hash construction? + return SpanByteComparer.StaticGetHashCode64(key.KeyBytes) ^ SpanByteComparer.StaticGetHashCode64(key.NamespaceBytes); + } + + // Generic cases + if (key.HasNamespace) + { + // TODO: Better hash construction? + return SpanByteComparer.StaticGetHashCode64(key.KeyBytes) ^ SpanByteComparer.StaticGetHashCode64(key.NamespaceBytes); + } + else + { + return SpanByteComparer.StaticGetHashCode64(key.KeyBytes); + } + } + } +} \ No newline at end of file diff --git a/libs/common/HashSlotUtils.cs b/libs/common/HashSlotUtils.cs index 67fbc4d29fd..31feff3d942 100644 --- a/libs/common/HashSlotUtils.cs +++ b/libs/common/HashSlotUtils.cs @@ -10,8 +10,6 @@ namespace Garnet.common { public static unsafe class HashSlotUtils { - public const ushort MaxHashSlot = 16_383; - /// /// This table is based on the CRC-16-CCITT polynomial (0x1021) /// @@ -73,19 +71,24 @@ internal static unsafe ushort Hash(byte* data, int len) return result; } + /// + /// Compute hash slot from the given ArgSlice + /// + public static unsafe ushort HashSlot(PinnedSpanByte argSlice) + => HashSlot(argSlice.ToPointer(), argSlice.Length); + /// /// Compute hash slot from the given SpanByte /// - /// - /// - public static unsafe ushort HashSlot(ref SpanByte key) - => HashSlot(key.ToPointer(), key.LengthWithoutMetadata); + public static unsafe ushort HashSlot(ReadOnlySpan key) + { + fixed (byte* keyPtr = key) + return HashSlot(keyPtr, key.Length); + } /// /// Compute hash slot of given data /// - /// - /// public static unsafe ushort HashSlot(Span key) { fixed (byte* keyPtr = key) @@ -103,14 +106,14 @@ public static unsafe ushort HashSlot(byte* keyPtr, int ksize) var startPtr = keyPtr; var end = keyPtr + ksize; - // Find first occurrence of '{' + // Find first occurence of '{' while (startPtr < end && *startPtr != '{') { startPtr++; } // Return early if did not find '{' - if (startPtr == end) return (ushort)(Hash(keyPtr, ksize) & MaxHashSlot); + if (startPtr == end) return (ushort)(Hash(keyPtr, ksize) & 16383); var endPtr = startPtr + 1; @@ -118,10 +121,10 @@ public static unsafe ushort HashSlot(byte* keyPtr, int ksize) while (endPtr < end && *endPtr != '}') { endPtr++; } // Return early if did not find '}' after '{' - if (endPtr == end || endPtr == startPtr + 1) return (ushort)(Hash(keyPtr, ksize) & MaxHashSlot); + if (endPtr == end || endPtr == startPtr + 1) return (ushort)(Hash(keyPtr, ksize) & 16383); // Return hash for byte sequence between brackets - return (ushort)(Hash(startPtr + 1, (int)(endPtr - startPtr - 1)) & MaxHashSlot); + return (ushort)(Hash(startPtr + 1, (int)(endPtr - startPtr - 1)) & 16383); } } } \ No newline at end of file diff --git a/libs/common/HashUtils.cs b/libs/common/HashUtils.cs index bbb325baeb0..000f6cead97 100644 --- a/libs/common/HashUtils.cs +++ b/libs/common/HashUtils.cs @@ -190,7 +190,7 @@ public static unsafe (ulong, ulong) MurmurHash3x128(byte* bString, int len, uint return (h1, h2); } - public static unsafe ulong MurmurHash2x64A(Span bString, uint seed = 0) + public static unsafe ulong MurmurHash2x64A(ReadOnlySpan bString, uint seed = 0) { fixed (byte* p = bString) { diff --git a/libs/common/LightClient.cs b/libs/common/LightClient.cs index d04bc392812..17169e231c5 100644 --- a/libs/common/LightClient.cs +++ b/libs/common/LightClient.cs @@ -50,6 +50,7 @@ public class LightClient : ClientBase, IServerHook, IMessageConsumer /// Callback that takes in a byte array and length, and returns the number of bytes read and the number of requests processed /// Message buffer size. /// SSL options + /// ILogger instance public unsafe LightClient( EndPoint endpoint, int opType, @@ -60,7 +61,7 @@ public unsafe LightClient( : base(endpoint, BufferSize) { this.networkBufferSettings = new NetworkBufferSettings(BufferSize, BufferSize); - this.networkPool = networkBufferSettings.CreateBufferPool(); + this.networkPool = networkBufferSettings.CreateBufferPool(ownerType: PoolOwnerType.LightClient, logger: logger); this.onResponseDelegateUnsafe = onResponseDelegateUnsafe ?? new OnResponseDelegateUnsafe(DefaultLightReceiveUnsafe); this.opType = opType; this.BufferSize = BufferSize; @@ -168,7 +169,7 @@ private async Task ConnectSendSocketAsync(CancellationToken cancellation NoDelay = true }; - if (await TryConnectSocketAsync(socket, endpoint, cancellationToken)) + if (await TryConnectSocketAsync(socket, endpoint, cancellationToken).ConfigureAwait(false)) return socket; } } @@ -178,7 +179,7 @@ private async Task ConnectSendSocketAsync(CancellationToken cancellation if (endpoint is not UnixDomainSocketEndPoint) socket.NoDelay = true; - if (await TryConnectSocketAsync(socket, endpoint, cancellationToken)) + if (await TryConnectSocketAsync(socket, endpoint, cancellationToken).ConfigureAwait(false)) return socket; } diff --git a/libs/common/Memory/LimitedFixedBufferPool.cs b/libs/common/Memory/LimitedFixedBufferPool.cs index 904bb3c1c20..20c40262464 100644 --- a/libs/common/Memory/LimitedFixedBufferPool.cs +++ b/libs/common/Memory/LimitedFixedBufferPool.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. using System; +#if DEBUG +using System.Collections.Concurrent; +#endif using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; @@ -26,6 +29,11 @@ public sealed class LimitedFixedBufferPool : IDisposable readonly int maxAllocationSize; readonly ILogger logger; + /// + /// Pool owner type, packed into byte 1 of each . + /// + readonly int ownerByte; + /// /// Min allocation size /// @@ -41,16 +49,29 @@ public sealed class LimitedFixedBufferPool : IDisposable /// int totalOutOfBoundAllocations; +#if DEBUG + /// + /// Tracks all outstanding (checked-out) pool entries for leak diagnosis. + /// + readonly ConcurrentDictionary outstandingEntries = new(); + + /// + /// Timeout in milliseconds for Dispose to wait before logging outstanding entries. + /// + const int DisposeWaitDiagnosticMs = 5_000; +#endif + /// /// Constructor /// - public LimitedFixedBufferPool(int minAllocationSize, int maxEntriesPerLevel = 16, int numLevels = 4, ILogger logger = null) + public LimitedFixedBufferPool(int minAllocationSize, int maxEntriesPerLevel = 16, int numLevels = 4, PoolOwnerType ownerType = PoolOwnerType.Unknown, ILogger logger = null) { this.minAllocationSize = minAllocationSize; this.maxAllocationSize = minAllocationSize << (numLevels - 1); this.maxEntriesPerLevel = maxEntriesPerLevel; this.numLevels = numLevels; this.logger = logger; + this.ownerByte = (int)ownerType << 8; pool = new PoolLevel[numLevels]; } @@ -85,6 +106,9 @@ public bool Validate(NetworkBufferSettings settings) [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Return(PoolEntry buffer) { +#if DEBUG + outstandingEntries.TryRemove(buffer, out _); +#endif var level = Position(buffer.entry.Length); if (level >= 0) { @@ -107,9 +131,10 @@ public void Return(PoolEntry buffer) /// Get buffer /// /// + /// Identifies the caller for leak diagnosis. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe PoolEntry Get(int size) + public unsafe PoolEntry Get(int size, PoolEntryBufferType bufferType = PoolEntryBufferType.Unknown) { if (Interlocked.Increment(ref totalReferences) < 0) { @@ -118,6 +143,8 @@ public unsafe PoolEntry Get(int size) return null; } + var source = ownerByte | (int)bufferType; + var level = Position(size); if (level == -1) Interlocked.Increment(ref totalOutOfBoundAllocations); @@ -132,10 +159,19 @@ public unsafe PoolEntry Get(int size) { Interlocked.Decrement(ref pool[level].size); page.Reuse(); + page.source = source; +#if DEBUG + outstandingEntries[page] = 0; +#endif return page; } } - return new PoolEntry(size, this); + var entry = new PoolEntry(size, this); + entry.source = source; +#if DEBUG + outstandingEntries[entry] = 0; +#endif + return entry; } /// @@ -157,23 +193,37 @@ public void Purge() } /// - /// Dipose pool entries from all levels + /// Dispose pool entries from all levels /// NOTE: /// This is used to destroy the instance and reclaim all allocated buffer pool entries. /// As a consequence it spin waits until totalReferences goes back down to 0 and blocks any future allocations. + /// In DEBUG builds, logs outstanding unreturned entries after a timeout for leak diagnosis. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Dispose() { -#if HANGDETECT - int count = 0; +#if DEBUG + var sw = Stopwatch.StartNew(); + var diagnosed = false; #endif while (totalReferences > int.MinValue && Interlocked.CompareExchange(ref totalReferences, int.MinValue, 0) != 0) { -#if HANGDETECT - if (++count % 10000 == 0) - logger?.LogTrace("Dispose iteration {count}, {activeHandlerCount}", count, activeHandlerCount); +#if DEBUG + if (!diagnosed && sw.ElapsedMilliseconds > DisposeWaitDiagnosticMs) + { + diagnosed = true; + var remaining = totalReferences; + var ownerType = (PoolOwnerType)(ownerByte >> 8); + logger?.LogError("LimitedFixedBufferPool.Dispose blocked with {remaining} unreturned references (poolOwner={ownerType}). Outstanding entries:", remaining, ownerType); + foreach (var kvp in outstandingEntries) + { + var entryBufferType = (PoolEntryBufferType)(kvp.Key.source & 0xFF); + var entryOwnerType = (PoolOwnerType)((kvp.Key.source >> 8) & 0xFF); + logger?.LogCritical(" Unreturned buffer: ownerType={ownerType}, bufferType={bufferType}, size={size}", + entryOwnerType, entryBufferType, kvp.Key.entry.Length); + } + } #endif Thread.Yield(); } diff --git a/libs/common/Memory/PoolEntry.cs b/libs/common/Memory/PoolEntry.cs index 2619dce2c6d..bbb90b834a3 100644 --- a/libs/common/Memory/PoolEntry.cs +++ b/libs/common/Memory/PoolEntry.cs @@ -25,6 +25,12 @@ public unsafe class PoolEntry : IDisposable readonly LimitedFixedBufferPool pool; bool disposed; + /// + /// Packed source identifier: low byte = , byte 1 = . + /// Set when the entry is acquired via . + /// + internal int source; + /// /// Constructor /// diff --git a/libs/common/Memory/PoolEntryTypes.cs b/libs/common/Memory/PoolEntryTypes.cs new file mode 100644 index 00000000000..5419a7c69ab --- /dev/null +++ b/libs/common/Memory/PoolEntryTypes.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.common +{ + /// + /// Identifies the buffer role when a is acquired from . + /// + public enum PoolEntryBufferType : byte + { + /// Default/unknown buffer type. + Unknown = 0, + + /// Initial network receive buffer (TcpNetworkHandlerBase). + NetworkReceiveBuffer = 1, + + /// Transport receive buffer for TLS (NetworkHandler). + TransportReceiveBuffer = 2, + + /// Transport send buffer for TLS (NetworkHandler). + TransportSendBuffer = 3, + + /// Doubled network receive buffer (NetworkHandler). + DoubleNetworkReceiveBuffer = 4, + + /// Shrunk network receive buffer (NetworkHandler). + ShrinkNetworkReceiveBuffer = 5, + + /// Doubled transport receive buffer for TLS (NetworkHandler). + DoubleTransportReceiveBuffer = 6, + + /// Send buffer for async socket operations (GarnetSaeaBuffer). + SaeaSendBuffer = 7, + } + + /// + /// Identifies the owner of a instance. + /// Set at pool construction time to indicate which subsystem created the pool. + /// + public enum PoolOwnerType : byte + { + /// Default/unknown owner. + Unknown = 0, + + /// Server-side network pool (GarnetServerTcp). + ServerNetwork = 1, + + /// Replication network pool (ReplicationManager). + Replication = 2, + + /// Client-side network pool (GarnetClientSession, self-managed). + GarnetClientSession = 3, + + /// Migration network pool (MigrationManager). + Migration = 4, + + /// Client-side network pool (LightClient, self-managed). + LightClient = 5, + + /// Client-side network pool (GarnetClient, self-managed). + GarnetClient = 6, + } +} \ No newline at end of file diff --git a/libs/common/Metrics/InfoMetricsType.cs b/libs/common/Metrics/InfoMetricsType.cs index 469fb162845..6e9f894ad8f 100644 --- a/libs/common/Metrics/InfoMetricsType.cs +++ b/libs/common/Metrics/InfoMetricsType.cs @@ -39,26 +39,14 @@ public enum InfoMetricsType : byte /// STORE, /// - /// Object store info - /// - OBJECTSTORE, - /// /// Store hash table info /// STOREHASHTABLE, /// - /// Object store hash table info - /// - OBJECTSTOREHASHTABLE, - /// /// Store revivification info /// STOREREVIV, /// - /// Object store hash table info - /// - OBJECTSTOREREVIV, - /// /// Persistence information /// PERSISTENCE, @@ -83,7 +71,7 @@ public enum InfoMetricsType : byte /// CINFO, /// - /// Scan and return distribution of in-memory portion of hybrid logs for main store and object store + /// Scan and return distribution of in-memory portion of hybrid logs /// HLOGSCAN, /// diff --git a/libs/common/NetworkBufferSettings.cs b/libs/common/NetworkBufferSettings.cs index bbad334899b..0f0b0b282d1 100644 --- a/libs/common/NetworkBufferSettings.cs +++ b/libs/common/NetworkBufferSettings.cs @@ -29,10 +29,13 @@ public class NetworkBufferSettings /// public readonly int maxReceiveBufferSize; + /// Reserve some space for overhead in send buffer when determining the max size of a single send buffer (e.g. for object serialization during migration) + public const int SendBufferOverheadReserve = 64; // TODO verify this value + /// /// Default constructor /// - public NetworkBufferSettings() : this(1 << 17, 1 << 17, 1 << 20) { } + public NetworkBufferSettings() : this(sendBufferSize: 1 << 17, initialReceiveBufferSize: 1 << 17, maxReceiveBufferSize: 1 << 20) { } /// /// Set network buffer sizes without allocating them @@ -70,9 +73,10 @@ public static NetworkBufferSettings GetInclusive(NetworkBufferSettings[] setting /// Allocate network buffer pool /// /// + /// /// /// - public LimitedFixedBufferPool CreateBufferPool(int maxEntriesPerLevel = 16, ILogger logger = null) + public LimitedFixedBufferPool CreateBufferPool(int maxEntriesPerLevel = 16, PoolOwnerType ownerType = PoolOwnerType.Unknown, ILogger logger = null) { var minSize = Math.Min(Math.Min(sendBufferSize, initialReceiveBufferSize), maxReceiveBufferSize); var maxSize = Math.Max(Math.Max(sendBufferSize, initialReceiveBufferSize), maxReceiveBufferSize); @@ -80,7 +84,7 @@ public LimitedFixedBufferPool CreateBufferPool(int maxEntriesPerLevel = 16, ILog var levels = LimitedFixedBufferPool.GetLevel(minSize, maxSize) + 1; Debug.Assert(levels >= 0); levels = Math.Max(4, levels); - return new LimitedFixedBufferPool(minSize, maxEntriesPerLevel: maxEntriesPerLevel, numLevels: levels, logger: logger); + return new LimitedFixedBufferPool(minSize, maxEntriesPerLevel: maxEntriesPerLevel, numLevels: levels, logger: logger, ownerType: ownerType); } public void Log(ILogger logger, string category) diff --git a/libs/common/Networking/GarnetSaeaBuffer.cs b/libs/common/Networking/GarnetSaeaBuffer.cs index bf2b362aaf1..70ad41d7a73 100644 --- a/libs/common/Networking/GarnetSaeaBuffer.cs +++ b/libs/common/Networking/GarnetSaeaBuffer.cs @@ -30,7 +30,7 @@ public GarnetSaeaBuffer(EventHandler eventHandler, Network { socketEventAsyncArgs = new SocketAsyncEventArgs(); - buffer = networkPool.Get(networkBufferSettings.sendBufferSize); + buffer = networkPool.Get(networkBufferSettings.sendBufferSize, PoolEntryBufferType.SaeaSendBuffer); socketEventAsyncArgs.SetBuffer(buffer.entry, 0, buffer.entry.Length); socketEventAsyncArgs.Completed += eventHandler; } diff --git a/libs/common/Networking/GarnetTcpNetworkSender.cs b/libs/common/Networking/GarnetTcpNetworkSender.cs index 29e71b39893..8c3d77a38d4 100644 --- a/libs/common/Networking/GarnetTcpNetworkSender.cs +++ b/libs/common/Networking/GarnetTcpNetworkSender.cs @@ -202,7 +202,7 @@ void ReturnBuffer(GarnetSaeaBuffer buffer) { if (responseObject != null) return responseObject.buffer.entryPtr + responseObject.buffer.entry.Length; - return base.GetResponseObjectHead(); + return base.GetResponseObjectTail(); } /// @@ -214,6 +214,7 @@ public override bool SendResponse(int offset, int size) responseObject = null; try { + // If this does not throw, _r is ReturnBuffer()ed when it completes. Send(socket, _r, offset, size); } catch diff --git a/libs/common/Networking/NetworkHandler.cs b/libs/common/Networking/NetworkHandler.cs index a473f453d4f..6a37a949861 100644 --- a/libs/common/Networking/NetworkHandler.cs +++ b/libs/common/Networking/NetworkHandler.cs @@ -129,11 +129,11 @@ public unsafe NetworkHandler(TServerHook serverHook, TNetworkSender networkSende expectingData = new SemaphoreSlim(0); cancellationTokenSource = new(); - transportReceiveBufferEntry = this.networkPool.Get(this.networkBufferSettings.initialReceiveBufferSize); + transportReceiveBufferEntry = this.networkPool.Get(this.networkBufferSettings.initialReceiveBufferSize, PoolEntryBufferType.TransportReceiveBuffer); transportReceiveBuffer = transportReceiveBufferEntry.entry; transportReceiveBufferPtr = transportReceiveBufferEntry.entryPtr; - transportSendBufferEntry = this.networkPool.Get(this.networkBufferSettings.sendBufferSize); + transportSendBufferEntry = this.networkPool.Get(this.networkBufferSettings.sendBufferSize, PoolEntryBufferType.TransportSendBuffer); transportSendBuffer = transportSendBufferEntry.entry; transportSendBufferPtr = transportSendBufferEntry.entryPtr; } @@ -516,7 +516,7 @@ unsafe bool TryProcessRequest() unsafe void DoubleNetworkReceiveBuffer() { - var tmp = networkPool.Get(networkReceiveBuffer.Length * 2); + var tmp = networkPool.Get(networkReceiveBuffer.Length * 2, PoolEntryBufferType.DoubleNetworkReceiveBuffer); Array.Copy(networkReceiveBuffer, tmp.entry, networkReceiveBuffer.Length); networkReceiveBufferEntry.Dispose(); networkReceiveBufferEntry = tmp; @@ -530,7 +530,7 @@ unsafe void ShrinkNetworkReceiveBuffer() { Debug.Assert(networkReadHead == 0, "Shouldn't call if remaining data not already moved to head of receive buffer"); - var tmp = networkPool.Get(networkBufferSettings.maxReceiveBufferSize); + var tmp = networkPool.Get(networkBufferSettings.maxReceiveBufferSize, PoolEntryBufferType.ShrinkNetworkReceiveBuffer); if (networkBytesRead > 0) { Array.Copy(networkReceiveBuffer, tmp.entry, networkBytesRead); @@ -558,7 +558,7 @@ unsafe void DoubleTransportReceiveBuffer() { if (sslStream != null) { - var tmp = networkPool.Get(transportReceiveBuffer.Length * 2); + var tmp = networkPool.Get(transportReceiveBuffer.Length * 2, PoolEntryBufferType.DoubleTransportReceiveBuffer); Array.Copy(transportReceiveBuffer, tmp.entry, transportReceiveBuffer.Length); transportReceiveBufferEntry.Dispose(); transportReceiveBufferEntry = tmp; diff --git a/libs/common/Networking/TcpNetworkHandlerBase.cs b/libs/common/Networking/TcpNetworkHandlerBase.cs index 235bb52fbf5..9dd64e56b70 100644 --- a/libs/common/Networking/TcpNetworkHandlerBase.cs +++ b/libs/common/Networking/TcpNetworkHandlerBase.cs @@ -246,7 +246,7 @@ private async ValueTask HandleReceiveWithTLSAsync(object sender, SocketAsyncEven var receiveTask = OnNetworkReceiveWithTLSAsync(e.BytesTransferred); if (!receiveTask.IsCompletedSuccessfully) { - await receiveTask; + await receiveTask.ConfigureAwait(false); } e.SetBuffer(networkReceiveBuffer, networkBytesRead, networkReceiveBuffer.Length - networkBytesRead); } while (!e.AcceptSocket.ReceiveAsync(e)); @@ -268,7 +268,7 @@ void HandleReceiveFailure(Exception ex, SocketAsyncEventArgs e) unsafe void AllocateNetworkReceiveBuffer() { - networkReceiveBufferEntry = networkPool.Get(networkBufferSettings.initialReceiveBufferSize); + networkReceiveBufferEntry = networkPool.Get(networkBufferSettings.initialReceiveBufferSize, PoolEntryBufferType.NetworkReceiveBuffer); networkReceiveBuffer = networkReceiveBufferEntry.entry; networkReceiveBufferPtr = networkReceiveBufferEntry.entryPtr; } diff --git a/libs/common/NumUtils.cs b/libs/common/NumUtils.cs index de3ecfc50ce..25972e46f18 100644 --- a/libs/common/NumUtils.cs +++ b/libs/common/NumUtils.cs @@ -4,6 +4,7 @@ using System; using System.Buffers.Text; using System.Diagnostics; +using System.Numerics; namespace Garnet.common { @@ -41,9 +42,10 @@ public static int WriteInt64(long value, Span destination) /// Writes 64-bit signed integer as ASCII. /// /// The value to write - /// + /// Number of digits in ; does *not* include space for the negative sign if is negative. + /// The space pointed to by must include sufficient space including sign if negative. /// Byte pointer, will be updated to point after the written number - public static unsafe void WriteInt64(long value, int length, ref byte* result) + public static void WriteInt64(long value, int length, ref byte* result) { var isNegative = value < 0; if (value == long.MinValue) @@ -85,7 +87,6 @@ public static unsafe void WriteInt64(long value, int length, ref byte* result) public static int WriteDouble(double value, Span destination) { var totalLen = CountCharsInDouble(value, out var integerDigits, out var signSize, out var fractionalDigits); - var isNegative = value < 0; if (totalLen > destination.Length) return 0; fixed (byte* ptr = destination) @@ -97,6 +98,20 @@ public static int WriteDouble(double value, Span destination) return totalLen; } + /// + /// Writes as ASCII. + /// + /// The value to write + /// Size of the destination at ; includes space for the negative sign if present + /// Byte pointer, will be updated to point after the written number + public static void WriteDouble(double value, int length, ref byte* result) + { + Debug.Assert(!double.IsNaN(value) && !double.IsInfinity(value), "Cannot convert NaN or Infinity to bytes."); + var totalLen = CountCharsInDouble(value, out var integerDigits, out var signSize, out var fractionalDigits); + if (totalLen <= length) + WriteDouble(value, integerDigits, fractionalDigits, ref result); + } + /// /// Writes as ASCII. /// @@ -104,7 +119,7 @@ public static int WriteDouble(double value, Span destination) /// Number of digits in the integer part of the double value /// Number of digits in the fractional part of the double value /// Byte pointer, will be updated to point after the written number - public static unsafe void WriteDouble(double value, int integerDigits, int fractionalDigits, ref byte* result) + public static void WriteDouble(double value, int integerDigits, int fractionalDigits, ref byte* result) { Debug.Assert(!double.IsNaN(value) && !double.IsInfinity(value), "Cannot convert NaN or Infinity to bytes."); @@ -523,5 +538,17 @@ public static bool TryParseWithInfinity(ReadOnlySpan source, out double va return RespReadUtils.TryReadInfinity(source, out value); } + + /// + /// Get the leftmost bit set offset and clear the associated bit + /// + /// + /// + public static int GetNextOffset(this ref ulong value) + { + var offset = BitOperations.TrailingZeroCount(value); + value &= ~((1UL) << offset); + return offset; + } } } \ No newline at end of file diff --git a/libs/common/Parsing/RespParsingException.cs b/libs/common/Parsing/RespParsingException.cs index b120e23296a..41dcb9683fd 100644 --- a/libs/common/Parsing/RespParsingException.cs +++ b/libs/common/Parsing/RespParsingException.cs @@ -54,6 +54,17 @@ public static void ThrowInvalidLength(long len) Throw($"Invalid length '{len}'."); } + /// + /// Throw an exception indicating that the RESP array argument count exceeds the allowed maximum. + /// + /// The argument count that was received. + /// The maximum allowed argument count. + [DoesNotReturn] + public static void ThrowExcessiveArgumentCount(int count, int maxCount) + { + Throw($"RESP array argument count '{count}' exceeds maximum allowed count of '{maxCount}'."); + } + /// /// Throw NaN (not a number) exception. /// diff --git a/libs/common/ReaderWriterLock.cs b/libs/common/ReaderWriterLock.cs index f2540d573e6..7c3da2795c4 100644 --- a/libs/common/ReaderWriterLock.cs +++ b/libs/common/ReaderWriterLock.cs @@ -77,15 +77,23 @@ public void WriteUnlock() /// Acquires a reader lock, allowing concurrent read access to the resource. /// public void ReadLock() - => ReaderLock(default); + => ReadLock(default); /// /// Acquires a reader lock, allowing concurrent read access to the resource. /// /// The cancellation token used to signal the operation's cancellation. - public void ReaderLock(CancellationToken token) + public void ReadLock(CancellationToken token) => Acquire(LockOperation.Reader, token); + /// + /// Acquires a reader lock, allowing concurrent read access to the resource. + /// + /// The cancellation token used to signal the operation's cancellation. + [Obsolete("Use ReadLock(CancellationToken) instead.")] + public void ReaderLock(CancellationToken token) + => ReadLock(token); + /// /// Release reader lock and wake one writer when this was the last active reader. /// diff --git a/libs/common/RespMemoryWriter.cs b/libs/common/RespMemoryWriter.cs index 8d6ee383374..5a81fc25469 100644 --- a/libs/common/RespMemoryWriter.cs +++ b/libs/common/RespMemoryWriter.cs @@ -25,7 +25,7 @@ namespace Garnet.common ref SpanByteAndMemory output; public readonly bool resp3; - public unsafe RespMemoryWriter(byte respVersion, ref SpanByteAndMemory output) + public RespMemoryWriter(byte respVersion, ref SpanByteAndMemory output) { this.output = ref output; ptrHandle = default; @@ -378,6 +378,17 @@ public void WriteSimpleString(ReadOnlySpan simpleString) ReallocateOutput(simpleString.Length); } + /// + /// Write simple string to memory. + /// + /// An ASCII encoded simple string. The string mustn't contain a CR (\r) or LF (\n) bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void WriteSimpleString(ReadOnlySpan simpleString) + { + while (!RespWriteUtils.TryWriteSimpleString(simpleString, ref curr, end)) + ReallocateOutput(simpleString.Length); + } + /// /// Write RESP3 true /// diff --git a/libs/common/RespReadUtils.cs b/libs/common/RespReadUtils.cs index 1202e8c0e09..f283a01ea34 100644 --- a/libs/common/RespReadUtils.cs +++ b/libs/common/RespReadUtils.cs @@ -7,6 +7,7 @@ using System.Runtime.InteropServices; using System.Text; using Garnet.common.Parsing; +using Tsavorite.core; namespace Garnet.common { @@ -139,7 +140,7 @@ public static bool TryReadInt64(ref byte* ptr, byte* end, out long value, out ul /// If parsing was successful, contains the parsed long value. /// If parsing was successful, contains the number of bytes that were parsed. /// True if +/- sign was read during parsing - /// True if overflow occured during parsing + /// True if overflow occurred during parsing /// True if leading zeros allowed /// /// True if a long was successfully parsed, false if the input string did not start with @@ -239,7 +240,7 @@ public static bool TryReadInt32(ref byte* ptr, byte* end, out int value, out ulo /// If parsing was successful, contains the parsed int value. /// If parsing was successful, contains the number of bytes that were parsed. /// True if +/- sign was read during parsing - /// True if overflow occured during parsing + /// True if overflow occurred during parsing /// True if leading zeros allowed /// /// True if an int was successfully parsed, false if the input string did not start with @@ -1225,85 +1226,31 @@ public static bool TryReadAsSpan(out ReadOnlySpan result, ref byte* ptr, b } /// - /// Read serialized data for migration - /// - public static bool TryReadSerializedSpanByte(ref byte* keyPtr, ref byte keyMetaDataSize, ref byte* valPtr, ref byte valMetaDataSize, ref byte* ptr, byte* end) + /// Read serialized data for migration and replication. For details of the layout see . + /// + public static bool GetSerializedRecordSpan(out PinnedSpanByte recordSpan, ref byte* ptr, byte* end) { - //1. safe read ksize + // 1. Safe read recordSize. if (ptr + sizeof(int) > end) + { + recordSpan = default; return false; - var ksize = *(int*)ptr; - ptr += sizeof(int); - - //2. safe read key bytes - if (ptr + ksize + 1 > end) - return false; - keyPtr = ptr - sizeof(int); - ptr += ksize; - keyMetaDataSize = *ptr++; - - //3. safe read vsize - if (ptr + 4 > end) - return false; - var vsize = *(int*)ptr; + } + var recordLength = *(int*)ptr; ptr += sizeof(int); - //4. safe read value bytes - if (ptr + vsize + 1 > end) - return false; - valPtr = ptr - sizeof(int); - ptr += vsize; - valMetaDataSize = *ptr++; - - return true; - } - - /// - /// Read serialized data for migration - /// - public static bool TryReadSerializedData(out byte[] key, out byte[] value, out long expiration, ref byte* ptr, byte* end) - { - expiration = -1; - key = null; - value = null; - - //1. safe read ksize - if (ptr + 4 > end) - return false; - var keyLen = *(int*)ptr; - ptr += 4; - - //2. safe read keyPtr - if (ptr + keyLen > end) - return false; - var keyPtr = ptr; - ptr += keyLen; - - //3. safe read vsize - if (ptr + 4 > end) - return false; - var valLen = *(int*)ptr; - ptr += 4; - - //4. safe read valPtr - if (ptr + valLen > end) - return false; - var valPtr = ptr; - ptr += valLen; - - //5. safe read expiration info - if (ptr + 8 > end) + // 2. Validate record fits within the payload boundary. + // Use subtraction instead of ptr + recordLength to avoid pointer arithmetic overflow. + if (recordLength < 0 || recordLength > end - ptr) + { + recordSpan = default; return false; - expiration = *(long*)ptr; - ptr += 8; + } - key = new byte[keyLen]; - value = new byte[valLen]; - fixed (byte* kPtr = key) - Buffer.MemoryCopy(keyPtr, kPtr, keyLen, keyLen); - fixed (byte* vPtr = value) - Buffer.MemoryCopy(valPtr, vPtr, valLen, valLen); + // 3. The record starts immediately after the length prefix. + recordSpan = PinnedSpanByte.FromPinnedPointer(ptr, recordLength); + ptr += recordLength; return true; } diff --git a/libs/common/RespWriteUtils.cs b/libs/common/RespWriteUtils.cs index 3a809318637..3319cf897a7 100644 --- a/libs/common/RespWriteUtils.cs +++ b/libs/common/RespWriteUtils.cs @@ -849,13 +849,13 @@ public static bool TryWriteOne(ref byte* curr, byte* end) public static void WriteEtagValArray(long etag, ref ReadOnlySpan value, ref byte* curr, byte* end, bool writeDirect) { // Writes a Resp encoded Array of Integer for ETAG as first element, and bulk string for value as second element - RespWriteUtils.TryWriteArrayLength(2, ref curr, end); - RespWriteUtils.TryWriteInt64(etag, ref curr, end); + TryWriteArrayLength(2, ref curr, end); + TryWriteInt64(etag, ref curr, end); if (writeDirect) - RespWriteUtils.TryWriteDirect(value, ref curr, end); + TryWriteDirect(value, ref curr, end); else - RespWriteUtils.TryWriteBulkString(value, ref curr, end); + TryWriteBulkString(value, ref curr, end); } /// diff --git a/libs/common/SequenceNumberGenerator.cs b/libs/common/SequenceNumberGenerator.cs new file mode 100644 index 00000000000..af9bc1551b1 --- /dev/null +++ b/libs/common/SequenceNumberGenerator.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Garnet.common +{ + /// + /// Sequence number generator + /// + /// + public sealed class SequenceNumberGenerator(long startingOffset) + { + readonly long baseTimestamp = Stopwatch.GetTimestamp(); + readonly long startingOffset = startingOffset; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetSequenceNumber() => Stopwatch.GetTimestamp() - baseTimestamp + startingOffset; + + public override string ToString() => $"{startingOffset},{baseTimestamp},{Stopwatch.GetTimestamp()}"; + } +} \ No newline at end of file diff --git a/libs/common/Synchronization/ActiveWorkerMonitor.cs b/libs/common/Synchronization/ActiveWorkerMonitor.cs new file mode 100644 index 00000000000..69fdf0239a8 --- /dev/null +++ b/libs/common/Synchronization/ActiveWorkerMonitor.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Garnet.common +{ + /// + /// Tracks the number of active workers within a critical region. + /// On dispose, atomically prevents new workers from entering and efficiently blocks + /// until all in-flight workers have exited, without spin-waiting. + /// + public sealed class ActiveWorkerMonitor + { + private int disposed = 0; + + /// + /// Active worker count. + /// + int workerCount = 0; + + /// + /// Signaled when the last worker exits after close. + /// Starts unsignaled. + /// + readonly ManualResetEventSlim drainEvent = new(false); + + /// + /// Gets the current active worker count (0 if closed). + /// + public int CurrentCount + { + get + { + var observedCount = workerCount; + return observedCount < 0 ? 0 : observedCount; + } + } + + /// + /// Closes the monitor and blocks until all active workers have exited. + /// + public void Dispose() + { + // Guard against dispose being called multiple times + if (Interlocked.Exchange(ref disposed, 1) != 0) + { + return; + } + + // Atomically flip the count negative; if no workers were active, skip wait + TryClose(); + drainEvent.Dispose(); + } + + /// + /// Attempts to open the resource if it is currently closed. + /// + /// true if the resource was successfully opened; otherwise, false. + public bool TryOpen() + { + Debug.Assert(workerCount == int.MinValue); + drainEvent.Reset(); + return Interlocked.CompareExchange(ref workerCount, 0, int.MinValue) == int.MinValue; + } + + /// + /// Attempts to close the resource gracefully, waiting for active workers to complete within the specified + /// timeout period. + /// + /// The maximum time, in milliseconds, to wait for active workers to finish before closing. Specify -1 to wait + /// indefinitely. + /// A cancellation token that can be used to cancel the close operation before the timeout elapses. + public void TryClose(int timeout = -1, CancellationToken token = default) + { + // Atomically flip the count negative; if no workers were active, skip wait + if (Interlocked.Add(ref workerCount, int.MinValue) != int.MinValue) + _ = drainEvent.Wait(timeout, token); + } + + /// + /// Attempts to register a new active worker. + /// Returns false if the monitor has been closed. + /// + public bool TryEnter() + => TryEnter(1, out _); + + /// + /// Attempts to register a new active worker. + /// Returns false if the monitor has been closed. + /// When successful, contains the new worker count (always > 0). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryEnter(int add, out int count) + { + var cnt = Interlocked.Add(ref workerCount, add); + if (cnt < 0) + { + // Closed — undo the increment; if we happen to be the last, signal drain. + if (Interlocked.Add(ref workerCount, -add) == int.MinValue) + drainEvent.Set(); + count = 0; + return false; + } + + count = cnt; + return true; + } + + /// + /// Signals that a worker has finished. Returns the count after decrementing. + /// If the monitor is closed and this was the last worker, signals the drain event. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Exit() + { + var cnt = Interlocked.Decrement(ref workerCount); + + // Signal drain when closed and all workers have exited + if (cnt == int.MinValue) + drainEvent.Set(); + + return cnt; + } + } +} \ No newline at end of file diff --git a/libs/common/CountingEventSlim.cs b/libs/common/Synchronization/CountingEventSlim.cs similarity index 100% rename from libs/common/CountingEventSlim.cs rename to libs/common/Synchronization/CountingEventSlim.cs diff --git a/libs/common/Synchronization/LeaderBarrier.cs b/libs/common/Synchronization/LeaderBarrier.cs new file mode 100644 index 00000000000..e6d1662dd0d --- /dev/null +++ b/libs/common/Synchronization/LeaderBarrier.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; + +namespace Garnet.common +{ + /// + /// Synchronizes a group of participants, allowing one to act as the leader while others wait until released. + /// + /// + public class LeaderBarrier(int participantCount) + { + readonly int participantCount = participantCount; + int arrivedCount = participantCount; + + ManualResetEventSlim releaseFirst = new(false); + ManualResetEventSlim releaseAll = new(false); + + /// + /// Attempts to signal arrival and wait for other participants within the specified timeout and cancellation + /// token. + /// + /// When this method returns, contains the exception that occurred during the operation, or null if no exception + /// was thrown. + /// The maximum time to wait for other participants. The default value is infinite. + /// A cancellation token to observe while waiting. + /// true if the caller is the first participant to arrive; otherwise, false. + public bool TrySignalOrWait(out Exception exception, TimeSpan timeout = default, CancellationToken cancellationToken = default) + { + exception = null; + var newValue = Interlocked.Decrement(ref arrivedCount); + + try + { + // First participant to arrive + if (newValue == participantCount - 1) + { + // Wait only if there are more participants to arrive + if (newValue > 0) + _ = releaseFirst.Wait(timeout, cancellationToken); + return true; + } + + // Last participant to arrive - release the first + if (newValue == 0) + releaseFirst.Set(); + + // All non-first participants wait for release + if (newValue >= 0) + { + _ = releaseAll.Wait(timeout, cancellationToken); + return false; + } + + // Invalid state + throw new Exception("Invalid count value < 0"); + } + catch (Exception ex) + { + exception = ex; + return false; + } + } + + /// + /// Release all waiting participants + /// + public void Release() => releaseAll.Set(); + } +} \ No newline at end of file diff --git a/libs/common/Synchronization/LeaderFollowerBarrier.cs b/libs/common/Synchronization/LeaderFollowerBarrier.cs new file mode 100644 index 00000000000..60007faf3af --- /dev/null +++ b/libs/common/Synchronization/LeaderFollowerBarrier.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Garnet.common +{ + /// + /// Synchronization primitive for coordinating a leader task with multiple participant tasks in a cyclic pattern. + /// The leader signals work readiness, waits for all participants to complete, then the cycle repeats. + /// Participants wait for work signal, process, signal completion, then wait for reset before next cycle. + /// + public sealed class LeaderFollowerBarrier + { + readonly int participantCount; + readonly SemaphoreSlim workReady = new(0); + readonly SemaphoreSlim workCompleted = new(0); + readonly SemaphoreSlim resetReady = new(0); + + /// + /// Initializes a new instance of the class. + /// + /// Number of participant tasks that will process work. + public LeaderFollowerBarrier(int participantCount) + { + ArgumentOutOfRangeException.ThrowIfLessThan(participantCount, 1); + this.participantCount = participantCount; + } + + static TimeSpan ProcessTimeSpan(TimeSpan timeout) + => timeout == default ? Timeout.InfiniteTimeSpan : timeout; + + /// + /// Leader: Waits for all participants to complete, then resets for next cycle. + /// + public bool WaitCompleted(TimeSpan timeout = default, CancellationToken cancellationToken = default) + { + var waitTimeout = ProcessTimeSpan(timeout); + for (var i = 0; i < participantCount; i++) + { + if (!AsyncUtils.BlockingWait(workCompleted.WaitAsync(waitTimeout, cancellationToken))) + return false; + } + + return true; + } + + /// + /// Leader: Release participants that are waiting inside + /// so they can proceed to the next cycle. + /// + public void Release() => resetReady.Release(participantCount); + + /// + /// Participant: Waits for work signal from leader. + /// + public async Task WaitReadyWorkAsync(CancellationToken cancellationToken = default) + { + await workReady.WaitAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Leader: Signals all participants that work is ready. + /// + public void SignalWorkReady() + { + workReady.Release(participantCount); + } + + /// + /// Participant: Signals completion and waits for leader to reset. + /// + public void SignalCompleted(CancellationToken cancellationToken = default) + { + workCompleted.Release(); + resetReady.Wait(cancellationToken); + } + } +} \ No newline at end of file diff --git a/libs/common/ReadOptimizedLock.cs b/libs/common/Synchronization/ReadOptimizedLock.cs similarity index 92% rename from libs/common/ReadOptimizedLock.cs rename to libs/common/Synchronization/ReadOptimizedLock.cs index 2d3e76b8861..a91ad94725a 100644 --- a/libs/common/ReadOptimizedLock.cs +++ b/libs/common/Synchronization/ReadOptimizedLock.cs @@ -125,7 +125,7 @@ public readonly int CalculateIndex(long hashLong, int currentProcessorHint) { // Throw away half the top half of the hash // - // This set of locks will be small enough that the extra bits shoulnd't matter + // This set of locks will be small enough that the extra bits shouldn't matter var hash = (int)hashLong; // Hint might be out of range, so force it into the space we expect @@ -308,6 +308,43 @@ public readonly void ReleaseExclusiveLock(int lockToken) } } + /// + /// Acquire an exclusive lock for all possible hashes, blocking until that succeeds. + /// + /// Will block all other locks until released. + /// + public readonly void AcquireAllExclusiveLock() + { + for (var i = 0; i < lockCounts.Length; i++) + { + ref var acquireRef = ref lockCounts[i]; + + while (Interlocked.CompareExchange(ref acquireRef, int.MinValue, 0) != 0) + { + // Optimistic shared lock got us, or conflict with some other excluive lock acquisition + // + // Backoff and try again + _ = Thread.Yield(); + } + } + } + + /// + /// Release a lock previously acquired with . + /// + public readonly void ReleaseAllExclusiveLock() + { + for (var i = 0; i < lockCounts.Length; i++) + { + ref var releaseRef = ref lockCounts[i]; + while (Interlocked.CompareExchange(ref releaseRef, 0, int.MinValue) != int.MinValue) + { + // Optimistic shared lock got us, back off and try again + _ = Thread.Yield(); + } + } + } + /// /// Attempt to promote a shared lock previously acquired via or to an exclusive lock. /// diff --git a/libs/common/SingleWriterMultiReaderLock.cs b/libs/common/Synchronization/SingleWriterMultiReaderLock.cs similarity index 100% rename from libs/common/SingleWriterMultiReaderLock.cs rename to libs/common/Synchronization/SingleWriterMultiReaderLock.cs diff --git a/libs/common/ExceptionInjectionHelper.cs b/libs/common/Testing/ExceptionInjectionHelper.cs similarity index 71% rename from libs/common/ExceptionInjectionHelper.cs rename to libs/common/Testing/ExceptionInjectionHelper.cs index 1bea8428f96..0d215bfb035 100644 --- a/libs/common/ExceptionInjectionHelper.cs +++ b/libs/common/Testing/ExceptionInjectionHelper.cs @@ -14,6 +14,9 @@ namespace Garnet.common /// public static class ExceptionInjectionHelper { + static object @lock = new(); + static TaskCompletionSource update = new(TaskCreationOptions.RunContinuationsAsynchronously); + /// /// Array of exception injection types /// @@ -40,6 +43,15 @@ public static void EnableException(ExceptionInjectionType exceptionType) } ExceptionInjectionTypes[(int)exceptionType] = true; + + TaskCompletionSource release; + + lock (@lock) + { + release = update; + update = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + _ = release.TrySetResult(true); } /// @@ -47,7 +59,18 @@ public static void EnableException(ExceptionInjectionType exceptionType) /// /// [Conditional("DEBUG")] - public static void DisableException(ExceptionInjectionType exceptionType) => ExceptionInjectionTypes[(int)exceptionType] = false; + public static void DisableException(ExceptionInjectionType exceptionType) + { + ExceptionInjectionTypes[(int)exceptionType] = false; + TaskCompletionSource release; + + lock (@lock) + { + release = update; + update = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + _ = release.TrySetResult(true); + } /// /// Trigger exception scenario (NOTE: add this to the location where the exception should be emulated/triggered) @@ -90,7 +113,7 @@ public static bool TriggerCondition(ExceptionInjectionType exceptionType) /// /// /// - public static async Task WaitOnSetAsync(ExceptionInjectionType exceptionType) + public static async Task ResetAndWaitAsync(ExceptionInjectionType exceptionType) { if (exceptionType == ExceptionInjectionType.None) { @@ -99,10 +122,19 @@ public static async Task WaitOnSetAsync(ExceptionInjectionType exceptionType) if (IsEnabled(exceptionType)) { - // Reset and wait to signaled to go forward + // Reset and wait to be signaled to go forward DisableException(exceptionType); while (!IsEnabled(exceptionType)) - await Task.Yield(); + { + Task task; + lock (@lock) + { + if (IsEnabled(exceptionType)) + break; + task = update.Task; + } + await task.ConfigureAwait(false); + } } } @@ -124,8 +156,17 @@ public static void WaitOnClear(ExceptionInjectionType exceptionType) /// public static async Task WaitOnClearAsync(ExceptionInjectionType exceptionType) { - while (ExceptionInjectionTypes[(int)exceptionType]) - await Task.Yield(); + while (IsEnabled(exceptionType)) + { + Task task; + lock (@lock) + { + if (!IsEnabled(exceptionType)) + break; + task = update.Task; + } + await task.ConfigureAwait(false); + } } } } \ No newline at end of file diff --git a/libs/common/ExceptionInjectionType.cs b/libs/common/Testing/ExceptionInjectionType.cs similarity index 94% rename from libs/common/ExceptionInjectionType.cs rename to libs/common/Testing/ExceptionInjectionType.cs index b097e0f6f9e..2d34166eaae 100644 --- a/libs/common/ExceptionInjectionType.cs +++ b/libs/common/Testing/ExceptionInjectionType.cs @@ -82,6 +82,10 @@ public enum ExceptionInjectionType /// VectorSet_Interrupt_Delete_2, /// + /// During deletion of a Vector Set, leaving it partially deleted - at a particular point of execution. + /// + VectorSet_Interrupt_Delete_3, + /// /// Failure after handler registered in activeHandlers but before Start() is called. /// This means no SAEA receive loop is running, so the only cleanup path is public Dispose(). /// diff --git a/libs/common/Testing/GarnetTestLoggingEventType.cs b/libs/common/Testing/GarnetTestLoggingEventType.cs new file mode 100644 index 00000000000..0d2af8df905 --- /dev/null +++ b/libs/common/Testing/GarnetTestLoggingEventType.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Microsoft.Extensions.Logging; + +namespace Garnet.common +{ + public enum GarnetTestLoggingEventType : int + { + LogPrimaryStreamType, + LogRunAofSyncTask + }; + + public struct GarnetTestLoggingEvent + { + public GarnetTestLoggingEventType Type; + public string Message; + + public override string ToString() => $"++<{Type}>++: {Message}"; + } + + public static class LoggingExtensions + { + public static void LogTesting(this ILogger logger, GarnetTestLoggingEvent state) + { + logger?.Log(LogLevel.Critical, + eventId: default, + state: state, + exception: null, + formatter: static (state, _) => $"{state}"); + } + } +} \ No newline at end of file diff --git a/libs/common/VectorElementKey.cs b/libs/common/VectorElementKey.cs new file mode 100644 index 00000000000..4b939f23c6f --- /dev/null +++ b/libs/common/VectorElementKey.cs @@ -0,0 +1,108 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +#if !NET9_0_OR_GREATER +using System.Runtime.InteropServices; +#endif +using Tsavorite.core; + +namespace Garnet.common +{ + /// + /// Key type for Vector Set element data - anything that's hidden in a namespace. + /// + /// Has same constraints as - must be pinned and "fixed" for duration of operation. + /// + /// Always has a namespace. + /// + public readonly +#if NET9_0_OR_GREATER + ref +#endif + struct VectorElementKey : IKey + { +#if !NET9_0_OR_GREATER + private readonly unsafe void* ptr; + private readonly int len; +#endif + + // TODO: When variable length namespaces are supported, this will need to change + private readonly byte namespaceByte; + + /// + public readonly bool IsPinned + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => true; + } + + /// + public readonly bool IsEmpty + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => false; + } + + /// + public readonly ReadOnlySpan KeyBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#if NET9_0_OR_GREATER + get; +#else + get + { + unsafe + { + return new(ptr, len); + } + } +#endif + } + + /// + public readonly bool HasNamespace + { + get => true; + } + + /// + [UnscopedRef] + public readonly ReadOnlySpan NamespaceBytes + { + get + { + return new(in namespaceByte); + } + } + + /// + /// Construct a new . + /// + /// Note that cannot be 0. + /// + public VectorElementKey(byte namespaceByte, ReadOnlySpan key) + { + Debug.Assert(namespaceByte != 0, "Namespace must be non-zero"); + + this.namespaceByte = namespaceByte; + +#if NET9_0_OR_GREATER + KeyBytes = key; +#else + unsafe + { + ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(key)); + } + len = key.Length; +#endif + } + + /// + public override readonly string ToString() => $"ns: {namespaceByte}, {SpanByte.ToShortString(KeyBytes)}"; + } +} \ No newline at end of file diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index b70264e363b..c2e816c32ac 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -60,24 +60,32 @@ internal sealed class Options : ICloneable public ClusterPreferredEndpointType ClusterPreferredEndpointType { get; set; } [MemorySizeValidation] - [Option('m', "memory", Required = false, HelpText = "Total log memory used in bytes (rounds down to power of 2)")] - public string MemorySize { get; set; } + [Option('m', "memory", Required = false, HelpText = "Total main-log memory (inline and heap) to use, in bytes. Does not need to be a power of 2")] + public string LogMemorySize { get; set; } [MemorySizeValidation] - [Option('p', "page", Required = false, HelpText = "Size of each page in bytes (rounds down to power of 2)")] + [Option('p', "page", Required = false, HelpText = "Size of each main-log page in bytes (rounds down to power of 2; minimum 512).")] public string PageSize { get; set; } + [IntRangeValidation(0, MemoryUtils.ArrayMaxLength)] + [Option("pagecount", Required = false, HelpText = "Number of main-log pages (rounds down to power of 2). This allows specifying less pages initially than LogMemorySize divided by PageSize.")] + public int PageCount { get; set; } + [MemorySizeValidation] - [Option('s', "segment", Required = false, HelpText = "Size of each log segment in bytes on disk (rounds down to power of 2)")] + [Option('s', "segment", Required = false, HelpText = "Size of each main-log segment in bytes on disk (rounds down to power of 2)")] public string SegmentSize { get; set; } + [MemorySizeValidation] + [Option("object-log-segment", Required = false, HelpText = "Size of each object-log segment in bytes on disk (rounds down to power of 2)")] + public string ObjectLogSegmentSize { get; set; } + [MemorySizeValidation] [Option('i', "index", Required = false, HelpText = "Start size of hash index in bytes (rounds down to power of 2)")] - public string IndexSize { get; set; } + public string IndexMemorySize { get; set; } [MemorySizeValidation(false)] [Option("index-max-size", Required = false, HelpText = "Max size of hash index in bytes (rounds down to power of 2)")] - public string IndexMaxSize { get; set; } + public string IndexMaxMemorySize { get; set; } [PercentageValidation(false)] [Option("mutable-percent", Required = false, HelpText = "Percentage of log memory that is kept mutable")] @@ -88,56 +96,16 @@ internal sealed class Options : ICloneable public bool? EnableReadCache { get; set; } [MemorySizeValidation] - [Option("readcache-memory", Required = false, HelpText = "Total read cache log memory used in bytes (rounds down to power of 2)")] + [Option("readcache-memory", Required = false, HelpText = "Total readcache-log memory (inline and heap) to use if readcache is enabled, in bytes. Does not need to be a power of 2")] public string ReadCacheMemorySize { get; set; } [MemorySizeValidation] - [Option("readcache-page", Required = false, HelpText = "Size of each read cache page in bytes (rounds down to power of 2)")] + [Option("readcache-page", Required = false, HelpText = "Size of each read cache page in bytes (rounds down to power of 2; minimum 512).")] public string ReadCachePageSize { get; set; } - [MemorySizeValidation(false)] - [Option("obj-heap-memory", Required = false, HelpText = "Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap)")] - public string ObjectStoreHeapMemorySize { get; set; } - - [MemorySizeValidation] - [Option("obj-log-memory", Required = false, HelpText = "Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log)")] - public string ObjectStoreLogMemorySize { get; set; } - - [MemorySizeValidation] - [Option("obj-page", Required = false, HelpText = "Size of each object store page in bytes (rounds down to power of 2)")] - public string ObjectStorePageSize { get; set; } - - [MemorySizeValidation] - [Option("obj-segment", Required = false, HelpText = "Size of each object store log segment in bytes on disk (rounds down to power of 2)")] - public string ObjectStoreSegmentSize { get; set; } - - [MemorySizeValidation] - [Option("obj-index", Required = false, HelpText = "Start size of object store hash index in bytes (rounds down to power of 2)")] - public string ObjectStoreIndexSize { get; set; } - - [MemorySizeValidation(false)] - [Option("obj-index-max-size", Required = false, HelpText = "Max size of object store hash index in bytes (rounds down to power of 2)")] - public string ObjectStoreIndexMaxSize { get; set; } - - [PercentageValidation] - [Option("obj-mutable-percent", Required = false, HelpText = "Percentage of object store log memory that is kept mutable")] - public int ObjectStoreMutablePercent { get; set; } - - [OptionValidation] - [Option("obj-readcache", Required = false, HelpText = "Enables object store read cache for faster access to on-disk records.")] - public bool? EnableObjectStoreReadCache { get; set; } - - [MemorySizeValidation] - [Option("obj-readcache-log-memory", Required = false, HelpText = "Total object store read cache log memory used in bytes (rounds down to power of 2)")] - public string ObjectStoreReadCacheLogMemorySize { get; set; } - - [MemorySizeValidation] - [Option("obj-readcache-page", Required = false, HelpText = "Size of each object store read cache page in bytes (rounds down to power of 2)")] - public string ObjectStoreReadCachePageSize { get; set; } - - [MemorySizeValidation(false)] - [Option("obj-readcache-heap-memory", Required = false, HelpText = "Object store read cache heap memory size in bytes (Sum of size taken up by all object instances in the heap)")] - public string ObjectStoreReadCacheHeapMemorySize { get; set; } + [IntRangeValidation(0, MemoryUtils.ArrayMaxLength)] + [Option("readcache-pagecount", Required = false, HelpText = "Number of readcache-log pages (rounds down to power of 2). This allows specifying less pages initially than ReadCacheMemorySize divided by ReadCachePageSize.")] + public int ReadCachePageCount { get; set; } [OptionValidation] [Option("storage-tier", Required = false, HelpText = "Enable tiering of records (hybrid log) to storage, to support a larger-than-memory store. Use --logdir to specify storage directory.")] @@ -147,10 +115,6 @@ internal sealed class Options : ICloneable [Option("copy-reads-to-tail", Required = false, HelpText = "When records are read from the main store's in-memory immutable region or storage device, copy them to the tail of the log.")] public bool? CopyReadsToTail { get; set; } - [OptionValidation] - [Option("obj-copy-reads-to-tail", Required = false, HelpText = "When records are read from the object store's in-memory immutable region or storage device, copy them to the tail of the log.")] - public bool? ObjectStoreCopyReadsToTail { get; set; } - [LogDirValidation(false, false)] [Option('l', "logdir", Required = false, HelpText = "Storage directory for tiered records (hybrid log), if storage tiering (--storage-tier) is enabled. Uses current directory if unspecified.")] public string LogDir { get; set; } @@ -167,10 +131,6 @@ internal sealed class Options : ICloneable [Option("no-pubsub", Required = false, HelpText = "Disable pub/sub feature on server.")] public bool? DisablePubSub { get; set; } - [OptionValidation] - [Option("incsnap", Required = false, HelpText = "Enable incremental snapshots.")] - public bool? EnableIncrementalSnapshots { get; set; } - [MemorySizeValidation] [Option("pubsub-pagesize", Required = false, HelpText = "Page size of log used for pub/sub (rounds down to power of 2)")] public string PubSubPageSize { get; set; } @@ -240,6 +200,22 @@ internal sealed class Options : ICloneable [Option("aof-page-size", Required = false, HelpText = "Size of each AOF page in bytes(rounds down to power of 2)")] public string AofPageSize { get; set; } + [MemorySizeValidation] + [Option("aof-segment-size", Required = false, HelpText = "Size of each AOF segment (file) in bytes on disk (rounds down to power of 2). This is the granularity at which AOF files are created and truncated.")] + public string AofSegmentSize { get; set; } + + [IntRangeValidation(1, AofAddress.MaxSublogCount, isRequired: false)] + [Option("aof-physical-sublog-count", Required = false, HelpText = "Number of AOF physical sublogs (i.e. TsavoriteLog instances) used (=1 equivalent to the legacy single log implementation >1: sharded log implementation.")] + public int AofPhysicalSublogCount { get; set; } + + [IntRangeValidation(1, 256, isRequired: false)] + [Option("aof-replay-task-count", Required = false, HelpText = "Number of replay tasks per physical sublog at the replica.")] + public int AofReplayTaskCount { get; set; } + + [IntRangeValidation(0, int.MaxValue)] + [Option("aof-tail-witness-freq", Required = false, HelpText = "Polling frequency of the background task responsible for moving time ahead for all physical sublogs (Used only with physical sublog value >1).")] + public int AofTailWitnessFreqMs { get; set; } + [IntRangeValidation(-1, int.MaxValue)] [Option("aof-commit-freq", Required = false, HelpText = "Write ahead logging (append-only file) commit issue frequency in milliseconds. 0 = issue an immediate commit per operation, -1 = manually issue commits using COMMITAOF command")] public int CommitFrequencyMs { get; set; } @@ -256,14 +232,6 @@ internal sealed class Options : ICloneable [Option("aof-size-limit-enforce-frequency", Required = false, HelpText = "Frequency (in secs) of execution of the AutoCheckpointBasedOnAofSizeLimit background task.")] public int AofSizeLimitEnforceFrequencySecs { get; set; } - [IntRangeValidation(0, int.MaxValue)] - [Option("aof-refresh-freq", Required = false, HelpText = "AOF replication (safe tail address) refresh frequency in milliseconds. 0 = auto refresh after every enqueue.")] - public int AofReplicationRefreshFrequencyMs { get; set; } - - [IntRangeValidation(0, int.MaxValue)] - [Option("subscriber-refresh-freq", Required = false, HelpText = "Subscriber (safe tail address) refresh frequency in milliseconds (for pub-sub). 0 = auto refresh after every enqueue.")] - public int SubscriberRefreshFrequencyMs { get; set; } - [IntRangeValidation(0, int.MaxValue)] [Option("compaction-freq", Required = false, HelpText = "Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead)")] public int CompactionFrequencySecs { get; set; } @@ -272,7 +240,7 @@ internal sealed class Options : ICloneable [Option("expired-object-collection-freq", Required = false, HelpText = "Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand.")] public int ExpiredObjectCollectionFrequencySecs { get; set; } - [Option("compaction-type", Required = false, HelpText = "Hybrid log compaction type. Value options: None - no compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss)")] + [Option("compaction-type", Required = false, HelpText = "Hybrid log compaction type. Value options: None - no compaction, Shift - shift begin address without compaction (data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss; recommended for production use), Scan - scan old pages and move live records to tail (no data loss; NOT RECOMMENDED - builds a temporary parallel KV index proportional to the keyspace, causing significant transient memory use; prefer Lookup)")] public LogCompactionType CompactionType { get; set; } [OptionValidation] @@ -283,10 +251,6 @@ internal sealed class Options : ICloneable [Option("compaction-max-segments", Required = false, HelpText = "Number of log segments created on disk before compaction triggers.")] public int CompactionMaxSegments { get; set; } - [IntRangeValidation(0, int.MaxValue)] - [Option("obj-compaction-max-segments", Required = false, HelpText = "Number of object store log segments created on disk before compaction triggers.")] - public int ObjectStoreCompactionMaxSegments { get; set; } - [OptionValidation] [Option("lua", Required = false, HelpText = "Enable Lua scripts on server.")] public bool? EnableLua { get; set; } @@ -425,9 +389,6 @@ internal sealed class Options : ICloneable [Option("checkpoint-throttle-delay", Required = false, HelpText = "Whether and by how much should we throttle the disk IO for checkpoints: -1 - disable throttling; >= 0 - run checkpoint flush in separate task, sleep for specified time after each WriteAsync")] public int CheckpointThrottleFlushDelayMs { get; set; } - [OptionValidation] - [Option("fast-commit", Required = false, HelpText = "Use FastCommit when writing AOF.")] - public bool? EnableFastCommit { get; set; } [IntRangeValidation(0, int.MaxValue)] [Option("fast-commit-throttle", Required = false, HelpText = "Throttle FastCommit to write metadata once every K commits.")] @@ -458,7 +419,7 @@ internal sealed class Options : ICloneable public bool? FastAofTruncate { get; set; } [OptionValidation] - [Option("on-demand-checkpoint", Required = false, HelpText = "Used with main-memory replication model. Take on demand checkpoint to avoid missing data when attaching")] + [Option("on-demand-checkpoint", Required = false, HelpText = "Used with fast-aof-truncate replication model. Take on demand checkpoint to avoid missing data when attaching")] public bool? OnDemandCheckpoint { get; set; } [OptionValidation] @@ -482,7 +443,7 @@ internal sealed class Options : ICloneable public string ReplicaDisklessSyncFullSyncAofThreshold { get; set; } [OptionValidation] - [Option("aof-null-device", Required = false, HelpText = "With main-memory replication, use null device for AOF. Ensures no disk IO, but can cause data loss during replication.")] + [Option("aof-null-device", Required = false, HelpText = "With fast-aof-truncate replication, use null device for AOF. Ensures no disk IO, but can cause data loss during replication.")] public bool? UseAofNullDevice { get; set; } [System.Text.Json.Serialization.JsonIgnore] @@ -535,8 +496,7 @@ internal sealed class Options : ICloneable [DoubleRangeValidation(0, 1)] [Option("reviv-fraction", Required = false, - HelpText = "#: Fraction of mutable in-memory log space, from the highest log address down to the read-only region, that is eligible for revivification." + - " Applies to both main and object store.")] + HelpText = "#: Fraction of mutable in-memory log space, from the highest log address down to the read-only region, that is eligible for revivification.")] public double RevivifiableFraction { get; set; } [OptionValidation] @@ -562,15 +522,9 @@ internal sealed class Options : ICloneable [OptionValidation] [Option("reviv-in-chain-only", Required = false, HelpText = "Revivify tombstoned records in tag chains only (do not use free list)." + - " Cannot be used with reviv-bin-record-sizes or reviv-bin-record-counts. Propagates to object store by default.")] + " Cannot be used with reviv-bin-record-sizes or reviv-bin-record-counts.")] public bool? RevivInChainOnly { get; set; } - [IntRangeValidation(0, int.MaxValue)] - [Option("reviv-obj-bin-record-count", Required = false, - HelpText = "Number of records in the single free record bin for the object store. The Object store has only a single bin, unlike the main store." + - " Ignored unless the main store is using the free record list.")] - public int RevivObjBinRecordCount { get; set; } - [IntRangeValidation(0, int.MaxValue)] [Option("object-scan-count-limit", Required = false, HelpText = "Limit of items to return in one iteration of *SCAN command")] public int ObjectScanCountLimit { get; set; } @@ -599,13 +553,21 @@ internal sealed class Options : ICloneable public bool? ExtensionAllowUnsignedAssemblies { get; set; } [IntRangeValidation(1, int.MaxValue, isRequired: false)] - [Option("index-resize-freq", Required = false, HelpText = "Index resize check frequency in seconds")] + [Option("index-resize-freq", Required = false, HelpText = "Hash-index resize check frequency in seconds")] public int IndexResizeFrequencySecs { get; set; } [IntRangeValidation(1, 100, isRequired: false)] - [Option("index-resize-threshold", Required = false, HelpText = "Overflow bucket count over total index size in percentage to trigger index resize")] + [Option("index-resize-threshold", Required = false, HelpText = "Hash-index Overflow bucket count over total index size in percentage to trigger index resize")] public int IndexResizeThreshold { get; set; } + // ValueOverflowThreshold must be at least 64 bytes and strictly less than PageSize (both after rounding down to the previous power of 2). + // Validated at server-options consumption time; see GarnetServerOptions.ValueOverflowThresholdBytes. Note that we do not have a KeyOverflowThreshold + // because it would complicate the minimum pagesize check that uses ValueOverflowThreshold check at startup; keys are usually small so calculating a + // minimum page size using a large(ish) Key threshold would have spurious errors. We'll defer that rare case to runtime checks. + [MemorySizeValidation(isRequired: false)] + [Option("value-overflow-threshold", Required = false, HelpText = "Max size of a value stored inline in the main-log page (larger values overflow to the heap). Accepts a memory size (e.g. 4k, 1m). Minimum 64 bytes; must be less than PageSize.")] + public string ValueOverflowThreshold { get; set; } + [OptionValidation] [Option("fail-on-recovery-error", Required = false, HelpText = "Server bootup should fail if errors happen during bootup of AOF and checkpointing")] public bool? FailOnRecoveryError { get; set; } @@ -680,6 +642,9 @@ public IEnumerable LuaAllowedFunctions [Option("vector-set-replay-task-count", Required = false, HelpText = "Configure how many replay tasks are used to replay VectorSet operations at the replica (default: 0 uses the machine CPU count)")] public int VectorSetReplayTaskCount { get; set; } + [Option("enable-range-index-preview", Required = false, HelpText = "Enable Range Index (preview) - this feature (and associated RI.* commands) are incomplete, unstable, and subject to change while still in preview")] + public bool EnableRangeIndexPreview { get; set; } + /// /// This property contains all arguments that were not parsed by the command line argument parser /// @@ -746,13 +711,9 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) var useAzureStorage = deviceType == DeviceType.AzureStorage; if (useAzureStorage && string.IsNullOrEmpty(AzureStorageConnectionString) && string.IsNullOrEmpty(AzureStorageServiceUri)) - { throw new InvalidAzureConfiguration("Cannot use AzureStorage device without supplying storage-string or storage-service-uri"); - } if (useAzureStorage && !string.IsNullOrEmpty(AzureStorageConnectionString) && !string.IsNullOrEmpty(AzureStorageServiceUri)) - { throw new InvalidAzureConfiguration("Cannot use AzureStorage device with both storage-string and storage-service-uri"); - } var logDir = LogDir; if (!useAzureStorage && enableStorageTier) logDir = new DirectoryInfo(string.IsNullOrEmpty(logDir) ? "." : logDir).FullName; @@ -817,12 +778,11 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno throw new Exception("Revivification cannot specify RevivifiableFraction without specifying bins."); } - // For backwards compatibility - if (CompactionType == LogCompactionType.ShiftForced) + // Warn users who explicitly opt into Scan compaction about the memory-spike cost. + // Scan builds a temporary parallel KV index proportional to the keyspace; Lookup is the recommended alternative. + if (CompactionType == LogCompactionType.Scan) { - logger?.LogWarning("Compaction type ShiftForced is deprecated. Use Shift instead along with CompactionForceDelete."); - CompactionType = LogCompactionType.Shift; - CompactionForceDelete = true; + logger?.LogWarning("Compaction type Scan builds a temporary parallel KV index proportional to the keyspace, causing significant transient memory use. Use Lookup instead unless you have a specific reason for Scan."); } if (SlowLogThreshold > 0) @@ -831,6 +791,7 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno throw new Exception("SlowLogThreshold must be at least 100 microseconds."); } + if (!EnableAOF.GetValueOrDefault()) { if (!string.IsNullOrEmpty(AofSizeLimit)) @@ -856,33 +817,23 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno ClusterAnnounceEndpoint = clusterAnnounceEndpoint?[0], ClusterAnnounceHostname = ClusterAnnounceHostname, ClusterPreferredEndpointType = ClusterPreferredEndpointType, - MemorySize = MemorySize, + LogMemorySize = LogMemorySize, PageSize = PageSize, + PageCount = PageCount, SegmentSize = SegmentSize, - IndexSize = IndexSize, - IndexMaxSize = IndexMaxSize, + ObjectLogSegmentSize = ObjectLogSegmentSize, + IndexMemorySize = IndexMemorySize, + IndexMaxMemorySize = IndexMaxMemorySize, MutablePercent = MutablePercent, EnableReadCache = EnableReadCache.GetValueOrDefault(), ReadCacheMemorySize = ReadCacheMemorySize, ReadCachePageSize = ReadCachePageSize, - ObjectStoreHeapMemorySize = ObjectStoreHeapMemorySize, - ObjectStoreLogMemorySize = ObjectStoreLogMemorySize, - ObjectStorePageSize = ObjectStorePageSize, - ObjectStoreSegmentSize = ObjectStoreSegmentSize, - ObjectStoreIndexSize = ObjectStoreIndexSize, - ObjectStoreIndexMaxSize = ObjectStoreIndexMaxSize, - ObjectStoreMutablePercent = ObjectStoreMutablePercent, - EnableObjectStoreReadCache = EnableObjectStoreReadCache.GetValueOrDefault(), - ObjectStoreReadCachePageSize = ObjectStoreReadCachePageSize, - ObjectStoreReadCacheLogMemorySize = ObjectStoreReadCacheLogMemorySize, - ObjectStoreReadCacheHeapMemorySize = ObjectStoreReadCacheHeapMemorySize, + ReadCachePageCount = ReadCachePageCount, EnableStorageTier = enableStorageTier, CopyReadsToTail = CopyReadsToTail.GetValueOrDefault(), - ObjectStoreCopyReadsToTail = ObjectStoreCopyReadsToTail.GetValueOrDefault(), LogDir = logDir, CheckpointDir = checkpointDir, Recover = Recover.GetValueOrDefault(), - EnableIncrementalSnapshots = EnableIncrementalSnapshots.GetValueOrDefault(), DisablePubSub = DisablePubSub.GetValueOrDefault(), PubSubPageSize = PubSubPageSize, DisableObjects = DisableObjects.GetValueOrDefault(), @@ -896,7 +847,10 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno LuaTransactionMode = LuaTransactionMode.GetValueOrDefault(), AofMemorySize = AofMemorySize, AofPageSize = AofPageSize, - AofReplicationRefreshFrequencyMs = AofReplicationRefreshFrequencyMs, + AofSegmentSize = AofSegmentSize, + AofPhysicalSublogCount = AofPhysicalSublogCount, + AofReplayTaskCount = AofReplayTaskCount, + AofTailWitnessFreqMs = AofTailWitnessFreqMs, CommitFrequencyMs = CommitFrequencyMs, WaitForCommit = WaitForCommit.GetValueOrDefault(), AofSizeLimit = AofSizeLimit, @@ -906,12 +860,10 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno CompactionType = CompactionType, CompactionForceDelete = CompactionForceDelete.GetValueOrDefault(), CompactionMaxSegments = CompactionMaxSegments, - ObjectStoreCompactionMaxSegments = ObjectStoreCompactionMaxSegments, GossipSamplePercent = GossipSamplePercent, GossipDelay = GossipDelay, ClusterTimeout = ClusterTimeout, ClusterConfigFlushFrequencyMs = ClusterConfigFlushFrequencyMs, - EnableFastCommit = EnableFastCommit.GetValueOrDefault(), FastCommitThrottleFreq = FastCommitThrottleFreq, NetworkSendThrottleMax = NetworkSendThrottleMax, TlsOptions = EnableTLS.GetValueOrDefault() ? new GarnetTlsOptions( @@ -963,13 +915,13 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno RevivBinBestFitScanLimit = RevivBinBestFitScanLimit, RevivNumberOfBinsToSearch = RevivNumberOfBinsToSearch, RevivInChainOnly = RevivInChainOnly.GetValueOrDefault(), - RevivObjBinRecordCount = RevivObjBinRecordCount, EnableDebugCommand = EnableDebugCommand, EnableModuleCommand = EnableModuleCommand, ExtensionBinPaths = FileUtils.ConvertToAbsolutePaths(ExtensionBinPaths), ExtensionAllowUnsignedAssemblies = ExtensionAllowUnsignedAssemblies.GetValueOrDefault(), IndexResizeFrequencySecs = IndexResizeFrequencySecs, IndexResizeThreshold = IndexResizeThreshold, + ValueOverflowThreshold = ValueOverflowThreshold, LoadModuleCS = LoadModuleCS, FailOnRecoveryError = FailOnRecoveryError.GetValueOrDefault(), LuaOptions = EnableLua.GetValueOrDefault() ? new LuaOptions(LuaMemoryManagementMode, LuaScriptMemoryLimit, LuaScriptTimeoutMs == 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromMilliseconds(LuaScriptTimeoutMs), LuaLoggingMode, LuaAllowedFunctions, logger) : null, @@ -980,7 +932,8 @@ endpoint is IPEndPoint listenEp && clusterAnnounceEndpoint[0] is IPEndPoint anno ClusterReplicationReestablishmentTimeout = ClusterReplicationReestablishmentTimeout, ClusterReplicaResumeWithData = ClusterReplicaResumeWithData, EnableVectorSetPreview = EnableVectorSetPreview, - VectorSetReplayTaskCount = VectorSetReplayTaskCount + VectorSetReplayTaskCount = VectorSetReplayTaskCount, + EnableRangeIndexPreview = EnableRangeIndexPreview, }; } diff --git a/libs/host/Configuration/OptionsValidators.cs b/libs/host/Configuration/OptionsValidators.cs index f865ff477d5..bca4e04ab6e 100644 --- a/libs/host/Configuration/OptionsValidators.cs +++ b/libs/host/Configuration/OptionsValidators.cs @@ -379,14 +379,14 @@ protected override ValidationResult IsValid(object value, ValidationContext vali [AttributeUsage(AttributeTargets.Property)] internal sealed class MemorySizeValidationAttribute : OptionValidationAttribute { - private const string MemorySizePattern = @"^\d+([K|k|M|m|G|g][B|b]{0,1})?$"; + private const string MemorySizePattern = @"^\d+([KkMmGg][Bb]?)?$"; internal MemorySizeValidationAttribute(bool isRequired = true) : base(isRequired) { } /// - /// Memory size validation logic, checks if string matches memory size regex pattern + /// Memory size validation logic, checks if string matches memory size regex pattern. /// /// String containing memory size /// Validation context @@ -396,12 +396,14 @@ protected override ValidationResult IsValid(object value, ValidationContext vali if (TryInitialValidation(value, validationContext, out var initValidationResult, out var memorySize)) return initValidationResult; - if (Regex.IsMatch(memorySize, MemorySizePattern)) - return ValidationResult.Success; + if (!Regex.IsMatch(memorySize, MemorySizePattern)) + { + var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; + var errorMessage = $"{baseError} Expected string in memory size format (e.g. 1k, 1kb, 10m, 10mb, 50g, 50gb etc). Actual value: {memorySize}"; + return new ValidationResult(errorMessage, [validationContext.MemberName]); + } - var baseError = validationContext.MemberName != null ? base.FormatErrorMessage(validationContext.MemberName) : string.Empty; - var errorMessage = $"{baseError} Expected string in memory size format (e.g. 1k, 1kb, 10m, 10mb, 50g, 50gb etc). Actual value: {memorySize}"; - return new ValidationResult(errorMessage, [validationContext.MemberName]); + return ValidationResult.Success; } } @@ -630,7 +632,7 @@ protected override ValidationResult IsValid(object value, ValidationContext vali } /// - /// Validate that, when annotated property is set, another option has a least a minimum memory value. + /// Validate that, when annotated property is set, another option has at least a minimum memory value. /// [AttributeUsage(AttributeTargets.Property)] internal sealed class RequiresMinimumMemory : OptionValidationAttribute diff --git a/libs/host/Configuration/Redis/RedisOptions.cs b/libs/host/Configuration/Redis/RedisOptions.cs index d41d3e63c59..8cd31362d14 100644 --- a/libs/host/Configuration/Redis/RedisOptions.cs +++ b/libs/host/Configuration/Redis/RedisOptions.cs @@ -39,7 +39,7 @@ internal class RedisOptions [RedisOption("port", nameof(Options.Port))] public Option Port { get; set; } - [RedisOption("maxmemory", nameof(Options.MemorySize))] + [RedisOption("maxmemory", nameof(Options.LogMemorySize))] public Option MaxMemory { get; set; } [RedisOption("logfile", nameof(Options.FileLogger))] diff --git a/libs/host/Garnet.host.csproj b/libs/host/Garnet.host.csproj index eb54caca462..e9e5b43dc47 100644 --- a/libs/host/Garnet.host.csproj +++ b/libs/host/Garnet.host.csproj @@ -15,8 +15,8 @@ - - + + @@ -35,6 +35,11 @@ + + + + + @@ -47,6 +52,16 @@ + + + $(DefineConstants);HOST_PROJECT + + + + + + + $(NoWarn);NU5118 diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 4533decc555..68651184064 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -20,12 +20,6 @@ namespace Garnet { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Implementation Garnet server /// @@ -54,7 +48,7 @@ static string GetVersion() private readonly ILoggerFactory loggerFactory; private readonly bool cleanupDir; private bool disposeLoggerFactory; - protected readonly LightEpoch storeEpoch, aofEpoch, pubSubEpoch; + protected readonly LightEpoch storeEpoch, pubSubEpoch; /// /// Store and associated information used by this Garnet server @@ -139,8 +133,6 @@ public GarnetServer(string[] commandLineArgs, ILoggerFactory loggerFactory = nul this.opts.AuthSettings = authenticationSettingsOverride ?? this.opts.AuthSettings; this.cleanupDir = cleanupDir; this.storeEpoch = new LightEpoch(); - if (this.opts.EnableAOF) - this.aofEpoch = new LightEpoch(); if (!this.opts.DisablePubSub) this.pubSubEpoch = new LightEpoch(); this.InitializeServer(); @@ -160,8 +152,6 @@ public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = nul this.loggerFactory = loggerFactory; this.cleanupDir = cleanupDir; this.storeEpoch = new LightEpoch(); - if (this.opts.EnableAOF) - this.aofEpoch = new LightEpoch(); if (!this.opts.DisablePubSub) this.pubSubEpoch = new LightEpoch(); try @@ -171,7 +161,6 @@ public GarnetServer(GarnetServerOptions opts, ILoggerFactory loggerFactory = nul catch { storeEpoch?.Dispose(); - aofEpoch?.Dispose(); pubSubEpoch?.Dispose(); throw; } @@ -199,6 +188,9 @@ private void InitializeServer() var clusterFactory = opts.EnableCluster ? new ClusterFactory() : null; + if (opts.EnableCluster && opts.EnableRangeIndexPreview) + throw new GarnetException("Range Index (preview) is not supported in cluster mode."); + this.logger = this.loggerFactory?.CreateLogger("GarnetServer"); logger?.LogInformation("Garnet {version} {bits} bit; {clusterMode} mode; Endpoint: [{endpoint}]", version, IntPtr.Size == 8 ? "64" : "32", @@ -252,7 +244,7 @@ private void InitializeServer() CreateDatabase(dbId, opts, clusterFactory, customCommandManager); if (!opts.DisablePubSub) - subscribeBroker = new SubscribeBroker(null, opts.PubSubPageSizeBytes(), opts.SubscriberRefreshFrequencyMs, pubSubEpoch, true, logger); + subscribeBroker = new SubscribeBroker(null, opts.PubSubPageSizeBytes(), pubSubEpoch, startFresh: true, logger); logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); @@ -283,13 +275,9 @@ private void InitializeServer() var configMemoryLimit = (storeWrapper.store.IndexSize * 64) + storeWrapper.store.Log.MaxMemorySizeBytes + (storeWrapper.store.ReadCache?.MaxMemorySizeBytes ?? 0) + - (storeWrapper.appendOnlyFile?.MaxMemorySizeBytes ?? 0); - if (storeWrapper.objectStore != null) - configMemoryLimit += (storeWrapper.objectStore.IndexSize * 64) + - storeWrapper.objectStore.Log.MaxMemorySizeBytes + - (storeWrapper.objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + - (storeWrapper.objectStoreSizeTracker?.TargetSize ?? 0) + - (storeWrapper.objectStoreSizeTracker?.ReadCacheTargetSize ?? 0); + (storeWrapper.appendOnlyFile?.Log.MaxMemorySizeBytes.AggregateDiff(0) ?? 0) + + (storeWrapper.sizeTracker?.TargetSize ?? 0) + + (storeWrapper.sizeTracker?.ReadCacheTargetSize ?? 0); logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); } @@ -319,21 +307,40 @@ private void InitializeServer() private GarnetDatabase CreateDatabase(int dbId, GarnetServerOptions serverOptions, ClusterFactory clusterFactory, CustomCommandManager customCommandManager) { - var store = CreateMainStore(dbId, clusterFactory, storeEpoch, out var stateMachineDriver, out var kvSettings); - var objectStore = CreateObjectStore(dbId, clusterFactory, customCommandManager, storeEpoch, stateMachineDriver, out var objectStoreSizeTracker, out var objKvSettings); - var (aofDevice, aof) = CreateAOF(dbId); - + var removeOutdated = !serverOptions.EnableCluster; + // Two-roots layout for RangeIndex files: + // riLogRoot — log-tied (working file + per-flush snapshots), co-located with hlog. + // Falls back through LogDir → CheckpointDir → cwd, mirroring Tsavorite's + // CheckpointBaseDirectory chain so RangeIndex works without storage tier. + // cprDir — checkpoint-tied (per-token snapshots live under /rangeindex/), + // alongside Tsavorite's cpr-checkpoints//info.dat etc. + // Construct the manager only when the feature is enabled. When disabled, the + // store wrapper / triggers / functions hold a null reference, and Tsavorite's + // record-trigger gates (CallOnFlush etc.) return false → zero per-op overhead. + RangeIndexManager rangeIndexManager = null; + if (serverOptions.EnableRangeIndexPreview) + { + var logRootBase = serverOptions.LogDir + ?? serverOptions.CheckpointDir + ?? Directory.GetCurrentDirectory(); + var riLogRoot = Path.Combine(logRootBase ?? string.Empty, "Store", "rangeindex"); + var cprDir = Path.Combine(serverOptions.GetStoreCheckpointDirectory(dbId), "cpr-checkpoints"); + + rangeIndexManager = new RangeIndexManager( + riLogRoot: riLogRoot, cprDir: cprDir, + storeEpoch: storeEpoch, + logger: loggerFactory?.CreateLogger("RangeIndexManager")); + } var vectorManager = new VectorManager( dbId, serverOptions, () => Provider.GetSession(WireFormat.ASCII, null), loggerFactory ); + var store = CreateStore(dbId, clusterFactory, customCommandManager, storeEpoch, rangeIndexManager, vectorManager, out var stateMachineDriver, out var sizeTracker, out var kvSettings); + var aof = CreateAOF(dbId); - return new GarnetDatabase(dbId, store, objectStore, kvSettings, objKvSettings, storeEpoch, stateMachineDriver, objectStoreSizeTracker, - aofDevice, aof, serverOptions.AdjustedIndexMaxCacheLines == 0, - serverOptions.AdjustedObjectStoreIndexMaxCacheLines == 0, - vectorManager); + return new GarnetDatabase(dbId, store, kvSettings, storeEpoch, stateMachineDriver, sizeTracker, aof, serverOptions.AdjustedIndexMaxCacheLines == 0, vectorManager, rangeIndexManager); } private void LoadModules(CustomCommandManager customCommandManager) @@ -359,9 +366,11 @@ private void LoadModules(CustomCommandManager customCommandManager) } } - private TsavoriteKV CreateMainStore(int dbId, IClusterFactory clusterFactory, - LightEpoch epoch, out StateMachineDriver stateMachineDriver, out KVSettings kvSettings) + private TsavoriteKV CreateStore(int dbId, IClusterFactory clusterFactory, CustomCommandManager customCommandManager, + LightEpoch epoch, RangeIndexManager rangeIndexManager, VectorManager vectorManager, out StateMachineDriver stateMachineDriver, out CacheSizeTracker sizeTracker, out KVSettings kvSettings) { + sizeTracker = null; + stateMachineDriver = new StateMachineDriver(epoch, loggerFactory?.CreateLogger($"StateMachineDriver")); kvSettings = opts.GetSettings(loggerFactory, epoch, stateMachineDriver, out logFactory); @@ -369,72 +378,49 @@ private TsavoriteKV // Run checkpoint on its own thread to control p99 kvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - var baseName = opts.GetMainStoreCheckpointDirectory(dbId); + var baseName = opts.GetStoreCheckpointDirectory(dbId); var defaultNamingScheme = new DefaultCheckpointNamingScheme(baseName); kvSettings.CheckpointManager = opts.EnableCluster ? - clusterFactory.CreateCheckpointManager(opts.DeviceFactoryCreator, defaultNamingScheme, isMainStore: true, logger) : - new GarnetCheckpointManager(opts.DeviceFactoryCreator, defaultNamingScheme, removeOutdated: true); - - return new TsavoriteKV(kvSettings - , StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - } - - private TsavoriteKV CreateObjectStore(int dbId, IClusterFactory clusterFactory, CustomCommandManager customCommandManager, - LightEpoch epoch, StateMachineDriver stateMachineDriver, out CacheSizeTracker objectStoreSizeTracker, out KVSettings objKvSettings) - { - objectStoreSizeTracker = null; - objKvSettings = null; - if (opts.DisableObjects) - return null; - - objKvSettings = opts.GetObjectStoreSettings(loggerFactory, epoch, stateMachineDriver, - out var objHeapMemorySize, out var objReadCacheHeapMemorySize); + clusterFactory.CreateCheckpointManager(opts.AofPhysicalSublogCount, opts.DeviceFactoryCreator, defaultNamingScheme, isMainStore: true, logger) : + new GarnetCheckpointManager(opts.AofPhysicalSublogCount, opts.DeviceFactoryCreator, defaultNamingScheme, removeOutdated: true); - // Run checkpoint on its own thread to control p99 - objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; - - var baseName = opts.GetObjectStoreCheckpointDirectory(dbId); - var defaultNamingScheme = new DefaultCheckpointNamingScheme(baseName); - - objKvSettings.CheckpointManager = opts.EnableCluster ? - clusterFactory.CreateCheckpointManager(opts.DeviceFactoryCreator, defaultNamingScheme, isMainStore: false, logger) : - new GarnetCheckpointManager(opts.DeviceFactoryCreator, defaultNamingScheme, removeOutdated: true); + // Create cache size tracker before the store. It will be initialized with the store + // after creation via Initialize() (late-bind to break circular dependency). + var cacheSizeTracker = new CacheSizeTracker(); - var objStore = new TsavoriteKV( - objKvSettings, - StoreFunctions.Create(new ByteArrayKeyComparer(), - () => new ByteArrayBinaryObjectSerializer(), - () => new GarnetObjectSerializer(customCommandManager)), - (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - - if (objHeapMemorySize > 0 || objReadCacheHeapMemorySize > 0) - objectStoreSizeTracker = new CacheSizeTracker(objStore, objKvSettings, objHeapMemorySize, objReadCacheHeapMemorySize, - this.loggerFactory); - - return objStore; + var store = new TsavoriteKV(kvSettings + , Tsavorite.core.StoreFunctions.Create(new GarnetKeyComparer(), + () => new GarnetObjectSerializer(customCommandManager), + new GarnetRecordTriggers(cacheSizeTracker, rangeIndexManager, vectorManager)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + if (kvSettings.LogMemorySize > 0 || kvSettings.ReadCacheMemorySize > 0) + { + cacheSizeTracker.Initialize(store, kvSettings.LogMemorySize, kvSettings.ReadCacheMemorySize, this.loggerFactory); + sizeTracker = cacheSizeTracker; + } + return store; } - private (IDevice, TsavoriteLog) CreateAOF(int dbId) + private GarnetAppendOnlyFile CreateAOF(int dbId) { if (!opts.EnableAOF) { if (opts.CommitFrequencyMs != 0 || opts.WaitForCommit) throw new Exception("Cannot use CommitFrequencyMs or CommitWait without EnableAOF"); - return (null, null); + return null; } if (opts.FastAofTruncate && opts.CommitFrequencyMs != -1) throw new Exception("Need to set CommitFrequencyMs to -1 (manual commits) with FastAofTruncate"); - opts.GetAofSettings(dbId, aofEpoch, out var aofSettings); - var aofDevice = aofSettings.LogDevice; - var appendOnlyFile = new TsavoriteLog(aofSettings, logger: this.loggerFactory?.CreateLogger("TsavoriteLog [aof]")); + opts.GetAofSettings(dbId, out var aofSettings); + var appendOnlyFile = new GarnetAppendOnlyFile(opts, aofSettings, logger: this.loggerFactory?.CreateLogger("GarnetLog [aof]")); + if (opts.CommitFrequencyMs < 0 && opts.WaitForCommit) throw new Exception("Cannot use CommitWait with manual commits"); - return (aofDevice, appendOnlyFile); + return appendOnlyFile; } /// @@ -491,7 +477,6 @@ private void InternalDispose() subscribeBroker?.Dispose(); storeEpoch?.Dispose(); - aofEpoch?.Dispose(); pubSubEpoch?.Dispose(); opts.AuthSettings?.Dispose(); if (disposeLoggerFactory) @@ -506,14 +491,10 @@ private static void DeleteDirectory(string path) try { foreach (string directory in Directory.GetDirectories(path)) - { DeleteDirectory(directory); - } - Directory.Delete(path, true); } - catch (Exception ex) when (ex is IOException || - ex is UnauthorizedAccessException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { try { diff --git a/libs/host/MemoryLogger.cs b/libs/host/MemoryLogger.cs index f5e4efbf73a..3094d6076f7 100644 --- a/libs/host/MemoryLogger.cs +++ b/libs/host/MemoryLogger.cs @@ -15,20 +15,12 @@ internal class MemoryLogger : ILogger { private readonly List<(LogLevel, Exception, string)> _memoryLog = new(); - public IDisposable BeginScope(TState state) - { - return null; - } + public IDisposable BeginScope(TState state) => null; - public bool IsEnabled(LogLevel logLevel) - { - return true; - } + public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) - { - this._memoryLog.Add((logLevel, exception, formatter(state, exception))); - } + => this._memoryLog.Add((logLevel, exception, formatter(state, exception))); /// /// Flushes logger entries into a destination logger. @@ -63,9 +55,6 @@ public void Dispose() internal static class LoggingBuilderExtensions { - public static ILoggingBuilder AddMemory(this ILoggingBuilder builder) - { - return builder.AddProvider(new MemoryLoggerProvider()); - } + public static ILoggingBuilder AddMemory(this ILoggingBuilder builder) => builder.AddProvider(new MemoryLoggerProvider()); } } \ No newline at end of file diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index b331c3a3b4d..83240f98631 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -21,20 +21,26 @@ /* Determines the endpoint type to be advertised to other nodes. (value options: ip, hostname, unknown-endpoint) */ "ClusterPreferredEndpointType" : "ip", - /* Total log memory used in bytes (rounds down to power of 2) */ - "MemorySize" : "16g", + /* Total main-log memory (inline and heap) to use, in bytes. Does not need to be a power of 2 */ + "LogMemorySize" : "16g", - /* Size of each page in bytes (rounds down to power of 2) */ - "PageSize" : "32m", + /* Size of each main-log page in bytes (rounds down to power of 2). Minimum 512 bytes to ensure a worst-case record (header + max inline key + max inline value + optionals + filler + page header) fits within a single page. */ + "PageSize" : "16m", - /* Size of each log segment in bytes on disk (rounds down to power of 2) */ + /* Number of main-log pages (rounds down to power of 2). This allows specifying less pages initially than LogMemorySize divided by PageSize. */ + "PageCount" : 0, + + /* Size of each main-log segment in bytes on disk (rounds down to power of 2) */ "SegmentSize" : "1g", + /* Size of each object-log segment in bytes on disk (rounds down to power of 2) */ + "ObjectLogSegmentSize" : "1g", + /* Start size of hash index in bytes (rounds down to power of 2) */ - "IndexSize" : "128m", + "IndexMemorySize" : "128m", /* Max size of hash index in bytes (rounds down to power of 2) */ - "IndexMaxSize": "", + "IndexMaxMemorySize": "", /* Percentage of log memory that is kept mutable */ "MutablePercent" : 90, @@ -42,54 +48,21 @@ /* Enable read cache for faster access to on-disk records */ "EnableReadCache" : false, - /* Total read cache log memory used in bytes (rounds down to power of 2) */ + /* Total readcache-log memory (inline and heap) to use if readcache is enabled, in bytes. Does not need to be a power of 2 */ "ReadCacheMemorySize" : "1g", - /* Size of each read cache page in bytes (rounds down to power of 2) */ - "ReadCachePageSize" : "32m", - - /* Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap) */ - "ObjectStoreHeapMemorySize" : "", - - /* Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log) */ - "ObjectStoreLogMemorySize" : "32m", - - /* Size of each object store page in bytes (rounds down to power of 2) */ - "ObjectStorePageSize" : "4k", - - /* Size of each object store log segment in bytes on disk (rounds down to power of 2) */ - "ObjectStoreSegmentSize" : "32m", - - /* Start size of object store hash index in bytes (rounds down to power of 2) */ - "ObjectStoreIndexSize" : "16m", - - /* Max size of object store hash index in bytes (rounds down to power of 2) */ - "ObjectStoreIndexMaxSize": "", - - /* Percentage of object store log memory that is kept mutable */ - "ObjectStoreMutablePercent" : 90, - - /* Enables object store read cache for faster access to on-disk records */ - "EnableObjectStoreReadCache" : false, - - /* Total object store read cache log memory used in bytes (rounds down to power of 2) */ - "ObjectStoreReadCacheLogMemorySize" : "32m", + /* Size of each read cache page in bytes (rounds down to power of 2). Minimum 512 bytes (same record-fit constraint as PageSize). */ + "ReadCachePageSize" : "4m", - /* Size of each object store read cache page in bytes (rounds down to power of 2) */ - "ObjectStoreReadCachePageSize" : "1m", - - /* Object store read cache heap memory size in bytes (Sum of size taken up by all object instances in the heap) */ - "ObjectStoreReadCacheHeapMemorySize" : "", + /* Number of readcache-log pages (rounds down to power of 2). This allows specifying less pages initially than ReadCacheMemorySize divided by ReadCachePageSize. */ + "ReadCachePageCount" : 0, /* Enable tiering of records (hybrid log) to storage, to support a larger-than-memory store. Use --logdir to specify storage directory. */ "EnableStorageTier" : false, - /* When records are read from the main store's in-memory immutable region or storage device, copy them to the tail of the log. */ + /* When records are read from the main store''s in-memory immutable region or storage device, copy them to the tail of the log. */ "CopyReadsToTail" : false, - /* When records are read from the object store's in-memory immutable region or storage device, copy them to the tail of the log. */ - "ObjectStoreCopyReadsToTail" : false, - /* Storage directory for tiered records (hybrid log), if storage tiering (--storage) is enabled. Uses current directory if unspecified. */ "LogDir" : null, @@ -102,9 +75,6 @@ /* Disable pub/sub feature on server. */ "DisablePubSub" : false, - /* Enable incremental snapshots. */ - "EnableIncrementalSnapshots" : false, - /* Page size of log used for pub/sub (rounds down to power of 2) */ "PubSubPageSize" : "4k", @@ -162,11 +132,17 @@ /* Size of each AOF page in bytes(rounds down to power of 2) */ "AofPageSize" : "4m", - /* AOF replication (safe tail address) refresh frequency in milliseconds. 0 = auto refresh after every enqueue. */ - "AofReplicationRefreshFrequencyMs": 10, + /* Size of each AOF segment (file) in bytes on disk (rounds down to power of 2). This is the granularity at which AOF files are created and truncated. */ + "AofSegmentSize" : "1g", + + /* Number of AOF physical sublogs (i.e. TsavoriteLog instances) used (=1 equivalent to the legacy single log implementation >1: sharded log implementation. */ + "AofPhysicalSublogCount": 1, + + /* Number of replay tasks per physical sublog at the replica. */ + "AofReplayTaskCount": 1, - /* Subscriber (safe tail address) refresh frequency in milliseconds (for pub-sub). 0 = auto refresh after every enqueue. */ - "SubscriberRefreshFrequencyMs": 0, + /* Polling frequency of the background task responsible for moving time ahead for all physical sublogs (Used only with physical sublog value >1). */ + "AofTailWitnessFreqMs": 10, /* Write ahead logging (append-only file) commit issue frequency in milliseconds. 0 = issue an immediate commit per operation, -1 = manually issue commits using COMMITAOF command */ "CommitFrequencyMs" : 0, @@ -189,8 +165,8 @@ /* Hybrid log compaction type. Value options: */ /* None - no compaction */ /* Shift - shift begin address without compaction (data loss) */ - /* Scan - scan old pages and move live records to tail (no data loss) */ - /* Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss) */ + /* Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss). Recommended for production use. */ + /* Scan - scan old pages and move live records to tail (no data loss). NOT RECOMMENDED: this strategy builds a temporary parallel KV index proportional to the keyspace, causing significant transient memory use. Prefer Lookup. */ "CompactionType" : "None", /* Forcefully delete the inactive segments immediately after the compaction strategy (type) is applied. */ @@ -200,9 +176,6 @@ /* Number of log segments created on disk before compaction triggers. */ "CompactionMaxSegments" : 32, - /* Number of object store log segments created on disk before compaction triggers. */ - "ObjectStoreCompactionMaxSegments" : 32, - /* Enable Lua scripts on server. */ "EnableLua" : false, @@ -311,9 +284,6 @@ /* Whether and by how much should we throttle the disk IO for checkpoints: -1 - disable throttling; >= 0 - run checkpoint flush in separate task, sleep for specified time after each WriteAsync */ "CheckpointThrottleFlushDelayMs" : 0, - /* Use FastCommit when writing AOF. */ - "EnableFastCommit" : true, - /* Throttle FastCommit to write metadata once every K commits. */ "FastCommitThrottleFreq" : 1000, @@ -389,9 +359,6 @@ /* Revivify tombstoned records in tag chains only (do not use free list). Cannot be used with reviv-bin-record-sizes or reviv-bin-record-counts. Propagates to object store by default. */ "RevivInChainOnly" : false, - /* Number of records in the single free record bin for the object store. The Object store has only a single bin, unlike the main store. Ignored unless the main store is using the free record list. */ - "RevivObjBinRecordCount" : 256, - /* Limit of items to return in one iteration of *SCAN command */ "ObjectScanCountLimit" : 1000, @@ -416,6 +383,9 @@ /* Overflow bucket count over total index size in percentage to trigger index resize */ "IndexResizeThreshold": 50, + /* Max size of a value stored inline in the main-log page (larger values overflow to the heap). Accepts a memory size (e.g. "4k", "1m"). Minimum 64 bytes; must be less than PageSize. */ + "ValueOverflowThreshold": "16k", + /* List of module paths to be loaded at startup */ "LoadModuleCS": null, @@ -458,6 +428,9 @@ /* Enable Vector Sets (preview) - this feature (and associated commands) are incomplete, unstable, and subject to change while still in preview */ "EnableVectorSetPreview": false, - /* Configure how many replay tasks are used to replay VectorSet operations at the replica (default: 0 uses the machine CPU count) */ - "VectorSetReplayTaskCount": 0 +/* Configure how many replay tasks are used to replay VectorSet operations at the replica (default: 0 uses the machine CPU count) */ + "VectorSetReplayTaskCount": 0, + + /* Enable Range Index (preview) - this feature (and associated RI.* commands) are incomplete, unstable, and subject to change while still in preview */ + "EnableRangeIndexPreview": false } \ No newline at end of file diff --git a/libs/native/bftree-garnet/.gitignore b/libs/native/bftree-garnet/.gitignore new file mode 100644 index 00000000000..8e0729cc633 --- /dev/null +++ b/libs/native/bftree-garnet/.gitignore @@ -0,0 +1,2 @@ +# Rust build artifacts +target/ diff --git a/libs/native/bftree-garnet/BfTreeInterop.csproj b/libs/native/bftree-garnet/BfTreeInterop.csproj new file mode 100644 index 00000000000..baffea35e8b --- /dev/null +++ b/libs/native/bftree-garnet/BfTreeInterop.csproj @@ -0,0 +1,98 @@ + + + + true + Garnet.server.BfTreeInterop + true + ../../../Garnet.snk + false + + + + + + + + + $(MSBuildThisFileDirectory)target/release/libbftree_garnet.so + $(MSBuildThisFileDirectory)runtimes/linux-x64/native/libbftree_garnet.so + + + $(MSBuildThisFileDirectory)target\release\bftree_garnet.dll + $(MSBuildThisFileDirectory)runtimes\win-x64\native\bftree_garnet.dll + + + $(MSBuildThisFileDirectory)target/release/libbftree_garnet.dylib + $(MSBuildThisFileDirectory)runtimes/osx-x64/native/libbftree_garnet.dylib + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/libs/native/bftree-garnet/BfTreeService.cs b/libs/native/bftree-garnet/BfTreeService.cs new file mode 100644 index 00000000000..a4af6a24120 --- /dev/null +++ b/libs/native/bftree-garnet/BfTreeService.cs @@ -0,0 +1,629 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using Tsavorite.core; + +namespace Garnet.server.BfTreeInterop +{ + /// + /// Result codes for BfTree read operations. + /// + public enum BfTreeReadResult + { + /// Value was found. + Found = 0, + /// Key was not found. + NotFound = -1, + /// Key was found but has been deleted. + Deleted = -2, + /// The key is invalid (e.g. too long). + InvalidKey = -3, + } + + /// + /// Result codes for BfTree insert operations. + /// + public enum BfTreeInsertResult + { + /// Insert succeeded. + Success = 0, + /// Key or value is invalid (e.g. exceeds configured limits). + InvalidKV = 1, + } + + /// + /// Specifies which fields a scan operation should return. + /// + public enum ScanReturnField : byte + { + /// Return only keys. + Key = 0, + /// Return only values. + Value = 1, + /// Return both keys and values. + KeyAndValue = 2, + } + + /// + /// Storage backend for the BfTree. + /// + public enum StorageBackendType : byte + { + /// + /// Disk-backed tree (default). Base pages are stored in a data file on disk. + /// The circular buffer acts as a hot-data cache. No data loss on + /// eviction. Total capacity is limited by disk space. + /// + Disk = 0, + + /// + /// Memory-only tree (maps to bf-tree's cache_only mode). All data lives + /// in a bounded in-memory circular buffer. Snapshot and recovery will be + /// supported in a future bf-tree release; currently throws at the FFI + /// boundary. + /// + Memory = 1, + } + + /// + /// Callback for zero-allocation scan. Receives key and value as spans into the scan buffer. + /// + /// Key bytes (empty if ScanReturnField.Value). + /// Value bytes (empty if ScanReturnField.Key). + /// True to continue scanning, false to stop early. + public delegate bool ScanRecordAction(ReadOnlySpan key, ReadOnlySpan value); + + /// + /// A single record returned by a scan operation. + /// + public readonly struct ScanRecord + { + /// The key bytes (empty if ScanReturnField.Value was used). + public ReadOnlyMemory Key { get; init; } + + /// The value bytes (empty if ScanReturnField.Key was used). + public ReadOnlyMemory Value { get; init; } + } + + /// + /// High-level managed wrapper for the native bftree-garnet library. + /// Provides safe C# access to BfTree lifecycle, point operations, scans, + /// and CPR snapshot/recovery. + /// + public sealed unsafe class BfTreeService : IDisposable + { + private nint _tree; + private int _disposed; + private readonly StorageBackendType _storageBackend; + private readonly string _filePath; + private readonly string _snapshotFilePath; + + /// + /// Gets the native tree pointer for storage in stubs and direct P/Invoke. + /// + public nint NativePtr => _tree; + + /// + /// Gets the data file path for disk-backed trees, or null for memory-only trees. + /// + public string FilePath => _filePath; + + /// + /// Gets the storage backend type (Disk or Memory). + /// + public StorageBackendType StorageBackend => _storageBackend; + + /// + /// Gets the CPR snapshot scratch file path configured at construction. Null if the + /// tree was created without snapshot support. + /// + public string SnapshotFilePath => _snapshotFilePath; + + /// + /// Creates a new BfTree with the given configuration. + /// Pass 0 for any numeric parameter to use the bf-tree default. + /// + /// Disk (default, file-backed) or Memory (bounded in-memory). + /// Data file path for disk-backed trees. Ignored for memory-only. + /// Scratch path for CPR snapshot output. Required if + /// will be called later. Null disables snapshots (legacy behavior). + /// Circular buffer size in bytes (hot-data cache for Disk; total capacity for Memory). + /// Minimum record size. + /// Maximum record size. + /// Maximum key length. + /// Leaf page size. + public BfTreeService( + StorageBackendType storageBackend = StorageBackendType.Disk, + string filePath = null, + string snapshotFilePath = null, + ulong cbSizeByte = 0, + uint cbMinRecordSize = 0, + uint cbMaxRecordSize = 0, + uint cbMaxKeyLen = 0, + uint leafPageSize = 0) + { + _storageBackend = storageBackend; + _filePath = filePath; + _snapshotFilePath = snapshotFilePath; + if (storageBackend == StorageBackendType.Disk && string.IsNullOrEmpty(filePath)) + throw new ArgumentException("filePath is required for disk-backed trees.", nameof(filePath)); + byte[] pathBytes = filePath != null ? Encoding.UTF8.GetBytes(filePath) : null; + byte[] snapBytes = snapshotFilePath != null ? Encoding.UTF8.GetBytes(snapshotFilePath) : null; + fixed (byte* pp = pathBytes) + fixed (byte* sp = snapBytes) + { + _tree = NativeBfTreeMethods.bftree_create( + cbSizeByte, cbMinRecordSize, cbMaxRecordSize, cbMaxKeyLen, leafPageSize, + (byte)storageBackend, pp, pathBytes?.Length ?? 0, + sp, snapBytes?.Length ?? 0); + } + if (_tree == 0) + throw new InvalidOperationException("Failed to create BfTree instance."); + } + + /// + /// Creates a BfTreeService wrapping an existing native tree pointer (e.g. from snapshot restore). + /// Takes ownership of the pointer. + /// + internal BfTreeService(nint treePtr, StorageBackendType storageBackend, string filePath = null, string snapshotFilePath = null) + { + if (treePtr == 0) + throw new ArgumentException("Tree pointer must not be null.", nameof(treePtr)); + _tree = treePtr; + _storageBackend = storageBackend; + _filePath = filePath; + _snapshotFilePath = snapshotFilePath; + } + + // --------------------------------------------------------------- + // Point operations — PinnedSpanByte (zero-overhead for Garnet hot paths) + // --------------------------------------------------------------- + + /// + /// Insert a key-value pair. Zero-overhead: passes pinned pointers directly to native code. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BfTreeInsertResult Insert(PinnedSpanByte key, PinnedSpanByte value) + { + return (BfTreeInsertResult)NativeBfTreeMethods.bftree_insert( + _tree, key.ToPointer(), key.Length, value.ToPointer(), value.Length); + } + + /// + /// Read the value for a key into a pinned output buffer. Zero-overhead. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BfTreeReadResult Read(PinnedSpanByte key, byte* outputBuffer, int outputBufferLen, out int bytesWritten) + { + int valueLen = 0; + var result = NativeBfTreeMethods.bftree_read( + _tree, key.ToPointer(), key.Length, outputBuffer, outputBufferLen, &valueLen); + bytesWritten = valueLen; + return (BfTreeReadResult)result; + } + + /// + /// Delete a key. Zero-overhead. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Delete(PinnedSpanByte key) + { + NativeBfTreeMethods.bftree_delete(_tree, key.ToPointer(), key.Length); + } + + /// + /// No-op P/Invoke for measuring pure FFI transition overhead. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Noop(PinnedSpanByte key) + { + return NativeBfTreeMethods.bftree_noop(_tree, key.ToPointer(), key.Length); + } + + // --------------------------------------------------------------- + // Static pointer-based operations (for hot paths using native ptr from stub) + // --------------------------------------------------------------- + + /// + /// Insert via native pointer. For hot-path use when the caller has the native ptr from the stub. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BfTreeInsertResult InsertByPtr(nint treePtr, PinnedSpanByte key, PinnedSpanByte value) + { + return (BfTreeInsertResult)NativeBfTreeMethods.bftree_insert( + treePtr, key.ToPointer(), key.Length, value.ToPointer(), value.Length); + } + + /// + /// Read via native pointer. Convenience overload that allocates output. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BfTreeReadResult ReadByPtr(nint treePtr, PinnedSpanByte key, out byte[] value) + { + value = []; + Span buffer = stackalloc byte[4096]; + int bytesWritten; + fixed (byte* bp = buffer) + { + int valueLen = 0; + var rc = NativeBfTreeMethods.bftree_read( + treePtr, key.ToPointer(), key.Length, bp, buffer.Length, &valueLen); + bytesWritten = valueLen; + if (rc == (int)BfTreeReadResult.Found && bytesWritten > 0) + { + value = buffer[..bytesWritten].ToArray(); + return BfTreeReadResult.Found; + } + return (BfTreeReadResult)rc; + } + } + + /// + /// Read via native pointer into a caller-provided buffer without allocating. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static BfTreeReadResult ReadByPtrInto(nint treePtr, PinnedSpanByte key, byte* outputBuffer, int outputBufferLen, out int bytesWritten) + { + int valueLen = 0; + var rc = NativeBfTreeMethods.bftree_read( + treePtr, key.ToPointer(), key.Length, outputBuffer, outputBufferLen, &valueLen); + bytesWritten = valueLen; + return (BfTreeReadResult)rc; + } + + /// + /// Delete via native pointer. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void DeleteByPtr(nint treePtr, PinnedSpanByte key) + { + NativeBfTreeMethods.bftree_delete(treePtr, key.ToPointer(), key.Length); + } + + /// + /// Scan with count via native pointer using a zero-allocation callback. + /// + /// Number of records passed to the callback. + public static int ScanWithCountByPtrCallback(nint treePtr, ReadOnlySpan startKey, int count, ScanReturnField returnField, ScanRecordAction onRecord) + { + nint handle; + fixed (byte* skp = startKey) + { + handle = NativeBfTreeMethods.bftree_scan_with_count( + treePtr, skp, startKey.Length, count, (byte)returnField); + } + try + { + Span buffer = stackalloc byte[8192]; + return DrainScanIteratorWithCallback(handle, buffer, returnField, onRecord); + } + finally + { + NativeBfTreeMethods.bftree_scan_drop(handle); + } + } + + /// + /// Scan with end key via native pointer using a zero-allocation callback. + /// + /// Number of records passed to the callback. + public static int ScanWithEndKeyByPtrCallback(nint treePtr, ReadOnlySpan startKey, ReadOnlySpan endKey, ScanReturnField returnField, ScanRecordAction onRecord) + { + nint handle; + fixed (byte* skp = startKey, ekp = endKey) + { + handle = NativeBfTreeMethods.bftree_scan_with_end_key( + treePtr, skp, startKey.Length, ekp, endKey.Length, (byte)returnField); + } + try + { + Span buffer = stackalloc byte[8192]; + return DrainScanIteratorWithCallback(handle, buffer, returnField, onRecord); + } + finally + { + NativeBfTreeMethods.bftree_scan_drop(handle); + } + } + + // --------------------------------------------------------------- + // Point operations — span-based (safe wrappers: fixed → PinnedSpanByte → native) + // --------------------------------------------------------------- + + /// + /// Insert a key-value pair into the BfTree. + /// + public BfTreeInsertResult Insert(ReadOnlySpan key, ReadOnlySpan value) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + fixed (byte* kp = key, vp = value) + return Insert( + PinnedSpanByte.FromPinnedPointer(kp, key.Length), + PinnedSpanByte.FromPinnedPointer(vp, value.Length)); + } + + /// + /// Read the value for a key into a caller-provided buffer. + /// + public BfTreeReadResult Read(ReadOnlySpan key, Span outputBuffer, out int bytesWritten) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + fixed (byte* kp = key, bp = outputBuffer) + return Read( + PinnedSpanByte.FromPinnedPointer(kp, key.Length), + bp, outputBuffer.Length, out bytesWritten); + } + + /// + /// Read the value for a key. Convenience overload that allocates a byte array. + /// For hot paths, prefer the PinnedSpanByte or span overloads. + /// + public BfTreeReadResult Read(ReadOnlySpan key, out byte[] value) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + value = []; + Span buffer = stackalloc byte[4096]; + var result = Read(key, buffer, out int bytesWritten); + if (result == BfTreeReadResult.Found && bytesWritten > 0) + value = buffer[..bytesWritten].ToArray(); + return result; + } + + /// + /// Delete a key from the BfTree. + /// + public void Delete(ReadOnlySpan key) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + fixed (byte* kp = key) + Delete(PinnedSpanByte.FromPinnedPointer(kp, key.Length)); + } + + /// + /// Scan entries starting from , returning up to + /// records. Invokes for each + /// record without allocating per-record. Zero-allocation on the hot path. + /// + /// Key to start scanning from (inclusive). + /// Maximum number of records to return. + /// Caller-provided buffer for scan output (must be large enough for max key+value). + /// Callback invoked for each record with key and value spans into . + /// Which fields to return. + /// Number of records scanned. + public int ScanWithCount( + ReadOnlySpan startKey, int count, + Span scanBuffer, + ScanRecordAction onRecord, + ScanReturnField returnField = ScanReturnField.KeyAndValue) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + nint handle; + fixed (byte* skp = startKey) + { + handle = NativeBfTreeMethods.bftree_scan_with_count( + _tree, skp, startKey.Length, count, (byte)returnField); + } + try + { + return DrainScanIteratorWithCallback(handle, scanBuffer, returnField, onRecord); + } + finally + { + NativeBfTreeMethods.bftree_scan_drop(handle); + } + } + + /// + /// Scan entries starting from , returning up to + /// records. Convenience overload that returns a list. + /// For hot paths, prefer the callback-based overload to avoid per-record allocations. + /// + public List ScanWithCount( + ReadOnlySpan startKey, int count, + ScanReturnField returnField = ScanReturnField.KeyAndValue) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + nint handle; + fixed (byte* skp = startKey) + { + handle = NativeBfTreeMethods.bftree_scan_with_count( + _tree, skp, startKey.Length, count, (byte)returnField); + } + try + { + return DrainScanIteratorToList(handle, returnField); + } + finally + { + NativeBfTreeMethods.bftree_scan_drop(handle); + } + } + + /// + /// Scan entries in the closed range [, ]. + /// + public List ScanWithEndKey( + ReadOnlySpan startKey, ReadOnlySpan endKey, + ScanReturnField returnField = ScanReturnField.KeyAndValue) + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + nint handle; + fixed (byte* skp = startKey, ekp = endKey) + { + handle = NativeBfTreeMethods.bftree_scan_with_end_key( + _tree, skp, startKey.Length, ekp, endKey.Length, (byte)returnField); + } + try + { + return DrainScanIteratorToList(handle, returnField); + } + finally + { + NativeBfTreeMethods.bftree_scan_drop(handle); + } + } + + private static readonly byte[] ScanAllStartKey = [0]; + + /// + /// Scan all entries in the tree, ordered by key. + /// Internally scans from the minimum key (\x00) with count = int.MaxValue. + /// Only supported for disk-backed trees (memory-only trees do not support scan). + /// + public List ScanAll( + ScanReturnField returnField = ScanReturnField.KeyAndValue) + { + return ScanWithCount(ScanAllStartKey, int.MaxValue, returnField); + } + + /// + /// Take a CPR (Concurrent Prefix Recovery) snapshot of this tree. Synchronous; + /// non-blocking to concurrent insert/read/delete callers. Writes the snapshot to + /// the path configured at construction (). + /// + /// To produce snapshots at multiple destination paths, the caller is expected + /// to File.Move / copy the configured snapshot file to the final destination + /// after each call. + /// + /// Internal snapshot_in_progress AtomicBool serializes concurrent calls; + /// losers no-op silently. Callers that need both snapshots to succeed must serialize + /// externally. + /// + public void CprSnapshot() + { + ObjectDisposedException.ThrowIf(_disposed != 0, this); + if (string.IsNullOrEmpty(_snapshotFilePath)) + throw new InvalidOperationException("CprSnapshot requires the tree to be constructed with a snapshotFilePath."); + int result = NativeBfTreeMethods.bftree_cpr_snapshot(_tree); + if (result != 0) + throw new InvalidOperationException("Failed to take CPR snapshot of BfTree."); + } + + /// + /// Take a CPR snapshot of a tree given only its native handle (no managed wrapper). + /// Used by RangeIndex's OnFlush path which has direct access to the stub's + /// TreeHandle but not the managed instance. + /// Snapshot is written to the path configured at the tree's construction time. + /// + /// Native BfTree pointer. + public static void CprSnapshotByPtr(nint handle) + { + if (handle == nint.Zero) + throw new ArgumentException("Native handle is null.", nameof(handle)); + int result = NativeBfTreeMethods.bftree_cpr_snapshot(handle); + if (result != 0) + throw new InvalidOperationException("Failed to take CPR snapshot of BfTree."); + } + + /// + /// Recover a BfTree from a CPR snapshot file. Unified API for disk-backed and + /// memory-backed (cache_only) trees — the storage backend is recorded in the + /// snapshot and inferred by the native library. + /// + /// Source CPR snapshot file path. + /// Scratch path for the recovered tree's future cpr_snapshot + /// calls. Pass null to disable snapshots on the recovered tree (legacy behavior). + /// Storage backend of the recovered tree (for managed tracking). + public static BfTreeService RecoverFromCprSnapshot( + string recoveryPath, + string newSnapshotPath, + StorageBackendType storageBackend) + { + if (string.IsNullOrEmpty(recoveryPath)) + throw new ArgumentException("recoveryPath is required.", nameof(recoveryPath)); + + var recoveryBytes = Encoding.UTF8.GetBytes(recoveryPath); + var newSnapBytes = newSnapshotPath != null ? Encoding.UTF8.GetBytes(newSnapshotPath) : null; + nint treePtr; + fixed (byte* rp = recoveryBytes) + fixed (byte* sp = newSnapBytes) + { + treePtr = NativeBfTreeMethods.bftree_new_from_cpr_snapshot( + rp, recoveryBytes.Length, + sp, newSnapBytes?.Length ?? 0, + null, 0); + } + if (treePtr == 0) + throw new InvalidOperationException($"Failed to recover BfTree from CPR snapshot '{recoveryPath}'."); + return new BfTreeService(treePtr, storageBackend, filePath: null, snapshotFilePath: newSnapshotPath); + } + + /// + /// Drains scan iterator via callback — zero per-record allocation. + /// + private static int DrainScanIteratorWithCallback( + nint handle, Span buffer, ScanReturnField returnField, ScanRecordAction onRecord) + { + int count = 0; + while (true) + { + int keyLen = 0, valueLen = 0; + int hasNext; + fixed (byte* bp = buffer) + hasNext = NativeBfTreeMethods.bftree_scan_next( + handle, bp, buffer.Length, &keyLen, &valueLen); + if (hasNext == 0) + break; + + var key = returnField != ScanReturnField.Value + ? buffer[..keyLen] : ReadOnlySpan.Empty; + var value = returnField != ScanReturnField.Key + ? buffer[keyLen..(keyLen + valueLen)] : ReadOnlySpan.Empty; + + count++; + if (!onRecord(key, value)) + break; + } + return count; + } + + /// + /// Drains scan iterator into a list — convenience, allocates per record. + /// + private static List DrainScanIteratorToList(nint handle, ScanReturnField returnField) + { + var results = new List(); + Span buffer = stackalloc byte[8192]; + while (true) + { + int keyLen = 0, valueLen = 0; + int hasNext; + fixed (byte* bp = buffer) + hasNext = NativeBfTreeMethods.bftree_scan_next( + handle, bp, buffer.Length, &keyLen, &valueLen); + if (hasNext == 0) + break; + + var record = new ScanRecord + { + Key = returnField != ScanReturnField.Value + ? buffer[..keyLen].ToArray() + : ReadOnlyMemory.Empty, + Value = returnField != ScanReturnField.Key + ? buffer[keyLen..(keyLen + valueLen)].ToArray() + : ReadOnlyMemory.Empty, + }; + results.Add(record); + } + return results; + } + + /// + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + if (_tree != 0) + { + NativeBfTreeMethods.bftree_drop(_tree); + _tree = 0; + } + } + } + } +} \ No newline at end of file diff --git a/libs/native/bftree-garnet/Cargo.lock b/libs/native/bftree-garnet/Cargo.lock new file mode 100644 index 00000000000..317c69f8726 --- /dev/null +++ b/libs/native/bftree-garnet/Cargo.lock @@ -0,0 +1,453 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bf-tree" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf5b5c651f5d867445b6509eec523e26ecabc856979fe7979f43d3f8b62286bd" +dependencies = [ + "cfg-if", + "io-uring", + "libc", + "rand 0.8.6", + "rand 0.9.2", + "serde", + "serde_json", + "thread_local", + "toml", + "windows-sys", +] + +[[package]] +name = "bftree-garnet" +version = "0.1.0" +dependencies = [ + "bf-tree", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "io-uring" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595a0399f411a508feb2ec1e970a4a30c249351e30208960d58298de8660b0e5" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/libs/native/bftree-garnet/Cargo.toml b/libs/native/bftree-garnet/Cargo.toml new file mode 100644 index 00000000000..db19b623c0a --- /dev/null +++ b/libs/native/bftree-garnet/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "bftree-garnet" +version = "0.1.0" +edition = "2021" +publish = false +license = "MIT" +description = "C FFI wrapper over bf-tree for Garnet P/Invoke interop" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +bf-tree = "0.5" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 diff --git a/libs/native/bftree-garnet/NativeBfTreeMethods.cs b/libs/native/bftree-garnet/NativeBfTreeMethods.cs new file mode 100644 index 00000000000..8a6caee4ef4 --- /dev/null +++ b/libs/native/bftree-garnet/NativeBfTreeMethods.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.InteropServices; + +namespace Garnet.server.BfTreeInterop +{ + /// + /// P/Invoke declarations for the native bftree-garnet library. + /// Uses source-generated LibraryImport for zero-overhead interop. + /// + internal static unsafe partial class NativeBfTreeMethods + { + private const string LibName = "bftree_garnet"; + + // --------------------------------------------------------------- + // Lifecycle + // --------------------------------------------------------------- + + /// + /// Create a new BfTree. Returns a native pointer, or IntPtr.Zero on failure. + /// Pass 0 for any numeric parameter to use the bf-tree default. + /// storage_backend: 0 = Disk, 1 = Memory. + /// For disk-backed trees, file_path/file_path_len specify the data file path. + /// For in-memory trees, file_path can be null. + /// snapshot_file_path/snapshot_file_path_len configure the CPR snapshot output + /// path (use_snapshot=true if non-empty). Pass null/0 to disable snapshots. + /// + [LibraryImport(LibName)] + internal static partial nint bftree_create( + ulong cb_size_byte, + uint cb_min_record_size, + uint cb_max_record_size, + uint cb_max_key_len, + uint leaf_page_size, + byte storage_backend, + byte* file_path, + int file_path_len, + byte* snapshot_file_path, + int snapshot_file_path_len); + + /// + /// Free a BfTree instance. + /// + [LibraryImport(LibName)] + internal static partial void bftree_drop(nint tree); + + // --------------------------------------------------------------- + // Point operations + // --------------------------------------------------------------- + + /// + /// Insert a key-value pair. Returns 0 on success, 1 on invalid KV. + /// + [LibraryImport(LibName)] + internal static partial int bftree_insert( + nint tree, + byte* key, int key_len, + byte* value, int value_len); + + /// + /// Read the value for a key into out_buffer. + /// Returns 0 (found), -1 (not found), -2 (deleted), -3 (invalid key). + /// On success, out_value_len is set to the number of bytes written. + /// + [LibraryImport(LibName)] + internal static partial int bftree_read( + nint tree, + byte* key, int key_len, + byte* out_buffer, int out_buffer_len, + int* out_value_len); + + /// + /// Delete a key from the tree. + /// + [LibraryImport(LibName)] + internal static partial void bftree_delete( + nint tree, + byte* key, int key_len); + + // --------------------------------------------------------------- + // Scan operations + // --------------------------------------------------------------- + + /// + /// Begin a scan-with-count. Returns an opaque iterator handle. + /// return_field: 0=Key, 1=Value, 2=KeyAndValue. + /// + [LibraryImport(LibName)] + internal static partial nint bftree_scan_with_count( + nint tree, + byte* start_key, int start_key_len, + int count, + byte return_field); + + /// + /// Begin a scan-with-end-key. Returns an opaque iterator handle. + /// + [LibraryImport(LibName)] + internal static partial nint bftree_scan_with_end_key( + nint tree, + byte* start_key, int start_key_len, + byte* end_key, int end_key_len, + byte return_field); + + /// + /// Advance the scan iterator. Returns 1 if a record was produced, 0 if exhausted. + /// + [LibraryImport(LibName)] + internal static partial int bftree_scan_next( + nint handle, + byte* out_buffer, int out_buffer_len, + int* out_key_len, int* out_value_len); + + /// + /// Free a scan iterator handle. + /// + [LibraryImport(LibName)] + internal static partial void bftree_scan_drop(nint handle); + + // --------------------------------------------------------------- + // Snapshot / Recovery (CPR — bftree 0.5+) + // --------------------------------------------------------------- + + /// + /// Take a CPR (Concurrent Prefix Recovery) snapshot of a BfTree. Synchronous; + /// designed to be non-blocking to concurrent insert/read/delete callers. Writes + /// the snapshot to the path configured at tree-creation time via + /// Config::snapshot_file_path / use_snapshot=true. + /// + /// Internal snapshot_in_progress AtomicBool serializes concurrent calls; + /// losers no-op silently. To produce snapshots at multiple destination paths, + /// the caller is expected to File.Move / copy the configured snapshot + /// file to the final destination after each call. + /// + /// Returns 0 on success, -1 on panic. + /// + [LibraryImport(LibName)] + internal static partial int bftree_cpr_snapshot(nint tree); + + /// + /// Recover a BfTree from a CPR snapshot file. Unified for disk-backed and + /// memory-backed (cache_only) trees — the storage backend is recorded in the + /// snapshot. + /// + /// recovery_path: source CPR snapshot file to recover from. + /// new_snapshot_path: scratch path for the recovered tree's future cpr_snapshot + /// calls. Pass null/0 to disable snapshots on the recovered tree. + /// buffer_ptr: optional pre-allocated buffer for the recovered tree's cache. + /// If null, bftree allocates and owns the buffer (freed on tree.Dispose). + /// If non-null, the caller owns the buffer. + /// + /// Returns a native pointer, or IntPtr.Zero on failure. + /// + [LibraryImport(LibName)] + internal static partial nint bftree_new_from_cpr_snapshot( + byte* recovery_path, int recovery_path_len, + byte* new_snapshot_path, int new_snapshot_path_len, + byte* buffer_ptr, nuint buffer_size); + + /// + /// Returns 1 if all threads have moved past the snapshot's version barrier, + /// 0 otherwise, -1 on panic. Useful for assertions/diagnostics. + /// + [LibraryImport(LibName)] + internal static partial int bftree_are_all_threads_in_next_version(nint tree); + + /// + /// No-op for measuring pure FFI transition overhead. + /// + [LibraryImport(LibName)] + internal static partial int bftree_noop( + nint tree, + byte* key, int key_len); + } +} \ No newline at end of file diff --git a/libs/native/bftree-garnet/examples/bench.rs b/libs/native/bftree-garnet/examples/bench.rs new file mode 100644 index 00000000000..61934ed629d --- /dev/null +++ b/libs/native/bftree-garnet/examples/bench.rs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Standalone Rust benchmark for bf-tree operations. +//! Run with: cargo run --release --manifest-path libs/native/bftree-garnet/Cargo.toml --example bench + +use bf_tree::{BfTree, Config, LeafReadResult, StorageBackend}; +use std::time::Instant; + +const ITERATIONS: usize = 2_000_000; +const WARMUP: usize = 10_000; + +fn bench(label: &str, mut f: F) { + for _ in 0..WARMUP { + f(); + } + let start = Instant::now(); + for _ in 0..ITERATIONS { + f(); + } + let ns = start.elapsed().as_nanos() / ITERATIONS as u128; + println!("{label}: {ns} ns/op"); +} + +fn run_benchmarks(label: &str, tree: &BfTree) { + let key = b"bench:key:00000"; + let value = [42u8; 128]; + + // Insert 64 consecutive keys so total data exceeds the base page size + // (~4 KB default). This ensures reads are served from the circular + // buffer cache and disk-backed reads don't hit a cold-page corner case. + for i in 0..64 { + let k = format!("bench:key:{i:05}"); + tree.insert(k.as_bytes(), &value); + } + + // Verify read returns correct data + let mut buf = [0u8; 256]; + match tree.read(key, &mut buf) { + LeafReadResult::Found(n) => { + assert_eq!(n, 128, "Expected 128 bytes, got {n}"); + assert_eq!(&buf[..128], &value, "Read value mismatch"); + } + other => panic!("Expected Found, got {other:?}"), + } + + println!("\n--- {label} ---"); + + bench(&format!("{label} read"), || { + let mut buf = [0u8; 256]; + let _ = tree.read(key, &mut buf); + }); + + bench(&format!("{label} insert"), || { + tree.insert(key, &value); + }); + + bench(&format!("{label} delete"), || { + tree.delete(key); + }); +} + +fn main() { + // Memory-only (cache_only) mode + { + let mut config = Config::default(); + config.cb_min_record_size(8); + config.cache_only(true); + let tree = BfTree::with_config(config, None).unwrap(); + run_benchmarks("Memory", &tree); + } + + // Disk-backed mode + { + let path = "/tmp/bftree_bench_disk.bftree"; + let _ = std::fs::remove_file(path); + let mut config = Config::default(); + config.cb_min_record_size(8); + config.storage_backend(StorageBackend::Std); + config.file_path(path); + let tree = BfTree::with_config(config, None).unwrap(); + run_benchmarks("Disk", &tree); + drop(tree); + let _ = std::fs::remove_file(path); + } +} diff --git a/libs/native/bftree-garnet/runtimes/linux-x64/native/libbftree_garnet.so b/libs/native/bftree-garnet/runtimes/linux-x64/native/libbftree_garnet.so new file mode 100755 index 00000000000..f0023064ae0 Binary files /dev/null and b/libs/native/bftree-garnet/runtimes/linux-x64/native/libbftree_garnet.so differ diff --git a/libs/native/bftree-garnet/runtimes/osx-arm64/native/libbftree_garnet.dylib b/libs/native/bftree-garnet/runtimes/osx-arm64/native/libbftree_garnet.dylib new file mode 100755 index 00000000000..f73677239e0 Binary files /dev/null and b/libs/native/bftree-garnet/runtimes/osx-arm64/native/libbftree_garnet.dylib differ diff --git a/libs/native/bftree-garnet/runtimes/osx-x64/native/libbftree_garnet.dylib b/libs/native/bftree-garnet/runtimes/osx-x64/native/libbftree_garnet.dylib new file mode 100755 index 00000000000..9d515df11ea Binary files /dev/null and b/libs/native/bftree-garnet/runtimes/osx-x64/native/libbftree_garnet.dylib differ diff --git a/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.dll b/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.dll new file mode 100644 index 00000000000..0030def3d59 Binary files /dev/null and b/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.dll differ diff --git a/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.pdb b/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.pdb new file mode 100644 index 00000000000..8f0b18e2f26 Binary files /dev/null and b/libs/native/bftree-garnet/runtimes/win-x64/native/bftree_garnet.pdb differ diff --git a/libs/native/bftree-garnet/rust-toolchain.toml b/libs/native/bftree-garnet/rust-toolchain.toml new file mode 100644 index 00000000000..76a06e6b881 --- /dev/null +++ b/libs/native/bftree-garnet/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.94.0" diff --git a/libs/native/bftree-garnet/src/lib.rs b/libs/native/bftree-garnet/src/lib.rs new file mode 100644 index 00000000000..1f0296be34c --- /dev/null +++ b/libs/native/bftree-garnet/src/lib.rs @@ -0,0 +1,474 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! C FFI wrapper over the bf-tree crate for Garnet P/Invoke interop. +//! +//! Every public function is `#[no_mangle] extern "C"` so it can be called +//! from C# via `[LibraryImport("bftree_garnet")]`. + +use bf_tree::{BfTree, Config, LeafInsertResult, LeafReadResult, ScanIter, ScanReturnField, StorageBackend}; +use std::path::{Path, PathBuf}; +use std::slice; + +// --------------------------------------------------------------------------- +// Result codes returned to C# +// --------------------------------------------------------------------------- + +/// Read result: value was found, the return value is the number of bytes. +const READ_FOUND: i32 = 0; // Actual byte count is in `out_value_len`. +const READ_NOT_FOUND: i32 = -1; +const READ_DELETED: i32 = -2; +const READ_INVALID_KEY: i32 = -3; + +const INSERT_SUCCESS: i32 = 0; +const INSERT_INVALID_KV: i32 = 1; + +// --------------------------------------------------------------------------- +// Storage backend constants (matches C# StorageBackendType enum) +// --------------------------------------------------------------------------- + +/// Disk-backed tree: base pages are stored in a data file. +const _STORAGE_DISK: u8 = 0; +/// Memory-only tree (bf-tree cache_only mode): bounded in-memory circular buffer. +const STORAGE_MEMORY: u8 = 1; + +/// Helper to apply common config fields. +unsafe fn apply_common_config( + config: &mut Config, + cb_size_byte: u64, + cb_min_record_size: u32, + cb_max_record_size: u32, + cb_max_key_len: u32, + leaf_page_size: u32, +) { + if cb_size_byte > 0 { + config.cb_size_byte(cb_size_byte as usize); + } + if cb_min_record_size > 0 { + config.cb_min_record_size(cb_min_record_size as usize); + } + if cb_max_record_size > 0 { + config.cb_max_record_size(cb_max_record_size as usize); + } + if cb_max_key_len > 0 { + config.cb_max_key_len(cb_max_key_len as usize); + } + if leaf_page_size > 0 { + config.leaf_page_size(leaf_page_size as usize); + } +} + +// --------------------------------------------------------------------------- +// Lifecycle +// --------------------------------------------------------------------------- + +/// Create a new BfTree with the given configuration. +/// +/// `storage_backend`: 0 = Disk (file-backed), 1 = Memory (cache_only). +/// For disk-backed trees, `file_path` / `file_path_len` specify the data file. +/// For memory-only trees, `file_path` is ignored. +/// +/// `snapshot_file_path` / `snapshot_file_path_len` configure the CPR snapshot +/// output path (see bftree 0.5 Config::snapshot_file_path / use_snapshot). +/// Required for both backends if `cpr_snapshot` will be called later. May be +/// null/zero-length to disable snapshots (legacy behavior). +/// +/// Returns a pointer to a heap-allocated BfTree, or null on failure. +/// +/// # Safety +/// The caller must eventually call `bftree_drop` to free the returned pointer. +#[no_mangle] +pub unsafe extern "C" fn bftree_create( + cb_size_byte: u64, + cb_min_record_size: u32, + cb_max_record_size: u32, + cb_max_key_len: u32, + leaf_page_size: u32, + storage_backend: u8, + file_path: *const u8, + file_path_len: i32, + snapshot_file_path: *const u8, + snapshot_file_path_len: i32, +) -> *mut BfTree { + let mut config = Config::default(); + apply_common_config( + &mut config, + cb_size_byte, cb_min_record_size, cb_max_record_size, + cb_max_key_len, leaf_page_size, + ); + + if storage_backend == STORAGE_MEMORY { + // Maps to bf-tree's cache_only mode: StorageBackend::Memory + cache_only=true + // Bounded in-memory circular buffer. + config.cache_only(true); + } else { + // STORAGE_DISK (default): file-backed tree. + if file_path.is_null() || file_path_len <= 0 { + return std::ptr::null_mut(); + } + let path_bytes = slice::from_raw_parts(file_path, file_path_len as usize); + let path_str = match std::str::from_utf8(path_bytes) { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + config.storage_backend(StorageBackend::Std); + config.file_path(Path::new(path_str)); + } + + if !snapshot_file_path.is_null() && snapshot_file_path_len > 0 { + let snap_bytes = slice::from_raw_parts(snapshot_file_path, snapshot_file_path_len as usize); + let snap_str = match std::str::from_utf8(snap_bytes) { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + config.snapshot_file_path(PathBuf::from(snap_str)); + config.use_snapshot(true); + } + + match BfTree::with_config(config, None) { + Ok(tree) => Box::into_raw(Box::new(tree)), + Err(_) => std::ptr::null_mut(), + } +} + +/// Drop (free) a BfTree instance. +/// +/// # Safety +/// `tree` must be a valid pointer returned by `bftree_create` and must not be +/// used after this call. +#[no_mangle] +pub unsafe extern "C" fn bftree_drop(tree: *mut BfTree) { + if !tree.is_null() { + drop(Box::from_raw(tree)); + } +} + +// --------------------------------------------------------------------------- +// Point operations +// --------------------------------------------------------------------------- + +/// Insert a key-value pair. Returns INSERT_SUCCESS (0) or INSERT_INVALID_KV (1). +/// +/// # Safety +/// `tree` must be a valid BfTree pointer. `key`/`value` must point to valid +/// memory of the specified lengths. +#[no_mangle] +pub unsafe extern "C" fn bftree_insert( + tree: *mut BfTree, + key: *const u8, + key_len: i32, + value: *const u8, + value_len: i32, +) -> i32 { + let tree = &*tree; + let key = slice::from_raw_parts(key, key_len as usize); + let value = slice::from_raw_parts(value, value_len as usize); + match tree.insert(key, value) { + LeafInsertResult::Success => INSERT_SUCCESS, + LeafInsertResult::InvalidKV(_) => INSERT_INVALID_KV, + } +} + +/// Read the value for a key into `out_buffer`. +/// +/// On success, writes the value bytes into `out_buffer` and sets +/// `*out_value_len` to the number of bytes written. Returns READ_FOUND (0). +/// +/// On failure, returns READ_NOT_FOUND (-1), READ_DELETED (-2), or +/// READ_INVALID_KEY (-3). +/// +/// # Safety +/// All pointer arguments must be valid. `out_buffer` must have at least +/// `out_buffer_len` bytes available. +#[no_mangle] +pub unsafe extern "C" fn bftree_read( + tree: *mut BfTree, + key: *const u8, + key_len: i32, + out_buffer: *mut u8, + out_buffer_len: i32, + out_value_len: *mut i32, +) -> i32 { + let tree = &*tree; + let key = slice::from_raw_parts(key, key_len as usize); + let buffer = slice::from_raw_parts_mut(out_buffer, out_buffer_len as usize); + match tree.read(key, buffer) { + LeafReadResult::Found(n) => { + if !out_value_len.is_null() { + *out_value_len = n as i32; + } + READ_FOUND + } + LeafReadResult::NotFound => READ_NOT_FOUND, + LeafReadResult::Deleted => READ_DELETED, + LeafReadResult::InvalidKey => READ_INVALID_KEY, + } +} + +/// Delete a key from the tree. +/// +/// # Safety +/// `tree` must be a valid BfTree pointer. `key` must point to valid memory. +#[no_mangle] +pub unsafe extern "C" fn bftree_delete( + tree: *mut BfTree, + key: *const u8, + key_len: i32, +) { + let tree = &*tree; + let key = slice::from_raw_parts(key, key_len as usize); + tree.delete(key); +} + +// --------------------------------------------------------------------------- +// Scan operations +// +// Scans are modeled as an opaque iterator that the caller advances one record +// at a time via `bftree_scan_next`, then frees with `bftree_scan_drop`. +// +// Because `ScanIter` borrows the `BfTree`, we box a helper struct that owns +// the necessary references. +// --------------------------------------------------------------------------- + +/// Opaque scan iterator handle. Caller must not interpret the pointer. +pub struct ScanHandle<'a> { + iter: ScanIter<'a, 'a>, +} + +/// Begin a scan-with-count. Returns an opaque iterator handle. +/// +/// `return_field`: 0 = Key, 1 = Value, 2 = KeyAndValue. +/// +/// # Safety +/// `tree` must be a valid BfTree pointer that outlives the returned handle. +/// Caller must free the handle with `bftree_scan_drop`. +#[no_mangle] +pub unsafe extern "C" fn bftree_scan_with_count( + tree: *mut BfTree, + start_key: *const u8, + start_key_len: i32, + count: i32, + return_field: u8, +) -> *mut ScanHandle<'static> { + let tree = &*tree; + let start = slice::from_raw_parts(start_key, start_key_len as usize); + let rf = match return_field { + 0 => ScanReturnField::Key, + 1 => ScanReturnField::Value, + _ => ScanReturnField::KeyAndValue, + }; + let iter = ScanIter::new_with_scan_count(tree, start, count as usize, rf); + // SAFETY: We transmute the lifetime to 'static. The caller is responsible + // for ensuring the BfTree outlives this handle and calling bftree_scan_drop. + let handle = Box::new(ScanHandle { + iter: std::mem::transmute(iter), + }); + Box::into_raw(handle) +} + +/// Begin a scan-with-end-key. Returns an opaque iterator handle. +/// +/// # Safety +/// Same requirements as `bftree_scan_with_count`. +#[no_mangle] +pub unsafe extern "C" fn bftree_scan_with_end_key( + tree: *mut BfTree, + start_key: *const u8, + start_key_len: i32, + end_key: *const u8, + end_key_len: i32, + return_field: u8, +) -> *mut ScanHandle<'static> { + let tree = &*tree; + let start = slice::from_raw_parts(start_key, start_key_len as usize); + let end = slice::from_raw_parts(end_key, end_key_len as usize); + let rf = match return_field { + 0 => ScanReturnField::Key, + 1 => ScanReturnField::Value, + _ => ScanReturnField::KeyAndValue, + }; + let iter = ScanIter::new_with_end_key(tree, start, end, rf); + let handle = Box::new(ScanHandle { + iter: std::mem::transmute(iter), + }); + Box::into_raw(handle) +} + +/// Advance the scan iterator by one record. +/// +/// Writes the record data into `out_buffer` and sets `*out_key_len` and +/// `*out_value_len` to the lengths of the key and value portions within +/// `out_buffer`. +/// +/// Returns 1 if a record was produced, 0 if the scan is exhausted. +/// +/// When `return_field` was Key: `out_buffer[..key_len]` is the key, +/// `out_value_len` is 0. +/// When Value: `out_buffer[..value_len]` is the value, `out_key_len` is 0. +/// When KeyAndValue: `out_buffer[..key_len]` is the key, +/// `out_buffer[key_len..key_len+value_len]` is the value. +/// +/// # Safety +/// `handle` must be a valid ScanHandle pointer. +#[no_mangle] +pub unsafe extern "C" fn bftree_scan_next( + handle: *mut ScanHandle<'static>, + out_buffer: *mut u8, + out_buffer_len: i32, + out_key_len: *mut i32, + out_value_len: *mut i32, +) -> i32 { + let handle = &mut *handle; + let buffer = slice::from_raw_parts_mut(out_buffer, out_buffer_len as usize); + match handle.iter.next(buffer) { + Some((key_len, value_len)) => { + if !out_key_len.is_null() { + *out_key_len = key_len as i32; + } + if !out_value_len.is_null() { + *out_value_len = value_len as i32; + } + 1 + } + None => 0, + } +} + +/// Free a scan iterator handle. +/// +/// # Safety +/// `handle` must be a valid pointer returned by `bftree_scan_with_count` or +/// `bftree_scan_with_end_key`, and must not be used after this call. +#[no_mangle] +pub unsafe extern "C" fn bftree_scan_drop(handle: *mut ScanHandle<'static>) { + if !handle.is_null() { + drop(Box::from_raw(handle)); + } +} + +// --------------------------------------------------------------------------- +// Snapshot / Recovery (CPR — bftree 0.5+) +// --------------------------------------------------------------------------- + +/// Take a CPR (Concurrent Prefix Recovery) snapshot of a BfTree. +/// +/// Synchronous; designed to be non-blocking to concurrent insert/read/delete +/// callers. Writes the snapshot to the path configured at tree-creation time +/// via `Config::snapshot_file_path` / `use_snapshot=true`. +/// +/// Internal `snapshot_in_progress` AtomicBool serializes concurrent calls; +/// losers no-op silently. To produce snapshots at multiple destination paths, +/// the caller is expected to `File.Move` / copy the configured snapshot file +/// to the final destination after each call. +/// +/// Returns 0 on success, -1 on panic. +/// +/// # Safety +/// `tree` must be a valid BfTree pointer. The tree must have been constructed +/// with `use_snapshot=true` and a non-empty `snapshot_file_path`. +#[no_mangle] +pub unsafe extern "C" fn bftree_cpr_snapshot(tree: *mut BfTree) -> i32 { + let tree = &*tree; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + tree.cpr_snapshot(); + })) { + Ok(_) => 0, + Err(_) => -1, + } +} + +/// Recover a BfTree from a CPR snapshot file. Unified for disk-backed and +/// memory-backed (cache_only) trees — the storage backend is recorded in the +/// snapshot. +/// +/// `recovery_path` / `recovery_path_len`: source CPR snapshot file to recover from. +/// `new_snapshot_path` / `new_snapshot_path_len`: scratch path for the recovered +/// tree's future cpr_snapshot calls. Optional (null/zero disables snapshots +/// on the recovered tree). +/// +/// `buffer_ptr` / `buffer_size`: optional pre-allocated buffer for the +/// recovered tree's cache. If `buffer_ptr` is null, bftree allocates and +/// owns the buffer (freed on `tree.Dispose`). If non-null, the caller owns +/// the buffer. +/// +/// Returns a pointer to the new BfTree, or null on failure. +/// +/// # Safety +/// Caller must eventually call `bftree_drop` on the returned pointer. +#[no_mangle] +pub unsafe extern "C" fn bftree_new_from_cpr_snapshot( + recovery_path: *const u8, + recovery_path_len: i32, + new_snapshot_path: *const u8, + new_snapshot_path_len: i32, + buffer_ptr: *mut u8, + buffer_size: usize, +) -> *mut BfTree { + if recovery_path.is_null() || recovery_path_len <= 0 { + return std::ptr::null_mut(); + } + let recovery_bytes = slice::from_raw_parts(recovery_path, recovery_path_len as usize); + let recovery_str = match std::str::from_utf8(recovery_bytes) { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + let recovery_pathbuf = PathBuf::from(recovery_str); + + let (use_snapshot, new_snapshot_pathbuf) = if !new_snapshot_path.is_null() && new_snapshot_path_len > 0 { + let snap_bytes = slice::from_raw_parts(new_snapshot_path, new_snapshot_path_len as usize); + let snap_str = match std::str::from_utf8(snap_bytes) { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + (true, Some(PathBuf::from(snap_str))) + } else { + (false, None) + }; + + let buf_ptr_opt = if buffer_ptr.is_null() { None } else { Some(buffer_ptr) }; + let buf_size_opt = if buffer_ptr.is_null() { None } else { Some(buffer_size) }; + + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + BfTree::new_from_cpr_snapshot( + recovery_pathbuf, + use_snapshot, + new_snapshot_pathbuf, + buf_ptr_opt, + buf_size_opt, + None, + ) + })) { + Ok(Ok(tree)) => Box::into_raw(Box::new(tree)), + Ok(Err(_)) => std::ptr::null_mut(), + Err(_) => std::ptr::null_mut(), + } +} + +/// Returns 1 if all threads have moved past the snapshot's version barrier, +/// 0 otherwise. Useful for assertions/diagnostics; not strictly required for +/// correctness because `cpr_snapshot` is synchronous. +/// +/// # Safety +/// `tree` must be a valid BfTree pointer constructed with `use_snapshot=true`. +#[no_mangle] +pub unsafe extern "C" fn bftree_are_all_threads_in_next_version(tree: *mut BfTree) -> i32 { + let tree = &*tree; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + tree.are_all_threads_in_next_version() + })) { + Ok(true) => 1, + Ok(false) => 0, + Err(_) => -1, + } +} + +/// No-op function for measuring pure FFI transition overhead. +#[no_mangle] +#[inline(never)] +pub unsafe extern "C" fn bftree_noop( + _tree: *mut BfTree, + _key: *const u8, + _key_len: i32, +) -> i32 { + 0 +} diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 049b46fd80f..bf42982faa7 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -131,7 +131,7 @@ { "Command": "APPEND", "Name": "APPEND", - "Summary": "Appends a string to the value of a key. Creates the key if it doesn\u0027t exist.", + "Summary": "Appends a string to the value of a key. Creates the key if it doesn't exist.", "Group": "String", "Complexity": "O(1). The amortized time complexity is O(1) assuming the appended value is small and the already present value is of any size, since the dynamic string library used by Redis will double the free space available on every reallocation.", "Arguments": [ @@ -682,7 +682,7 @@ "Name": "BLMPOP", "Summary": "Pops the first element from one of multiple lists. Blocks until an element is available otherwise. Deletes the list if the last element was popped.", "Group": "List", - "Complexity": "O(N\u002BM) where N is the number of provided keys and M is the number of elements returned.", + "Complexity": "O(N+M) where N is the number of provided keys and M is the number of elements returned.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -788,7 +788,7 @@ "Group": "List", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060BLMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments", + "ReplacedBy": "`BLMOVE` with the `RIGHT` and `LEFT` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -817,7 +817,7 @@ "Name": "BZMPOP", "Summary": "Removes and returns a member by score from one or more sorted sets. Blocks until a member is available otherwise. Deletes the sorted set if the last element was popped.", "Group": "SortedSet", - "Complexity": "O(K) \u002B O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.", + "Complexity": "O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -1630,7 +1630,7 @@ "Group": "Cluster", "Complexity": "O(N) where N is the total number of Cluster nodes", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060CLUSTER SHARDS\u0060" + "ReplacedBy": "`CLUSTER SHARDS`" }, { "Command": "CLUSTER_SPUBLISH", @@ -1638,6 +1638,13 @@ "Summary": "Processes a forwarded published message from a node in the same shard", "Group": "Cluster", "Complexity": "O(1)" + }, + { + "Command": "CLUSTER_MLOG_KEY_TIME", + "Name": "CLUSTER|MLOG_KEY_TIME", + "Summary": "Returns sequence number for provided key.", + "Group": "Cluster", + "Complexity": "O(1)" } ] }, @@ -1866,7 +1873,7 @@ { "Command": "DECR", "Name": "DECR", - "Summary": "Decrements the integer value of a key by one. Uses 0 as initial value if the key doesn\u0027t exist.", + "Summary": "Decrements the integer value of a key by one. Uses 0 as initial value if the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -1882,7 +1889,7 @@ { "Command": "DECRBY", "Name": "DECRBY", - "Summary": "Decrements a number from the integer value of a key. Uses 0 as initial value if the key doesn\u0027t exist.", + "Summary": "Decrements a number from the integer value of a key. Uses 0 as initial value if the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -1952,7 +1959,7 @@ "Name": "DUMP", "Summary": "Returns a serialized representation of the value stored at a key.", "Group": "Generic", - "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1).", + "Complexity": "O(1) to access the key and additional O(N*M) to serialize it, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1).", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -2346,7 +2353,7 @@ { "Command": "GEOADD", "Name": "GEOADD", - "Summary": "Adds one or more members to a geospatial index. The key is created if it doesn\u0027t exist.", + "Summary": "Adds one or more members to a geospatial index. The key is created if it doesn't exist.", "Group": "Geo", "Complexity": "O(log(N)) for each item added, where N is the number of elements in the sorted set.", "Arguments": [ @@ -2530,9 +2537,9 @@ "Name": "GEORADIUS", "Summary": "Queries a geospatial index for members within a distance from a coordinate, optionally stores the result.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", + "Complexity": "O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060GEOSEARCH\u0060 and \u0060GEOSEARCHSTORE\u0060 with the \u0060BYRADIUS\u0060 argument", + "ReplacedBy": "`GEOSEARCH` and `GEOSEARCHSTORE` with the `BYRADIUS` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -2694,9 +2701,9 @@ "Name": "GEORADIUS_RO", "Summary": "Returns members from a geospatial index that are within a distance from a coordinate.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", + "Complexity": "O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060GEOSEARCH\u0060 with the \u0060BYRADIUS\u0060 argument", + "ReplacedBy": "`GEOSEARCH` with the `BYRADIUS` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -2834,9 +2841,9 @@ "Name": "GEORADIUSBYMEMBER", "Summary": "Queries a geospatial index for members within a distance from a member, optionally stores the result.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", + "Complexity": "O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060GEOSEARCH\u0060 and \u0060GEOSEARCHSTORE\u0060 with the \u0060BYRADIUS\u0060 and \u0060FROMMEMBER\u0060 arguments", + "ReplacedBy": "`GEOSEARCH` and `GEOSEARCHSTORE` with the `BYRADIUS` and `FROMMEMBER` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -2992,9 +2999,9 @@ "Name": "GEORADIUSBYMEMBER_RO", "Summary": "Returns members from a geospatial index that are within a distance from a member.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", + "Complexity": "O(N+log(M)) where N is the number of elements inside the bounding box of the circular area delimited by center and radius and M is the number of items inside the index.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060GEOSEARCH\u0060 with the \u0060BYRADIUS\u0060 and \u0060FROMMEMBER\u0060 arguments", + "ReplacedBy": "`GEOSEARCH` with the `BYRADIUS` and `FROMMEMBER` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -3126,7 +3133,7 @@ "Name": "GEOSEARCH", "Summary": "Queries a geospatial index for members inside an area of a box or a circle.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", + "Complexity": "O(N+log(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -3356,7 +3363,7 @@ "Name": "GEOSEARCHSTORE", "Summary": "Queries a geospatial index for members inside an area of a box or a circle, optionally stores the result.", "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", + "Complexity": "O(N+log(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -3688,7 +3695,7 @@ { "Command": "GETIFNOTMATCH", "Name": "GETIFNOTMATCH", - "Summary": "Gets the ETag and value if the key\u0027s current etag does not match the given etag.", + "Summary": "Gets the ETag and value if the key's current etag does not match the given etag.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -3742,7 +3749,7 @@ "Group": "String", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060SET\u0060 with the \u0060!GET\u0060 argument", + "ReplacedBy": "`SET` with the `!GET` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -4114,7 +4121,7 @@ { "Command": "HINCRBY", "Name": "HINCRBY", - "Summary": "Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn\u0027t exist.", + "Summary": "Increments the integer value of a field in a hash by a number. Uses 0 as initial value if the field doesn't exist.", "Group": "Hash", "Complexity": "O(1)", "Arguments": [ @@ -4142,7 +4149,7 @@ { "Command": "HINCRBYFLOAT", "Name": "HINCRBYFLOAT", - "Summary": "Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn\u0027t exist.", + "Summary": "Increments the floating point value of a field by a number. Uses 0 as initial value if the field doesn't exist.", "Group": "Hash", "Complexity": "O(1)", "Arguments": [ @@ -4229,7 +4236,7 @@ "Group": "Hash", "Complexity": "O(N) where N is the number of fields being set.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060HSET\u0060 with multiple field-value pairs", + "ReplacedBy": "`HSET` with multiple field-value pairs", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -4644,7 +4651,7 @@ { "Command": "HSETNX", "Name": "HSETNX", - "Summary": "Sets the value of a field in a hash only when the field doesn\u0027t exist.", + "Summary": "Sets the value of a field in a hash only when the field doesn't exist.", "Group": "Hash", "Complexity": "O(1)", "Arguments": [ @@ -4747,7 +4754,7 @@ { "Command": "INCR", "Name": "INCR", - "Summary": "Increments the integer value of a key by one. Uses 0 as initial value if the key doesn\u0027t exist.", + "Summary": "Increments the integer value of a key by one. Uses 0 as initial value if the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -4763,7 +4770,7 @@ { "Command": "INCRBY", "Name": "INCRBY", - "Summary": "Increments the integer value of a key by a number. Uses 0 as initial value if the key doesn\u0027t exist.", + "Summary": "Increments the integer value of a key by a number. Uses 0 as initial value if the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -4785,7 +4792,7 @@ { "Command": "INCRBYFLOAT", "Name": "INCRBYFLOAT", - "Summary": "Increment the floating point value of a key by a number. Uses 0 as initial value if the key doesn\u0027t exist.", + "Summary": "Increment the floating point value of a key by a number. Uses 0 as initial value if the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -5112,7 +5119,7 @@ "Name": "LMPOP", "Summary": "Returns multiple elements from a list after removing them. Deletes the list if the last element was popped.", "Group": "List", - "Complexity": "O(N\u002BM) where N is the number of provided keys and M is the number of elements returned.", + "Complexity": "O(N+M) where N is the number of provided keys and M is the number of elements returned.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -5231,7 +5238,7 @@ { "Command": "LPUSH", "Name": "LPUSH", - "Summary": "Prepends one or more elements to a list. Creates the key if it doesn\u0027t exist.", + "Summary": "Prepends one or more elements to a list. Creates the key if it doesn't exist.", "Group": "List", "Complexity": "O(1) for each element added, so O(N) to add N elements when the command is called with multiple arguments.", "Arguments": [ @@ -5279,7 +5286,7 @@ "Name": "LRANGE", "Summary": "Returns a range of elements from a list.", "Group": "List", - "Complexity": "O(S\u002BN) where S is the distance of start offset from HEAD for small lists, from nearest end (HEAD or TAIL) for large lists; and N is the number of elements in the specified range.", + "Complexity": "O(S+N) where S is the distance of start offset from HEAD for small lists, from nearest end (HEAD or TAIL) for large lists; and N is the number of elements in the specified range.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -5307,7 +5314,7 @@ "Name": "LREM", "Summary": "Removes elements from a list. Deletes the list if the last element was removed.", "Group": "List", - "Complexity": "O(N\u002BM) where N is the length of the list and M is the number of elements removed.", + "Complexity": "O(N+M) where N is the length of the list and M is the number of elements removed.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -5441,7 +5448,7 @@ "Name": "MIGRATE", "Summary": "Atomically transfers a key from one Redis instance to another.", "Group": "Generic", - "Complexity": "This command actually executes a DUMP\u002BDEL in the source instance, and a RESTORE in the target instance. See the pages of these commands for time complexity. Also an O(N) data transfer between the two instances is performed.", + "Complexity": "This command actually executes a DUMP+DEL in the source instance, and a RESTORE in the target instance. See the pages of these commands for time complexity. Also an O(N) data transfer between the two instances is performed.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -5621,7 +5628,7 @@ { "Command": "MSETNX", "Name": "MSETNX", - "Summary": "Atomically modifies the string values of one or more keys only when all keys don\u0027t exist.", + "Summary": "Atomically modifies the string values of one or more keys only when all keys don't exist.", "Group": "String", "Complexity": "O(N) where N is the number of keys to set.", "Arguments": [ @@ -5806,7 +5813,7 @@ { "Command": "PFADD", "Name": "PFADD", - "Summary": "Adds elements to a HyperLogLog key. Creates the key if it doesn\u0027t exist.", + "Summary": "Adds elements to a HyperLogLog key. Creates the key if it doesn't exist.", "Group": "HyperLogLog", "Complexity": "O(1) to add every element.", "Arguments": [ @@ -5870,7 +5877,7 @@ { "Command": "PING", "Name": "PING", - "Summary": "Returns the server\u0027s liveliness response.", + "Summary": "Returns the server's liveliness response.", "Group": "Connection", "Complexity": "O(1)", "Arguments": [ @@ -5886,11 +5893,11 @@ { "Command": "PSETEX", "Name": "PSETEX", - "Summary": "Sets both string value and expiration time in milliseconds of a key. The key is created if it doesn\u0027t exist.", + "Summary": "Sets both string value and expiration time in milliseconds of a key. The key is created if it doesn't exist.", "Group": "String", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060SET\u0060 with the \u0060PX\u0060 argument", + "ReplacedBy": "`SET` with the `PX` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -5950,7 +5957,7 @@ "Name": "PUBLISH", "Summary": "Posts a message to a channel.", "Group": "PubSub", - "Complexity": "O(N\u002BM) where N is the number of clients subscribed to the receiving channel and M is the total number of subscribed patterns (by any client).", + "Complexity": "O(N+M) where N is the number of clients subscribed to the receiving channel and M is the total number of subscribed patterns (by any client).", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -6320,7 +6327,7 @@ "Name": "RESTORE", "Summary": "Creates a key from the serialized representation of a value.", "Group": "Generic", - "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)\u002BO(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).", + "Complexity": "O(1) to create the new key and additional O(N*M) to reconstruct the serialized value, where N is the number of Redis objects composing the value and M their average size. For small string values the time complexity is thus O(1)+O(1*M) where M is small, so simply O(1). However for sorted set values the complexity is O(N*M*log(N)) because inserting values into sorted sets is O(log(N)).", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -6375,6 +6382,321 @@ } ] }, + { + "Command": "RICREATE", + "Name": "RI.CREATE", + "Summary": "Creates a new RangeIndex backed by a BfTree. Returns an error if the key already exists.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "BACKEND", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "DISK", + "DisplayText": "path", + "Type": "String", + "Token": "DISK" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMORY", + "DisplayText": "memory", + "Type": "PureToken", + "Token": "MEMORY" + } + ] + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "CACHESIZE", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "CACHESIZE", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MINRECORD", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MINRECORD", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MAXRECORD", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MAXRECORD", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MAXKEYLEN", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MAXKEYLEN", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "PAGESIZE", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "PAGESIZE", + "ArgumentFlags": "Optional" + } + ] + }, + { + "Command": "RICONFIG", + "Name": "RI.CONFIG", + "Summary": "Returns the configuration of a RangeIndex as alternating field-value pairs.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIDEL", + "Name": "RI.DEL", + "Summary": "Deletes an entry from a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + } + ] + }, + { + "Command": "RIEXISTS", + "Name": "RI.EXISTS", + "Summary": "Checks whether a key exists and is a RangeIndex. Returns 1 if true, 0 otherwise.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIGET", + "Name": "RI.GET", + "Summary": "Gets the value of an entry in a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + } + ] + }, + { + "Command": "RIMETRICS", + "Name": "RI.METRICS", + "Summary": "Returns runtime metrics for a RangeIndex including tree handle status and flags.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIRANGE", + "Name": "RI.RANGE", + "Summary": "Scans entries in a closed key range [start, end] from a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N + M) where M is the number of entries in the range", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "START", + "DisplayText": "start", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "END", + "DisplayText": "end", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS", + "Type": "OneOf", + "Token": "FIELDS", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KEY_FIELD", + "DisplayText": "key", + "Type": "PureToken", + "Token": "KEY" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE_FIELD", + "DisplayText": "value", + "Type": "PureToken", + "Token": "VALUE" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "BOTH_FIELD", + "DisplayText": "both", + "Type": "PureToken", + "Token": "BOTH" + } + ] + } + ] + }, + { + "Command": "RISCAN", + "Name": "RI.SCAN", + "Summary": "Scans entries from a RangeIndex starting at a key, returning up to COUNT entries.", + "Group": "Generic", + "Complexity": "O(log N + M) where M is the count", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "START", + "DisplayText": "start", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "COUNT", + "DisplayText": "count", + "Type": "Integer", + "Token": "COUNT" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS", + "Type": "OneOf", + "Token": "FIELDS", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KEY_FIELD", + "DisplayText": "key", + "Type": "PureToken", + "Token": "KEY" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE_FIELD", + "DisplayText": "value", + "Type": "PureToken", + "Token": "VALUE" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "BOTH_FIELD", + "DisplayText": "both", + "Type": "PureToken", + "Token": "BOTH" + } + ] + } + ] + }, + { + "Command": "RISET", + "Name": "RI.SET", + "Summary": "Inserts or updates an entry in a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE", + "DisplayText": "value", + "Type": "String" + } + ] + }, { "Command": "ROLE", "Name": "ROLE", @@ -6412,7 +6734,7 @@ "Group": "List", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060LMOVE\u0060 with the \u0060RIGHT\u0060 and \u0060LEFT\u0060 arguments", + "ReplacedBy": "`LMOVE` with the `RIGHT` and `LEFT` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -6433,7 +6755,7 @@ { "Command": "RPUSH", "Name": "RPUSH", - "Summary": "Appends one or more elements to a list. Creates the key if it doesn\u0027t exist.", + "Summary": "Appends one or more elements to a list. Creates the key if it doesn't exist.", "Group": "List", "Complexity": "O(1) for each element added, so O(N) to add N elements when the command is called with multiple arguments.", "Arguments": [ @@ -6502,7 +6824,7 @@ { "Command": "SADD", "Name": "SADD", - "Summary": "Adds one or more members to a set. Creates the key if it doesn\u0027t exist.", + "Summary": "Adds one or more members to a set. Creates the key if it doesn't exist.", "Group": "Set", "Complexity": "O(1) for each element added, so O(N) to add N elements when the command is called with multiple arguments.", "Arguments": [ @@ -6780,7 +7102,7 @@ { "Command": "SET", "Name": "SET", - "Summary": "Sets the string value of a key, ignoring its type. The key is created if it doesn\u0027t exist.", + "Summary": "Sets the string value of a key, ignoring its type. The key is created if it doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -6883,7 +7205,7 @@ { "Command": "SETBIT", "Name": "SETBIT", - "Summary": "Sets or clears the bit at offset of the string value. Creates the key if it doesn\u0027t exist.", + "Summary": "Sets or clears the bit at offset of the string value. Creates the key if it doesn't exist.", "Group": "Bitmap", "Complexity": "O(1)", "Arguments": [ @@ -6911,11 +7233,11 @@ { "Command": "SETEX", "Name": "SETEX", - "Summary": "Sets the string value and expiration time of a key. Creates the key if it doesn\u0027t exist.", + "Summary": "Sets the string value and expiration time of a key. Creates the key if it doesn't exist.", "Group": "String", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060SET\u0060 with the \u0060EX\u0060 argument", + "ReplacedBy": "`SET` with the `EX` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -7054,14 +7376,58 @@ } ] }, + { + "Command": "SETWITHETAG", + "Name": "SETWITHETAG", + "Summary": "Sets a key-value pair with an ETag. If the key already exists, the value is overwritten and the ETag is incremented. Returns the ETag.", + "Group": "String", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE", + "DisplayText": "value", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "EXPIRATION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SECONDS", + "DisplayText": "seconds", + "Type": "Integer", + "Token": "EX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MILLISECONDS", + "DisplayText": "milliseconds", + "Type": "Integer", + "Token": "PX" + } + ] + } + ] + }, { "Command": "SETNX", "Name": "SETNX", - "Summary": "Set the string value of a key only when the key doesn\u0027t exist.", + "Summary": "Set the string value of a key only when the key doesn't exist.", "Group": "String", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060SET\u0060 with the \u0060NX\u0060 argument", + "ReplacedBy": "`SET` with the `NX` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -7081,7 +7447,7 @@ { "Command": "SETRANGE", "Name": "SETRANGE", - "Summary": "Overwrites a part of a string value with another by an offset. Creates the key if it doesn\u0027t exist.", + "Summary": "Overwrites a part of a string value with another by an offset. Creates the key if it doesn't exist.", "Group": "String", "Complexity": "O(1), not counting the time taken to copy the new string in place. Usually, this string is very small so the amortized complexity is O(1). Otherwise, complexity is O(M) with M being the length of the value argument.", "Arguments": [ @@ -7207,7 +7573,7 @@ "Group": "Server", "Complexity": "O(1)", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060REPLICAOF\u0060", + "ReplacedBy": "`REPLICAOF`", "Arguments": [ { "TypeDiscriminator": "RespCommandContainerArgument", @@ -7268,7 +7634,7 @@ { "Command": "SLOWLOG_GET", "Name": "SLOWLOG|GET", - "Summary": "Returns the slow log\u0027s entries.", + "Summary": "Returns the slow log's entries.", "Group": "Server", "Complexity": "O(N) where N is the number of entries returned", "Arguments": [ @@ -7555,7 +7921,7 @@ "Group": "String", "Complexity": "O(N) where N is the length of the returned string. The complexity is ultimately determined by the returned length, but because creating a substring from an existing string is very cheap, it can be considered O(1) for small strings.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060GETRANGE\u0060", + "ReplacedBy": "`GETRANGE`", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -7971,7 +8337,7 @@ { "Command": "ZADD", "Name": "ZADD", - "Summary": "Adds one or more members to a sorted set, or updates their scores. Creates the key if it doesn\u0027t exist.", + "Summary": "Adds one or more members to a sorted set, or updates their scores. Creates the key if it doesn't exist.", "Group": "SortedSet", "Complexity": "O(log(N)) for each item added, where N is the number of elements in the sorted set.", "Arguments": [ @@ -8119,7 +8485,7 @@ "Name": "ZDIFF", "Summary": "Returns the difference between multiple sorted sets.", "Group": "SortedSet", - "Complexity": "O(L \u002B (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.", + "Complexity": "O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -8150,7 +8516,7 @@ "Name": "ZDIFFSTORE", "Summary": "Stores the difference of multiple sorted sets in a key.", "Group": "SortedSet", - "Complexity": "O(L \u002B (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.", + "Complexity": "O(L + (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -8403,7 +8769,7 @@ "Name": "ZINTER", "Summary": "Returns the intersect of multiple sorted sets.", "Group": "SortedSet", - "Complexity": "O(N*K)\u002BO(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.", + "Complexity": "O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -8503,7 +8869,7 @@ "Name": "ZINTERSTORE", "Summary": "Stores the intersect of multiple sorted sets in a key.", "Group": "SortedSet", - "Complexity": "O(N*K)\u002BO(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.", + "Complexity": "O(N*K)+O(M*log(M)) worst case with N being the smallest input sorted set, K being the number of input sorted sets and M being the number of elements in the resulting sorted set.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -8599,7 +8965,7 @@ "Name": "ZMPOP", "Summary": "Returns the highest- or lowest-scoring members from one or more sorted sets after removing them. Deletes the sorted set if the last member was popped.", "Group": "SortedSet", - "Complexity": "O(K) \u002B O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.", + "Complexity": "O(K) + O(M*log(N)) where K is the number of provided keys, N being the number of elements in the sorted set, and M being the number of elements popped.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -9027,7 +9393,7 @@ "Name": "ZRANGE", "Summary": "Returns members in a sorted set within a range of indexes.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements returned.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements returned.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9114,9 +9480,9 @@ "Name": "ZRANGEBYLEX", "Summary": "Returns members in a sorted set within a lexicographical range.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060BYLEX\u0060 argument", + "ReplacedBy": "`ZRANGE` with the `BYLEX` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9165,9 +9531,9 @@ "Name": "ZRANGEBYSCORE", "Summary": "Returns members in a sorted set within a range of scores.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060BYSCORE\u0060 argument", + "ReplacedBy": "`ZRANGE` with the `BYSCORE` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9224,7 +9590,7 @@ "Name": "ZRANGESTORE", "Summary": "Stores a range of members from sorted set in a key.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements stored into the destination key.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements stored into the destination key.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9363,7 +9729,7 @@ "Name": "ZREMRANGEBYLEX", "Summary": "Removes members in a sorted set within a lexicographical range. Deletes the sorted set if all members were removed.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9391,7 +9757,7 @@ "Name": "ZREMRANGEBYRANK", "Summary": "Removes members in a sorted set within a range of indexes. Deletes the sorted set if all members were removed.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9419,7 +9785,7 @@ "Name": "ZREMRANGEBYSCORE", "Summary": "Removes members in a sorted set within a range of scores. Deletes the sorted set if all members were removed.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements removed by the operation.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9447,9 +9813,9 @@ "Name": "ZREVRANGE", "Summary": "Returns members in a sorted set within a range of indexes in reverse order.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements returned.", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements returned.", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 argument", + "ReplacedBy": "`ZRANGE` with the `REV` argument", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9485,9 +9851,9 @@ "Name": "ZREVRANGEBYLEX", "Summary": "Returns members in a sorted set within a lexicographical range in reverse order.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYLEX\u0060 arguments", + "ReplacedBy": "`ZRANGE` with the `REV` and `BYLEX` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9536,9 +9902,9 @@ "Name": "ZREVRANGEBYSCORE", "Summary": "Returns members in a sorted set within a range of scores in reverse order.", "Group": "SortedSet", - "Complexity": "O(log(N)\u002BM) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", + "Complexity": "O(log(N)+M) with N being the number of elements in the sorted set and M the number of elements being returned. If M is constant (e.g. always asking for the first 10 elements with LIMIT), you can consider it O(log(N)).", "DocFlags": "Deprecated", - "ReplacedBy": "\u0060ZRANGE\u0060 with the \u0060REV\u0060 and \u0060BYSCORE\u0060 arguments", + "ReplacedBy": "`ZRANGE` with the `REV` and `BYSCORE` arguments", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9722,7 +10088,7 @@ "Name": "ZUNION", "Summary": "Returns the union of multiple sorted sets.", "Group": "SortedSet", - "Complexity": "O(N)\u002BO(M*log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.", + "Complexity": "O(N)+O(M*log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.", "Arguments": [ { "TypeDiscriminator": "RespCommandBasicArgument", @@ -9791,7 +10157,7 @@ "Name": "ZUNIONSTORE", "Summary": "Stores the union of multiple sorted sets in a key.", "Group": "SortedSet", - "Complexity": "O(N)\u002BO(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.", + "Complexity": "O(N)+O(M log(M)) with N being the sum of the sizes of the input sorted sets, and M being the number of elements in the resulting sorted set.", "Arguments": [ { "TypeDiscriminator": "RespCommandKeyArgument", @@ -9854,4 +10220,4 @@ } ] } -] \ No newline at end of file +] diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 40aa9686505..f7a3dc4b550 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -614,10 +614,10 @@ "AclCategories": "Admin, Dangerous, Slow" }, { - "Command": "CLUSTER_AOFSYNC", - "Name": "CLUSTER|AOFSYNC", + "Command": "CLUSTER_ADVANCE_TIME", + "Name": "CLUSTER|ADVANCE_TIME", "IsInternal": true, - "Arity": 3, + "Arity": 2, "Flags": "Admin, NoMulti, NoScript", "AclCategories": "Admin, Dangerous, Slow, Garnet" }, @@ -903,6 +903,14 @@ "Flags": "Admin, NoMulti, NoScript", "AclCategories": "Admin, Dangerous, Slow, Garnet" }, + { + "Command": "CLUSTER_SNAPSHOT_DATA", + "Name": "CLUSTER|SNAPSHOT_DATA", + "IsInternal": true, + "Arity": 6, + "Flags": "Admin, NoMulti, NoScript", + "AclCategories": "Admin, Dangerous, Slow, Garnet" + }, { "Command": "CLUSTER_SETCONFIGEPOCH", "Name": "CLUSTER|SET-CONFIG-EPOCH", @@ -979,6 +987,13 @@ } ] }, + { + "Command": "CLUSTER_MLOG_KEY_TIME", + "Name": "CLUSTER|MLOG_KEY_TIME", + "Arity": -2, + "Flags": "Admin, Readonly, NoMulti, NoScript", + "AclCategories": "Admin, Slow, Garnet" + }, { "Command": "CLUSTER_SYNC", "Name": "CLUSTER|SYNC", @@ -3918,6 +3933,240 @@ ], "StoreType": "All" }, + { + "Command": "RICREATE", + "Name": "RI.CREATE", + "Arity": -2, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert" + } + ], + "StoreType": "Main" + }, + { + "Command": "RICONFIG", + "Name": "RI.CONFIG", + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIDEL", + "Name": "RI.DEL", + "Arity": 3, + "Flags": "Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Delete" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIEXISTS", + "Name": "RI.EXISTS", + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIGET", + "Name": "RI.GET", + "Arity": 3, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIMETRICS", + "Name": "RI.METRICS", + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIRANGE", + "Name": "RI.RANGE", + "Arity": -4, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RISCAN", + "Name": "RI.SCAN", + "Arity": -5, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Read, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RISET", + "Name": "RI.SET", + "Arity": 4, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Write, Garnet", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert" + } + ], + "StoreType": "Main" + }, { "Command": "ROLE", "Name": "ROLE", @@ -4268,7 +4517,7 @@ "KeyStep": 1, "Limit": 0 }, - "Notes": "RW and ACCESS due to the optional \u0060GET\u0060 argument", + "Notes": "RW and ACCESS due to the optional `GET` argument", "Flags": "RW, Access, Update, VariableFlags" } ], @@ -4376,6 +4625,32 @@ ], "StoreType": "Main" }, + { + "Command": "SETWITHETAG", + "Name": "SETWITHETAG", + "Arity": -3, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, String, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert, Update" + } + ], + "StoreType": "Main" + }, { "Command": "SETNX", "Name": "SETNX", @@ -6611,4 +6886,4 @@ ], "StoreType": "Object" } -] \ No newline at end of file +] diff --git a/libs/server/ACL/ACLParser.cs b/libs/server/ACL/ACLParser.cs index 2ee3297867c..7fb41f57c12 100644 --- a/libs/server/ACL/ACLParser.cs +++ b/libs/server/ACL/ACLParser.cs @@ -253,9 +253,13 @@ static bool TryParseCommandForAcl(string commandName, out RespCommand command) if (!Enum.TryParse(effectiveName, ignoreCase: true, out command) || !IsValidParse(command, effectiveName)) { - // We handle these commands specially because blind replacements would cause - // us to be too accepting of different values - if (commandName.Equals("SLAVEOF", StringComparison.OrdinalIgnoreCase)) + // Try replacing dots with empty strings for commands like RI.CREATE -> RICREATE + string dotlessName = effectiveName.Replace(".", ""); + if (dotlessName != effectiveName && Enum.TryParse(dotlessName, ignoreCase: true, out command) && IsValidParse(command, dotlessName)) + { + // Successfully parsed after removing dots — fall through to validation below + } + else if (commandName.Equals("SLAVEOF", StringComparison.OrdinalIgnoreCase)) { command = RespCommand.SECONDARYOF; } diff --git a/libs/server/AOF/AofAddress.cs b/libs/server/AOF/AofAddress.cs new file mode 100644 index 00000000000..47d1fb80f5e --- /dev/null +++ b/libs/server/AOF/AofAddress.cs @@ -0,0 +1,392 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Garnet.server +{ + /// + /// Represents a fixed-size collection of addresses used for append-only file (AOF) operations, supporting efficient + /// serialization, comparison, and manipulation of address sequences. + /// + public unsafe struct AofAddress + { + readonly byte length; + fixed long addresses[MaxSublogCount]; + + /// + /// Maximum number of sublogs supported + /// + public const int MaxSublogCount = 4; + + /// + /// AofAddress length + /// + public readonly int Length => length; + + /// + /// Provides a span of bytes representing the underlying addresses array. + /// + public Span Span + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => MemoryMarshal.CreateSpan(ref Unsafe.As(ref addresses[0]), sizeof(long) * length); + } + + /// + /// Indexer + /// + /// + /// + public long this[int i] + { + readonly get + { + return addresses[i]; + } + set + { + addresses[i] = value; + } + } + + /// + /// Determines whether the current instance and the specified instance represent the + /// same address sequence. + /// + /// The instance to compare with the current instance. This parameter cannot be null + /// and must have the same length as the current instance. + /// if the specified is equal to the current instance; + /// otherwise, . + public bool Equals(in AofAddress other) + { + Debug.Assert(other.Length == Length); + for (var i = 0; i < Length; i++) + if (addresses[i] != other[i]) return false; + return true; + } + + /// + /// AofAddress constructor + /// + /// + internal AofAddress(int length) + { + Debug.Assert(length <= MaxSublogCount); + this.length = (byte)length; + } + + /// + /// Convert to byte array + /// + /// + public byte[] ToByteArray() + { + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms, Encoding.ASCII); + Serialize(writer); + return ms.ToArray(); + } + + /// + /// Convert to AofAddress from byte array + /// + /// + /// + public static AofAddress FromByteArray(byte[] data) + { + using var ms = new MemoryStream(data); + using var reader = new BinaryReader(ms, Encoding.ASCII); + return Deserialize(reader); + } + + /// + /// Create AofAddress from span + /// + /// + /// + public static AofAddress FromSpan(Span span) + { + var length = span.Length >> 3; + var aofAddress = new AofAddress(length); + fixed (byte* ptr = span) + { + var curr = ptr; + for (var i = 0; i < length; i++) + { + aofAddress[i] = *(long*)curr; + curr += sizeof(long); + } + } + return aofAddress; + } + + /// + /// Comma separate string of valid addresses in this AofAddress + /// + /// + public override string ToString() + { + var sb = new StringBuilder(); + _ = sb.Append(addresses[0]); + for (var i = 1; i < Length; i++) + { + _ = sb.Append(','); + _ = sb.Append(addresses[i]); + } + return sb.ToString(); + } + + /// + /// Create AofAddress from command separated string of addresses + /// + /// + /// + public static AofAddress FromString(string input) + { + var span = input.AsSpan(); + + // Count commas to determine array size + var count = 1; + for (var i = 0; i < span.Length; i++) + if (span[i] == ',') count++; + + var aofAddress = new AofAddress(count); + var idx = 0; + var value = 0L; + var negative = false; + for (var i = 0; i < span.Length; i++) + { + var c = span[i]; + if (c == ',') + { + aofAddress[idx++] = value; + value = 0; + } + else if (c >= '0' && c <= '9') + { + value = value * 10 + (c - '0'); + } + else if (c == '-') + { + negative = true; + } + else + { + throw new FormatException($"Invalid character '{c}' in AofAddress string."); + } + } + + // Handle last value + aofAddress[idx] = value * (negative ? -1 : 1); + return aofAddress; + } + + /// + /// Serialize contents using provided BinaryWriter + /// + /// + public void Serialize(BinaryWriter writer) + { + writer.Write(length); + for (var i = 0; i < Length; i++) + writer.Write(addresses[i]); + } + + /// + /// Deserialize contents and allocate a new instance using provided BinaryReader + /// + /// + /// + public static AofAddress Deserialize(BinaryReader reader) + { + var length = reader.ReadByte(); + var aofAddress = new AofAddress(length); + for (var i = 0; i < length; i++) + aofAddress[i] = reader.ReadInt64(); + return aofAddress; + } + + /// + /// Set to value if address equals to comparand + /// + /// + /// + public void SetValueIf(long value, long comparand) + { + for (var i = 0; i < Length; i++) + if (addresses[i] == comparand) + addresses[i] = value; + } + + /// + /// Set to value if address equals to comparand + /// + /// + /// + public void SetValueIf(in AofAddress value, long comparand) + { + for (var i = 0; i < Length; i++) + { + if (addresses[i] == comparand) + addresses[i] = value[i]; + } + } + + /// + /// Set to value from aofAddress + /// + /// + public void SetValue(ref AofAddress aofAddress) + { + for (var i = 0; i < Length; i++) + addresses[i] = aofAddress[i]; + } + + /// + /// Set to value + /// + /// + public void SetValue(long value) + { + for (var i = 0; i < Length; i++) + addresses[i] = value; + } + + /// + /// Allocate AofAddress of provided length and set to value + /// + /// + /// + /// + public static AofAddress Create(int length, long value) + { + var aofAddress = new AofAddress(length); + for (var i = 0; i < length; i++) + aofAddress[i] = value; + return aofAddress; + } + + /// + /// Allocate AofAddress and assign its contents with the min-pairwise value of the provided inputs + /// + /// + /// + /// + public static AofAddress Min(ref AofAddress a, ref AofAddress b) + { + var aofAddress = new AofAddress(a.Length); + for (var i = 0; i < a.Length; i++) + aofAddress[i] = Math.Min(a[i], b[i]); + return aofAddress; + } + + public void MonotonicUpdate(ref AofAddress update) + { + for (var i = 0; i < Length; i++) + _ = Tsavorite.core.Utility.MonotonicUpdate(ref addresses[i], update[i], out _); + } + + public void MonotonicUpdate(long update, int physicalSublogIdx) + { + _ = Tsavorite.core.Utility.MonotonicUpdate(ref addresses[physicalSublogIdx], update, out _); + } + + public void MinExchange(in AofAddress address) + { + for (var i = 0; i < Length; i++) + addresses[i] = Math.Min(addresses[i], address[i]); + } + + public void MaxExchange(long address) + { + for (var i = 0; i < Length; i++) + addresses[i] = Math.Max(addresses[i], address); + } + + public bool AnyLesser(in AofAddress address) + { + for (var i = 0; i < Length; i++) + if (addresses[i] < address[i]) return true; + return false; + } + + public bool AnyGreater(in AofAddress address) + { + for (var i = 0; i < Length; i++) + if (addresses[i] > address[i]) return true; + return false; + } + + public bool AnyGreater(long value) + { + for (var i = 0; i < Length; i++) + if (addresses[i] > value) return false; + return true; + } + + public AofAddress Diff(in AofAddress other) + { + Debug.Assert(other.Length == Length); + var aofAddress = new AofAddress(other.Length); + for (var i = 0; i < other.Length; i++) + aofAddress[i] = this.addresses[i] - other[i]; + return aofAddress; + } + + public long AggregateDiff(in AofAddress aofAddress) + { + var diff = 0L; + for (var i = 0; i < Length; i++) + diff += addresses[i] - aofAddress[i]; + return diff; + } + + public long AggregateDiff(long value) + { + var diff = 0L; + for (var i = 0; i < Length; i++) + diff += addresses[i] - value; + return diff; + } + + public bool EqualsAll(in AofAddress input) + { + for (var i = 0; i < Length; i++) + if (addresses[i] != input[i]) + return false; + return true; + } + + public bool IsOutOfRange(in AofAddress begin, in AofAddress end) + { + for (var i = 0; i < Length; i++) + { + if (addresses[i] < begin[i] || addresses[i] > end[i]) + return true; + } + return false; + } + + public long Max() + { + var max = 0L; + for (var i = 0; i < Length; i++) + max = Math.Max(max, addresses[i]); + return max; + } + + public long Min() + { + var max = 0L; + for (var i = 0; i < Length; i++) + max = Math.Min(max, addresses[i]); + return max; + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/AofEntryType.cs b/libs/server/AOF/AofEntryType.cs index 92d70dd0bb8..6c8af82a1be 100644 --- a/libs/server/AOF/AofEntryType.cs +++ b/libs/server/AOF/AofEntryType.cs @@ -78,6 +78,23 @@ public enum AofEntryType : byte /// Flush db /// FlushDb = 0x61, + + /// + /// Unified store upsert string + /// + UnifiedStoreStringUpsert = 0x70, + /// + /// Unified store upsert object + /// + UnifiedStoreObjectUpsert = 0x71, + /// + /// Unified store RMW + /// + UnifiedStoreRMW = 0x72, + /// + /// Unified store delete + /// + UnifiedStoreDelete = 0x73, } internal enum AofStoreType : byte @@ -89,4 +106,26 @@ internal enum AofStoreType : byte CheckpointType = 0x4, FlushDbType = 0x5, } + + internal static class AofEntryTypeExtensions + { + /// + /// Returns true if the entry type carries a key payload after the header. + /// Keyless entries (transactions, checkpoints, flush, stored procedures) have no key. + /// + internal static bool HasKey(this AofEntryType opType) => opType switch + { + AofEntryType.StoreUpsert or + AofEntryType.StoreRMW or + AofEntryType.StoreDelete or + AofEntryType.ObjectStoreUpsert or + AofEntryType.ObjectStoreRMW or + AofEntryType.ObjectStoreDelete or + AofEntryType.UnifiedStoreStringUpsert or + AofEntryType.UnifiedStoreObjectUpsert or + AofEntryType.UnifiedStoreRMW or + AofEntryType.UnifiedStoreDelete => true, + _ => false, + }; + } } \ No newline at end of file diff --git a/libs/server/AOF/AofHeader.cs b/libs/server/AOF/AofHeader.cs index 8d60d03392c..d1a27b1d83d 100644 --- a/libs/server/AOF/AofHeader.cs +++ b/libs/server/AOF/AofHeader.cs @@ -1,18 +1,167 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Diagnostics; using System.Runtime.InteropServices; +using Garnet.common; namespace Garnet.server { - [StructLayout(LayoutKind.Explicit, Size = 16)] + // AOF Header Hierarchy + // + // The header type determines the wire format of each AOF entry based on the log topology: + // + // AofHeader (16B) — Base header for all entries. Used standalone with single-log mode. + // │ + // ├── AofShardedHeader (24B) = AofHeader + sequenceNumber + // │ Used for per-key entries in multi-physical-log (sharded) mode. + // │ The sequence number enables cross-sublog ordering. + // │ + // ├── AofSingleLogTransactionHeader (50B) = AofHeader + participantCount + replayTaskAccessVector + // │ Used for coordinated/broadcast operations (transactions, checkpoints, flush) + // │ in single-physical-log + multi-replay mode. Uses log addresses for ordering + // │ instead of embedded sequence numbers, saving 8B per entry. + // │ + // └── AofShardedLogTransactionHeader (58B) = AofShardedHeader + participantCount + replayTaskAccessVector + // Used for coordinated/broadcast operations in multi-physical-log (sharded) mode. + // Embeds a sequence number (via AofShardedHeader) for cross-sublog ordering. + // + // Selection logic: + // Single log (1 physical, 1 replay task) → BasicHeader + // Single physical log, multi-replay → BasicHeader (per-key), SingleLogTransactionHeader (broadcast) + // Multi physical log, multi-replay → ShardedHeader (per-key), ShardedLogTransactionHeader (broadcast) + internal enum AofHeaderType : byte + { + BasicHeader = 0, + ShardedHeader = 1, + SingleLogTransactionHeader = 2, + ShardedLogTransactionHeader = 3, + } + + /// + /// Used for coordinated operations + /// + [StructLayout(LayoutKind.Explicit, Size = TotalSize)] + unsafe struct AofShardedLogTransactionHeader + { + public const int TotalSize = AofShardedHeader.TotalSize + 2 + 32; + // maximum 256 replay tasks per physical sublog, hence 32 bytes bitmap + public const int ReplayTaskAccessVectorBytes = 32; + + /// + /// AofShardedHeader used with multi-log + /// + [FieldOffset(0)] + public AofShardedHeader shardedHeader; + + /// + /// Used for synchronizing virtual sublog replay + /// NOTE: This stores the total number of replay tasks that participate in a given transaction. + /// + [FieldOffset(AofShardedHeader.TotalSize)] + public short participantCount; + + /// + /// Used to track replay task participating in the txn + /// + [FieldOffset(AofShardedHeader.TotalSize + 2)] + public fixed byte replayTaskAccessVector[ReplayTaskAccessVectorBytes]; + } + + /// + /// Used for single-physical-log with multi-replay to carry transaction participant info + /// without embedding a sequence number (log addresses are used instead). + /// + [StructLayout(LayoutKind.Explicit, Size = TotalSize)] + unsafe struct AofSingleLogTransactionHeader + { + public const int TotalSize = AofHeader.TotalSize + 2 + AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes; + + /// + /// Basic AOF header + /// + [FieldOffset(0)] + public AofHeader basicHeader; + + /// + /// Used for synchronizing virtual sublog replay + /// NOTE: This stores the total number of replay tasks that participate in a given transaction. + /// + [FieldOffset(AofHeader.TotalSize)] + public short participantCount; + + /// + /// Used to track replay task participating in the txn + /// + [FieldOffset(AofHeader.TotalSize + 2)] + public fixed byte replayTaskAccessVector[AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes]; + } + + /// + /// Used for sharded log to add a k + /// + [StructLayout(LayoutKind.Explicit, Size = TotalSize)] + struct AofShardedHeader + { + public const int TotalSize = AofHeader.TotalSize + 8; + + /// + /// Basic AOF header used with single log. + /// + [FieldOffset(0)] + public AofHeader basicHeader; + + /// + /// Used with multi-log to implement read consistency protocol. + /// + [FieldOffset(AofHeader.TotalSize)] + public long sequenceNumber; + }; + + /// + /// Basic AOF header + /// + [StructLayout(LayoutKind.Explicit, Size = TotalSize)] struct AofHeader { + public static unsafe byte* SkipHeader(byte* entryPtr) + { + var header = *(AofHeader*)entryPtr; + var headerType = header.HeaderType; + return headerType switch + { + AofHeaderType.BasicHeader => entryPtr + TotalSize, + AofHeaderType.ShardedHeader => entryPtr + AofShardedHeader.TotalSize, + AofHeaderType.ShardedLogTransactionHeader => entryPtr + AofShardedLogTransactionHeader.TotalSize, + AofHeaderType.SingleLogTransactionHeader => entryPtr + AofSingleLogTransactionHeader.TotalSize, + _ => throw new GarnetException($"Type not supported {headerType}"), + }; + } + + public const int TotalSize = 16; + // Important: Update version number whenever any of the following change: // * Layout, size, contents of this struct // * Any of the AofEntryType or AofStoreType enums' existing value mappings // * SpanByte format or header - const byte AofHeaderVersion = 2; + // Version 3 repurposes the flags byte as a bitfield containing the header type + // plus chunked-record and unsafe-truncate markers. + const byte AofHeaderVersion = 3; + + /// + /// Bits in that identify the + /// + internal const byte AofHeaderTypeMask = 0b0011; + + /// + /// Bit in that indicates that the record is chunked + /// + internal const byte ChunkedRecordFlag = 0b0100; + + /// + /// Bit in that indicates Unsafe truncate log (used with FLUSH command) + /// + internal const byte UnsafeTruncateLogFlag = 0b1000; /// /// Version of AOF @@ -20,20 +169,27 @@ struct AofHeader [FieldOffset(0)] public byte aofHeaderVersion; /// - /// Padding, for alignment and future use + /// Flags, for current and future use /// [FieldOffset(1)] - public byte padding; + public byte flags; /// /// Type of operation /// [FieldOffset(2)] public AofEntryType opType; + /// - /// Procedure ID + /// Procedure ID; union with /// [FieldOffset(3)] public byte procedureId; + /// + /// Database ID (used with FLUSH command); union with + /// + [FieldOffset(3)] + public byte databaseId; + /// /// Store version /// @@ -44,20 +200,36 @@ struct AofHeader /// [FieldOffset(12)] public int sessionID; + /// /// Unsafe truncate log (used with FLUSH command) /// - [FieldOffset(1)] - public byte unsafeTruncateLog; - /// - /// Database ID (used with FLUSH command) - /// - [FieldOffset(3)] - public byte databaseId; + public bool UnsafeTruncateLog + { + get => (flags & UnsafeTruncateLogFlag) != 0; + set + { + if (value) + flags |= UnsafeTruncateLogFlag; + else + flags = (byte)(flags & ~UnsafeTruncateLogFlag); + } + } + + public AofHeaderType HeaderType + { + get => (AofHeaderType)(flags & AofHeaderTypeMask); + set + { + Debug.Assert((int)value <= AofHeaderTypeMask, $"value {value} does not fit in AofHeaderTypeMask"); + flags = (byte)((flags & ~AofHeaderTypeMask) | (byte)value); + } + } public AofHeader() { - this.aofHeaderVersion = AofHeaderVersion; + flags = 0; + aofHeaderVersion = AofHeaderVersion; } } } \ No newline at end of file diff --git a/libs/server/AOF/AofProcessor.cs b/libs/server/AOF/AofProcessor.cs index 18fdb613c5b..3e368a0813b 100644 --- a/libs/server/AOF/AofProcessor.cs +++ b/libs/server/AOF/AofProcessor.cs @@ -2,21 +2,83 @@ // Licensed under the MIT license. using System; -using System.Buffers; using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Threading; +using System.Threading.Tasks; using Garnet.common; using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; + unsafe ref struct PreparedParameters + { + public Span Key; + public long KeyHash; + public byte* PayloadPtr; + } - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; + interface IPreprocessKey + { + public abstract unsafe void PrepareKey(int virtualSublogIdx, byte* entryPtr, long logAddressSequenceNumber, out PreparedParameters preparedParameters); + } + + struct SingleLogPreprocessKey : IPreprocessKey + { + public unsafe void PrepareKey(int virtualSublogIdx, byte* entryPtr, long logAddressSequenceNumber, out PreparedParameters preparedParameters) + { + var keyPtr = entryPtr + sizeof(AofHeader); + preparedParameters = new() + { + Key = SpanByte.FromLengthPrefixedPinnedPointer(keyPtr) + }; + preparedParameters.KeyHash = GarnetLog.HASH(preparedParameters.Key); + preparedParameters.PayloadPtr = keyPtr + preparedParameters.Key.TotalSize(); + } + } + + struct SinglePhysicalLogPreprocessKey : IPreprocessKey + { + public GarnetAppendOnlyFile appendOnlyFile; + + public unsafe void PrepareKey(int virtualSublogIdx, byte* entryPtr, long logAddressSequenceNumber, out PreparedParameters preparedParameters) + { + // Single physical log + multi-replay: entries use BasicHeader, ordering via log address + var keyPtr = entryPtr + sizeof(AofHeader); + preparedParameters = new() + { + Key = SpanByte.FromLengthPrefixedPinnedPointer(keyPtr) + }; + preparedParameters.KeyHash = GarnetLog.HASH(preparedParameters.Key); + preparedParameters.PayloadPtr = keyPtr + preparedParameters.Key.TotalSize(); + Debug.Assert(logAddressSequenceNumber > 0, "Entry address must be positive for single-physical-log consistency updates"); + appendOnlyFile.readConsistencyManager.UpdateVirtualSublogKeySequenceNumber( + virtualSublogIdx, + preparedParameters.KeyHash, + logAddressSequenceNumber); + } + } + + struct ShardedLogPreprocessKey : IPreprocessKey + { + public GarnetAppendOnlyFile appendOnlyFile; + + public unsafe void PrepareKey(int virtualSublogIdx, byte* entryPtr, long logAddressSequenceNumber, out PreparedParameters preparedParameters) + { + var shardedHeader = *(AofShardedHeader*)entryPtr; + var keyPtr = entryPtr + sizeof(AofShardedHeader); + preparedParameters = new() + { + Key = SpanByte.FromLengthPrefixedPinnedPointer(keyPtr) + }; + preparedParameters.KeyHash = GarnetLog.HASH(preparedParameters.Key); + preparedParameters.PayloadPtr = keyPtr + preparedParameters.Key.TotalSize(); + appendOnlyFile.readConsistencyManager.UpdateVirtualSublogKeySequenceNumber( + virtualSublogIdx, + preparedParameters.KeyHash, + shardedHeader.sequenceNumber); + } + } /// /// Wrapper for store and store-specific information @@ -24,36 +86,36 @@ namespace Garnet.server public sealed unsafe partial class AofProcessor { readonly StoreWrapper storeWrapper; - readonly RespServerSession respServerSession; readonly AofReplayCoordinator aofReplayCoordinator; int activeDbId; internal VectorManager activeVectorManager; + RangeIndexManager activeRangeIndexManager; /// /// Set ReadWriteSession on the cluster session (NOTE: used for replaying stored procedures only) /// public void SetReadWriteSession() { - respServerSession.clusterSession.SetReadWriteSession(); + for (var i = 0; i < storeWrapper.serverOptions.AofVirtualSublogCount; i++) + { + var respServerSession = aofReplayCoordinator.GetReplayContext(i).respServerSession; + respServerSession.clusterSession.SetReadWriteSession(); + } } - /// - /// Session for main store - /// - BasicContext basicContext; - - /// - /// Session for object store - /// - BasicContext objectStoreBasicContext; - + readonly StoreWrapper replayAofStoreWrapper; readonly IClusterProvider clusterProvider; - readonly ILogger logger; - readonly Func obtainServerSession; + readonly ILogger logger; + readonly bool usingShardedLog; + readonly bool usingSinglePhysicalLogMultiReplay; + SingleLogPreprocessKey singleLogPreprocessKey; + SinglePhysicalLogPreprocessKey singlePhysicalLogPreprocessKey; + ShardedLogPreprocessKey shardedLogPreprocessKey; + /// /// Create new AOF processor /// @@ -64,20 +126,25 @@ public AofProcessor( ILogger logger = null) { this.storeWrapper = storeWrapper; - this.clusterProvider = clusterProvider; - var replayAofStoreWrapper = new StoreWrapper(storeWrapper, recordToAof); - - obtainServerSession = () => new(0, networkSender: null, storeWrapper: replayAofStoreWrapper, subscribeBroker: null, authenticator: null, enableScripts: false, clusterProvider: clusterProvider); + this.replayAofStoreWrapper = new StoreWrapper(storeWrapper, recordToAof); this.activeDbId = 0; - this.respServerSession = obtainServerSession(); + this.usingShardedLog = storeWrapper.serverOptions.AofPhysicalSublogCount > 1 || storeWrapper.serverOptions.AofReplayTaskCount > 1; + this.usingSinglePhysicalLogMultiReplay = storeWrapper.serverOptions.AofPhysicalSublogCount == 1 && storeWrapper.serverOptions.AofReplayTaskCount > 1; + if (storeWrapper.serverOptions.AofPhysicalSublogCount > 1) + this.shardedLogPreprocessKey = new ShardedLogPreprocessKey() { appendOnlyFile = storeWrapper.appendOnlyFile }; + else if (usingSinglePhysicalLogMultiReplay) + this.singlePhysicalLogPreprocessKey = new SinglePhysicalLogPreprocessKey() { appendOnlyFile = storeWrapper.appendOnlyFile }; + else + this.singleLogPreprocessKey = new SingleLogPreprocessKey(); + this.obtainServerSession = () => new(0, networkSender: null, storeWrapper: replayAofStoreWrapper, subscribeBroker: null, authenticator: null, enableScripts: false, clusterProvider: clusterProvider); + + this.aofReplayCoordinator = new AofReplayCoordinator(storeWrapper.serverOptions, this, logger); + this.logger = logger; // Switch current contexts to match the default database SwitchActiveDatabaseContext(storeWrapper.DefaultDatabase, true); - - aofReplayCoordinator = new AofReplayCoordinator(this, logger); - this.logger = logger; } /// @@ -87,97 +154,31 @@ public void Dispose() { activeVectorManager?.WaitForVectorOperationsToComplete(); activeVectorManager?.ShutdownReplayTasks(); - - aofReplayCoordinator.Dispose(); - respServerSession.Dispose(); + aofReplayCoordinator?.Dispose(); } - /// - /// Recover store using AOF - /// - /// Database to recover - /// Tail address for recovery - /// Tail address - public long Recover(GarnetDatabase db, long untilAddress = -1) - { - var start = Stopwatch.GetTimestamp(); - var total_number_of_replayed_records = 0L; - try - { - logger?.LogInformation("Begin AOF recovery for DB ID: {id}", db.Id); - return RecoverReplay(db, untilAddress); - } - finally - { - var end = Stopwatch.GetTimestamp(); - var elapsed = Stopwatch.GetElapsedTime(start, end); - var seconds = elapsed.TotalMilliseconds / 1000.0; - var aofSize = db.AppendOnlyFile.TailAddress - db.AppendOnlyFile.BeginAddress; - var recordsPerSec = total_number_of_replayed_records / seconds; - var gigabytesPerSec = (aofSize / seconds) / (double)1_000_000_000; - - logger?.LogInformation("AOF Recovery in {seconds} secs", seconds); - logger?.LogInformation("Total number of replayed records {total_number_of_replayed_records:N0} bytes", total_number_of_replayed_records); - logger?.LogInformation("Throughput {recordsPerSec:N2} records/sec", recordsPerSec); - logger?.LogInformation("AOF Recovery size {aofSize:N0}", aofSize); - logger?.LogInformation("AOF Recovery throughput {GiBperSecs:N2} GiB/secs", gigabytesPerSec); - } + private RespServerSession ObtainServerSession() + => new(0, networkSender: null, storeWrapper: replayAofStoreWrapper, subscribeBroker: null, authenticator: null, enableScripts: false, clusterProvider: clusterProvider); - long RecoverReplay(GarnetDatabase db, long untilAddress) + private void SwitchActiveDatabaseContext(GarnetDatabase db, bool initialSetup = false) + { + for (var i = 0; i < storeWrapper.serverOptions.AofVirtualSublogCount; i++) { - // Begin replay for specified database - logger?.LogInformation("Begin AOF replay for DB ID: {id}", db.Id); - try + var respServerSession = aofReplayCoordinator.GetReplayContext(i).respServerSession; + // Switch the session's context to match the specified database, if necessary + if (respServerSession.activeDbId != db.Id) { - // Fetch the database AOF and update the current database context for the processor - var appendOnlyFile = db.AppendOnlyFile; - SwitchActiveDatabaseContext(db); - - // Set the tail address for replay recovery to the tail address of the AOF if none specified - if (untilAddress == -1) - untilAddress = appendOnlyFile.TailAddress; - - // Run recover replay task - RecoverReplayTask(untilAddress); - - void RecoverReplayTask(long untilAddress) - { - var count = 0; - using var scan = appendOnlyFile.Scan(appendOnlyFile.BeginAddress, untilAddress); - - // Replay each AOF record in the current database context - while (scan.GetNext(MemoryPool.Shared, out var entry, out var length, out _, out long nextAofAddress)) - { - count++; - ProcessAofRecord(entry, length); - if (count % 100_000 == 0) - logger?.LogTrace("Completed AOF replay of {count} records, until AOF address {nextAofAddress} (DB ID: {id})", count, nextAofAddress, db.Id); - } - - logger?.LogInformation("Completed full AOF sublog replay of {count:N0} records (DB ID: {id})", count, db.Id); - _ = Interlocked.Add(ref total_number_of_replayed_records, count); - } - - unsafe void ProcessAofRecord(IMemoryOwner entry, int length) - { - fixed (byte* ptr = entry.Memory.Span) - { - ProcessAofRecordInternal(ptr, length, asReplica: false, out _); - } - entry.Dispose(); - } - - return untilAddress; + var switchDbSuccessful = respServerSession.TrySwitchActiveDatabaseSession(db.Id); + Debug.Assert(switchDbSuccessful); } - catch (Exception ex) - { - logger?.LogError(ex, "An error occurred AofProcessor.RecoverReplay"); - if (storeWrapper.serverOptions.FailOnRecoveryError) - throw; + // Switch the storage context to match the session, if necessary + if (activeDbId != db.Id || initialSetup) + { + activeDbId = db.Id; + activeVectorManager = db.VectorManager; + activeRangeIndexManager = db.RangeIndexManager; } - - return -1; } } @@ -185,29 +186,65 @@ unsafe void ProcessAofRecord(IMemoryOwner entry, int length) /// Wait for any queued Vector Set operations to complete. /// public void WaitForVectorOperationsToComplete() - => activeVectorManager?.WaitForVectorOperationsToComplete(); + => activeVectorManager?.WaitForVectorOperationsToComplete(); + + /// + /// Extracts sequence number and participant count from a transaction header entry. + /// For single-physical-log + multi-replay, uses entry address; for multi-physical-log, uses embedded sequence number. + /// + void GetSynchronizedOperationParams(byte* ptr, long entryAddress, out long sequenceNumber, out short participantCount) + { + var headerType = (*(AofHeader*)ptr).HeaderType; + switch (headerType) + { + case AofHeaderType.SingleLogTransactionHeader: + sequenceNumber = entryAddress; + participantCount = (*(AofSingleLogTransactionHeader*)ptr).participantCount; + break; + case AofHeaderType.ShardedLogTransactionHeader: + var txnHeader = *(AofShardedLogTransactionHeader*)ptr; + sequenceNumber = txnHeader.shardedHeader.sequenceNumber; + participantCount = txnHeader.participantCount; + break; + case AofHeaderType.BasicHeader: + sequenceNumber = entryAddress; + participantCount = (short)storeWrapper.serverOptions.AofReplayTaskCount; + break; + case AofHeaderType.ShardedHeader: + sequenceNumber = (*(AofShardedHeader*)ptr).sequenceNumber; + participantCount = (short)storeWrapper.serverOptions.AofReplayTaskCount; + break; + default: + throw new GarnetException($"Unsupported header type: {headerType}"); + } + } /// /// Process AOF record internal + /// NOTE: This method is shared between recover replay and replication replay /// + /// /// /// /// /// - public unsafe void ProcessAofRecordInternal(byte* ptr, int length, bool asReplica, out bool isCheckpointStart) + /// + public void ProcessAofRecordInternal(int virtualSublogIdx, byte* ptr, int length, bool asReplica, out bool isCheckpointStart, long logAddressSequenceNumber = 0) { var header = *(AofHeader*)ptr; - var replayContext = aofReplayCoordinator.GetReplayContext(); + var replayContext = aofReplayCoordinator.GetReplayContext(virtualSublogIdx); isCheckpointStart = false; - // Aggressively do not move data if VADD are being replayed + // StoreRMW can queue VADDs onto different threads + // but everything else needs to WAIT for those to complete + // otherwise we might loose consistency if (header.opType != AofEntryType.StoreRMW) { activeVectorManager.WaitForVectorOperationsToComplete(); } // Handle transactions - if (aofReplayCoordinator.AddOrReplayTransactionOperation(ptr, length, asReplica)) + if (aofReplayCoordinator.AddOrReplayTransactionOperation(virtualSublogIdx, ptr, length, asReplica, logAddressSequenceNumber)) return; switch (header.opType) @@ -219,13 +256,19 @@ public unsafe void ProcessAofRecordInternal(byte* ptr, int length, bool asReplic { if (replayContext.inFuzzyRegion) { - logger?.LogInformation("Encountered new CheckpointStartCommit before prior CheckpointEndCommit. Clearing {fuzzyRegionBufferCount} records from previous fuzzy region", - aofReplayCoordinator.FuzzyRegionBufferCount()); - aofReplayCoordinator.ClearFuzzyRegionBuffer(); + logger?.LogInformation("Encountered new CheckpointStartCommit before prior CheckpointEndCommit. Clearing {fuzzyRegionBufferCount} records from previous fuzzy region", aofReplayCoordinator.FuzzyRegionBufferCount(virtualSublogIdx)); + aofReplayCoordinator.ClearFuzzyRegionBuffer(virtualSublogIdx); } Debug.Assert(!replayContext.inFuzzyRegion); replayContext.inFuzzyRegion = true; } + + if (usingShardedLog) + { + // For single-physical-log + multi-replay, use entry address; otherwise use embedded sequence number + var checkpointSequenceNumber = usingSinglePhysicalLogMultiReplay ? logAddressSequenceNumber : (*(AofShardedHeader*)ptr).sequenceNumber; + storeWrapper.appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(virtualSublogIdx, checkpointSequenceNumber); + } break; case AofEntryType.CheckpointEndCommit: if (header.aofHeaderVersion > 1) @@ -236,57 +279,162 @@ public unsafe void ProcessAofRecordInternal(byte* ptr, int length, bool asReplic } else { + Debug.Assert(replayContext.inFuzzyRegion); replayContext.inFuzzyRegion = false; // Take checkpoint after the fuzzy region if (asReplica && header.storeVersion > storeWrapper.store.CurrentVersion) { - // Must block here, cannot move off the thread - _ = AsyncUtils.BlockingWait(storeWrapper.TakeCheckpointAsync(background: false, logger)); + if (!usingShardedLog) + { + // Must block here, cannot move off the thread + _ = AsyncUtils.BlockingWait(storeWrapper.TakeCheckpointAsync(background: false, logger: logger)); + } + else + { + GetSynchronizedOperationParams(ptr, logAddressSequenceNumber, out var seqNum, out var partCount); + aofReplayCoordinator.ProcessSynchronizedOperation( + virtualSublogIdx, + seqNum, + partCount, + (int)LeaderBarrierType.CHECKPOINT, + () => storeWrapper.TakeCheckpointAsync(background: false, logger: logger)); + } } // Process buffered records - aofReplayCoordinator.ProcessFuzzyRegionOperations(storeWrapper.store.CurrentVersion, asReplica); - aofReplayCoordinator.ClearFuzzyRegionBuffer(); + aofReplayCoordinator.ProcessFuzzyRegionOperations(virtualSublogIdx, storeWrapper.store.CurrentVersion, asReplica); + aofReplayCoordinator.ClearFuzzyRegionBuffer(virtualSublogIdx); } } break; case AofEntryType.MainStoreStreamingCheckpointStartCommit: - Debug.Assert(storeWrapper.serverOptions.ReplicaDisklessSync); - if (asReplica && header.storeVersion > storeWrapper.store.CurrentVersion) - { - storeWrapper.store.SetVersion(header.storeVersion); - } - break; case AofEntryType.ObjectStoreStreamingCheckpointStartCommit: Debug.Assert(storeWrapper.serverOptions.ReplicaDisklessSync); if (asReplica && header.storeVersion > storeWrapper.store.CurrentVersion) { - storeWrapper.objectStore.SetVersion(header.storeVersion); + if (!usingShardedLog) + { + storeWrapper.store.SetVersion(header.storeVersion); + } + else + { + GetSynchronizedOperationParams(ptr, logAddressSequenceNumber, out var seqNum, out var partCount); + aofReplayCoordinator.ProcessSynchronizedOperation( + virtualSublogIdx, + seqNum, + partCount, + (int)LeaderBarrierType.STREAMING_CHECKPOINT, + () => { storeWrapper.store.SetVersion(header.storeVersion); return Task.CompletedTask; } + ); + } } break; case AofEntryType.MainStoreStreamingCheckpointEndCommit: case AofEntryType.ObjectStoreStreamingCheckpointEndCommit: Debug.Assert(storeWrapper.serverOptions.ReplicaDisklessSync); + if (usingShardedLog) + { + // For single-physical-log + multi-replay, use entry address; otherwise use embedded sequence number + var streamingSequenceNumber = usingSinglePhysicalLogMultiReplay ? logAddressSequenceNumber : (*(AofShardedHeader*)ptr).sequenceNumber; + storeWrapper.appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(virtualSublogIdx, streamingSequenceNumber); + } break; case AofEntryType.FlushAll: - storeWrapper.FlushAllDatabases(unsafeTruncateLog: header.unsafeTruncateLog == 1); + if (!usingShardedLog) + { + storeWrapper.FlushAllDatabases(unsafeTruncateLog: header.UnsafeTruncateLog); + } + else + { + GetSynchronizedOperationParams(ptr, logAddressSequenceNumber, out var seqNum, out var partCount); + aofReplayCoordinator.ProcessSynchronizedOperation( + virtualSublogIdx, + seqNum, + partCount, + (int)LeaderBarrierType.FLUSH_DB_ALL, + () => { storeWrapper.FlushAllDatabases(unsafeTruncateLog: header.UnsafeTruncateLog); return Task.CompletedTask; } + ); + } break; case AofEntryType.FlushDb: - storeWrapper.FlushDatabase(unsafeTruncateLog: header.unsafeTruncateLog == 1, dbId: header.databaseId); + if (!usingShardedLog) + { + storeWrapper.FlushDatabase(unsafeTruncateLog: header.UnsafeTruncateLog, dbId: header.databaseId); + } + else + { + GetSynchronizedOperationParams(ptr, logAddressSequenceNumber, out var seqNum, out var partCount); + aofReplayCoordinator.ProcessSynchronizedOperation( + virtualSublogIdx, + seqNum, + partCount, + (int)LeaderBarrierType.FLUSH_DB, + () => { storeWrapper.FlushDatabase(unsafeTruncateLog: header.UnsafeTruncateLog, dbId: header.databaseId); return Task.CompletedTask; } + ); + } + break; + case AofEntryType.StoredProcedure: + aofReplayCoordinator.ReplayStoredProc(virtualSublogIdx, header.procedureId, ptr, logAddressSequenceNumber); + break; + case AofEntryType.TxnCommit: + aofReplayCoordinator.ProcessFuzzyRegionTransactionGroup(virtualSublogIdx, ptr, asReplica, logAddressSequenceNumber); break; default: - _ = ReplayOp(basicContext, objectStoreBasicContext, ptr, length, asReplica); + _ = ReplayOpDispatch( + virtualSublogIdx, + header, + replayContext, + replayContext.StringBasicContext, + replayContext.ObjectBasicContext, + replayContext.UnifiedBasicContext, + ptr, + length, + asReplica, + logAddressSequenceNumber); break; } } - private unsafe bool ReplayOp(TContext storeContext, TObjectContext objectStoreContext, byte* entryPtr, int length, bool asReplica) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext + internal bool ReplayOpDispatch( + int virtualSublogIdx, + AofHeader header, + AofReplayContext replayContext, + TStringContext stringContext, + TObjectContext objectContext, + TUnifiedContext unifiedContext, + byte* entryPtr, + int length, + bool asReplica, + long logAddressSequenceNumber = 0) + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext { - var header = *(AofHeader*)entryPtr; - var replayContext = aofReplayCoordinator.GetReplayContext(); + if (storeWrapper.serverOptions.AofPhysicalSublogCount > 1) + return ReplayOp(virtualSublogIdx, header, replayContext, shardedLogPreprocessKey, stringContext, objectContext, unifiedContext, entryPtr, length, asReplica, logAddressSequenceNumber); + else if (usingSinglePhysicalLogMultiReplay) + return ReplayOp(virtualSublogIdx, header, replayContext, singlePhysicalLogPreprocessKey, stringContext, objectContext, unifiedContext, entryPtr, length, asReplica, logAddressSequenceNumber); + else + return ReplayOp(virtualSublogIdx, header, replayContext, singleLogPreprocessKey, stringContext, objectContext, unifiedContext, entryPtr, length, asReplica, logAddressSequenceNumber); + } + private bool ReplayOp( + int virtualSublogIdx, + AofHeader header, + AofReplayContext replayContext, + TPreprocessKey preprocessKey, + TStringContext stringContext, + TObjectContext objectContext, + TUnifiedContext unifiedContext, + byte* entryPtr, + int length, + bool asReplica, + long logAddressSequenceNumber) + where TPreprocessKey : IPreprocessKey + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext + { // StoreRMW can queue VADDs onto different threads // but everything else needs to WAIT for those to complete // otherwise we might loose consistency @@ -296,36 +444,43 @@ private unsafe bool ReplayOp(TContext storeContext, TO } // Skips (1) entries with versions that were part of prior checkpoint; and (2) future entries in fuzzy region - if (SkipRecord(replayContext.inFuzzyRegion, entryPtr, length, asReplica)) + if (SkipRecord(virtualSublogIdx, replayContext.inFuzzyRegion, entryPtr, length, asReplica)) return false; var bufferPtr = (byte*)Unsafe.AsPointer(ref replayContext.objectOutputBuffer[0]); var bufferLength = replayContext.objectOutputBuffer.Length; + preprocessKey.PrepareKey(virtualSublogIdx, entryPtr, logAddressSequenceNumber, out var preparedParameters); switch (header.opType) { case AofEntryType.StoreUpsert: - StoreUpsert(storeContext, replayContext.storeInput, entryPtr + sizeof(AofHeader)); + StoreUpsert(preparedParameters, stringContext, ref replayContext.parseState); break; case AofEntryType.StoreRMW: - StoreRMW(storeContext, replayContext.storeInput, activeVectorManager, respServerSession, obtainServerSession, entryPtr + sizeof(AofHeader)); + StoreRMW(preparedParameters, stringContext, ref replayContext.parseState, activeVectorManager, activeRangeIndexManager, replayContext.respServerSession, obtainServerSession); break; case AofEntryType.StoreDelete: - StoreDelete(storeContext, activeVectorManager, respServerSession.storageSession, entryPtr + sizeof(AofHeader)); - break; - case AofEntryType.ObjectStoreRMW: - ObjectStoreRMW(objectStoreContext, replayContext.objectStoreInput, entryPtr + sizeof(AofHeader), bufferPtr, bufferLength); + StoreDelete(preparedParameters, stringContext); break; case AofEntryType.ObjectStoreUpsert: - ObjectStoreUpsert(objectStoreContext, storeWrapper.GarnetObjectSerializer, entryPtr + sizeof(AofHeader), bufferPtr, bufferLength); + ObjectStoreUpsert(preparedParameters, objectContext, storeWrapper.GarnetObjectSerializer, bufferPtr, bufferLength); + break; + case AofEntryType.ObjectStoreRMW: + ObjectStoreRMW(preparedParameters, objectContext, ref replayContext.parseState, bufferPtr, bufferLength); break; case AofEntryType.ObjectStoreDelete: - ObjectStoreDelete(objectStoreContext, entryPtr + sizeof(AofHeader)); + ObjectStoreDelete(preparedParameters, objectContext); break; - case AofEntryType.StoredProcedure: - aofReplayCoordinator.ReplayStoredProc(header.procedureId, entryPtr); + case AofEntryType.UnifiedStoreStringUpsert: + UnifiedStoreStringUpsert(preparedParameters, unifiedContext, ref replayContext.parseState, bufferPtr, bufferLength); break; - case AofEntryType.TxnCommit: - aofReplayCoordinator.ProcessFuzzyRegionTransactionGroup(entryPtr, asReplica); + case AofEntryType.UnifiedStoreRMW: + UnifiedStoreRMW(preparedParameters, unifiedContext, ref replayContext.parseState, bufferPtr, bufferLength); + break; + case AofEntryType.UnifiedStoreObjectUpsert: + UnifiedStoreObjectUpsert(preparedParameters, unifiedContext, storeWrapper.GarnetObjectSerializer, bufferPtr, bufferLength); + break; + case AofEntryType.UnifiedStoreDelete: + UnifiedStoreDelete(preparedParameters, unifiedContext, activeVectorManager, replayContext.respServerSession.storageSession); break; default: throw new GarnetException($"Unknown AOF header operation type {header.opType}"); @@ -334,71 +489,41 @@ private unsafe bool ReplayOp(TContext storeContext, TO return true; } - private void SwitchActiveDatabaseContext(GarnetDatabase db, bool initialSetup = false) + static void StoreUpsert(PreparedParameters preparedParameters, TStringContext stringContext, ref SessionParseState parseState) + where TStringContext : ITsavoriteContext { - // Switch the session's context to match the specified database, if necessary - if (respServerSession.activeDbId != db.Id) - { - var switchDbSuccessful = respServerSession.TrySwitchActiveDatabaseSession(db.Id); - Debug.Assert(switchDbSuccessful); - } - - // Switch the storage context to match the session, if necessary - if (this.activeDbId != db.Id || initialSetup) - { - var session = respServerSession.storageSession.basicContext.Session; - basicContext = session.BasicContext; - var objectStoreSession = respServerSession.storageSession.objectStoreBasicContext.Session; - if (objectStoreSession is not null) - objectStoreBasicContext = objectStoreSession.BasicContext; - this.activeDbId = db.Id; - } - - activeVectorManager = db.VectorManager; - } - - static void StoreUpsert( - TContext context, - RawStringInput storeInput, - byte* ptr) - where TContext : ITsavoriteContext - { - var curr = ptr; - ref var key = ref Unsafe.AsRef(curr); - curr += key.TotalSize; - - ref var value = ref Unsafe.AsRef(curr); + var curr = preparedParameters.PayloadPtr; + var value = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); curr += value.TotalSize; - // Reconstructing RawStringInput - _ = storeInput.DeserializeFrom(curr); + var stringInput = new StringInput { parseState = parseState }; + _ = stringInput.DeserializeFrom(curr); - SpanByteAndMemory output = default; - context.Upsert(ref key, ref storeInput, ref value, ref output); - if (!output.IsSpanByte) - output.Memory.Dispose(); + StringOutput output = default; + var upsertOptions = new UpsertOptions() { KeyHash = preparedParameters.KeyHash }; + _ = stringContext.Upsert((FixedSpanByteKey)preparedParameters.Key, ref stringInput, value, ref output, ref upsertOptions); + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Dispose(); } - static void StoreRMW( - TContext context, - RawStringInput storeInput, + static void StoreRMW( + PreparedParameters preparedParameters, + TStringContext stringContext, + ref SessionParseState parseState, VectorManager vectorManager, - RespServerSession currentSession, - Func obtainServerSession, - byte* ptr) - where TContext : ITsavoriteContext + RangeIndexManager rangeIndexManager, + RespServerSession activeServerSession, + Func obtainServerSession) + where TStringContext : ITsavoriteContext { - var curr = ptr; - ref var key = ref Unsafe.AsRef(curr); - curr += key.TotalSize; - - // Reconstructing RawStringInput - _ = storeInput.DeserializeFrom(curr); + var curr = preparedParameters.PayloadPtr; + var stringInput = new StringInput { parseState = parseState }; + _ = stringInput.DeserializeFrom(curr); // VADD requires special handling, shove it over to the VectorManager - if (storeInput.header.cmd == RespCommand.VADD) + if (stringInput.header.cmd == RespCommand.VADD) { - vectorManager.HandleVectorSetAddReplication(currentSession.storageSession, obtainServerSession, ref key, ref storeInput); + vectorManager.HandleVectorSetAddReplication(activeServerSession.storageSession, obtainServerSession, preparedParameters.Key, ref stringInput); return; } else @@ -407,160 +532,289 @@ static void StoreRMW( vectorManager.WaitForVectorOperationsToComplete(); // VREM is also read-like, so requires special handling - shove it over to the VectorManager - if (storeInput.header.cmd == RespCommand.VREM) + if (stringInput.header.cmd == RespCommand.VREM) { - vectorManager.HandleVectorSetRemoveReplication(currentSession.storageSession, ref key, ref storeInput); + vectorManager.HandleVectorSetRemoveReplication(activeServerSession.storageSession, preparedParameters.Key, ref stringInput); return; } } - var pbOutput = stackalloc byte[32]; - var output = new SpanByteAndMemory(pbOutput, 32); - - if (context.RMW(ref key, ref storeInput, ref output).IsPending) - _ = context.CompletePending(true); + // RangeIndex commands need actual execution on replay + if (stringInput.header.cmd == RespCommand.RICREATE) + { + rangeIndexManager?.HandleRangeIndexCreateReplay(activeServerSession.storageSession, preparedParameters.Key, ref stringInput); + return; + } + if (stringInput.header.cmd == RespCommand.RISET) + { + rangeIndexManager.HandleRangeIndexSetReplay(activeServerSession.storageSession, preparedParameters.Key, ref stringInput); + return; + } + if (stringInput.header.cmd == RespCommand.RIDEL) + { + rangeIndexManager.HandleRangeIndexDelReplay(activeServerSession.storageSession, preparedParameters.Key, ref stringInput); + return; + } - if (!output.IsSpanByte) - output.Memory.Dispose(); + var output = StringOutput.FromPinnedSpan(stackalloc byte[32]); + var rmwOptions = new RMWOptions { KeyHash = preparedParameters.KeyHash }; + var status = stringContext.RMW((FixedSpanByteKey)preparedParameters.Key, ref stringInput, ref output, ref rmwOptions); + if (status.IsPending) + StorageSession.CompletePendingForSession(ref status, ref output, ref stringContext); + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Dispose(); } - static void StoreDelete( - TContext context, - VectorManager vectorManager, - StorageSession storageSession, - byte* ptr) - where TContext : ITsavoriteContext + static void StoreDelete(PreparedParameters preparedParameters, TStringContext stringContext) + where TStringContext : ITsavoriteContext + => stringContext.Delete((FixedSpanByteKey)preparedParameters.Key); + + static void ObjectStoreUpsert(PreparedParameters preparedParameters, TObjectContext objectContext, GarnetObjectSerializer garnetObjectSerializer, byte* outputPtr, int outputLength) + where TObjectContext : ITsavoriteContext { - ref var key = ref Unsafe.AsRef(ptr); - var res = context.Delete(ref key); + var curr = preparedParameters.PayloadPtr; + var valueSpan = SpanByte.FromLengthPrefixedPinnedPointer(curr); + var valueObject = garnetObjectSerializer.Deserialize(valueSpan.ToArray()); // TODO native deserializer to avoid alloc and copy - if (res.IsCanceled) - { - // Might be a vector set - res = vectorManager.TryDeleteVectorSet(storageSession, ref key, out _); - if (res.IsPending) - _ = context.CompletePending(true); - } + var output = ObjectOutput.FromPinnedPointer(outputPtr, outputLength); + var upsertOptions = new UpsertOptions() { KeyHash = preparedParameters.KeyHash }; + _ = objectContext.Upsert((FixedSpanByteKey)preparedParameters.Key, valueObject, ref upsertOptions); + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Dispose(); } - static void ObjectStoreUpsert( - TObjectContext objectContext, - GarnetObjectSerializer garnetObjectSerializer, - byte* ptr, - byte* outputPtr, - int outputLength) - where TObjectContext : ITsavoriteContext + static void ObjectStoreRMW(PreparedParameters preparedParameters, TObjectContext objectContext, ref SessionParseState parseState, byte* outputPtr, int outputLength) + where TObjectContext : ITsavoriteContext { - ref var key = ref Unsafe.AsRef(ptr); - var keyB = key.ToByteArray(); + var curr = preparedParameters.PayloadPtr; + + var objectInput = new ObjectInput { parseState = parseState }; + _ = objectInput.DeserializeFrom(curr); - ref var value = ref Unsafe.AsRef(ptr + key.TotalSize); - var valB = garnetObjectSerializer.Deserialize(value.ToByteArray()); + // Call RMW with the reconstructed key & ObjectInput + var output = ObjectOutput.FromPinnedPointer(outputPtr, outputLength); + var rmwOptions = new RMWOptions { KeyHash = preparedParameters.KeyHash }; + var status = objectContext.RMW((FixedSpanByteKey)preparedParameters.Key, ref objectInput, ref output, ref rmwOptions); + if (status.IsPending) + StorageSession.CompletePendingForObjectStoreSession(ref status, ref output, ref objectContext); - var output = new GarnetObjectStoreOutput(new(outputPtr, outputLength)); - _ = objectContext.Upsert(ref keyB, ref valB); if (!output.SpanByteAndMemory.IsSpanByte) - output.SpanByteAndMemory.Memory.Dispose(); + output.SpanByteAndMemory.Dispose(); } - static void ObjectStoreRMW( - TObjectContext objectContext, - ObjectInput objectStoreInput, - byte* ptr, - byte* outputPtr, - int outputLength) - where TObjectContext : ITsavoriteContext + static void ObjectStoreDelete(PreparedParameters preparedParameters, TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => objectContext.Delete((FixedSpanByteKey)preparedParameters.Key); + + static void UnifiedStoreStringUpsert(PreparedParameters preparedParameters, TUnifiedContext unifiedContext, ref SessionParseState parseState, byte* outputPtr, int outputLength) + where TUnifiedContext : ITsavoriteContext { - var curr = ptr; - ref var key = ref Unsafe.AsRef(curr); - curr += key.TotalSize; - var keyB = key.ToByteArray(); + var curr = preparedParameters.PayloadPtr; - // Reconstructing ObjectInput - _ = objectStoreInput.DeserializeFrom(curr); + var value = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + curr += value.TotalSize; - // Call RMW with the reconstructed key & ObjectInput - var output = new GarnetObjectStoreOutput(new(outputPtr, outputLength)); - if (objectContext.RMW(ref keyB, ref objectStoreInput, ref output).IsPending) - _ = objectContext.CompletePending(true); + var unifiedInput = new UnifiedInput { parseState = parseState }; + _ = unifiedInput.DeserializeFrom(curr); + var output = UnifiedOutput.FromPinnedPointer(outputPtr, outputLength); + var upsertOptions = new UpsertOptions() { KeyHash = preparedParameters.KeyHash }; + _ = unifiedContext.Upsert((FixedSpanByteKey)preparedParameters.Key, ref unifiedInput, value, ref output, ref upsertOptions); if (!output.SpanByteAndMemory.IsSpanByte) - output.SpanByteAndMemory.Memory.Dispose(); + output.SpanByteAndMemory.Dispose(); } - static void ObjectStoreDelete( - TObjectContext objectContext, - byte* ptr) - where TObjectContext : ITsavoriteContext + static void UnifiedStoreObjectUpsert(PreparedParameters preparedParameters, TUnifiedContext unifiedContext, GarnetObjectSerializer garnetObjectSerializer, byte* outputPtr, int outputLength) + where TUnifiedContext : ITsavoriteContext { - ref var key = ref Unsafe.AsRef(ptr); - var keyB = key.ToByteArray(); - _ = objectContext.Delete(ref keyB); + var curr = preparedParameters.PayloadPtr; + + var valueSpan = SpanByte.FromLengthPrefixedPinnedPointer(curr); + var valueObject = garnetObjectSerializer.Deserialize(valueSpan.ToArray()); // TODO native deserializer to avoid alloc and copy + + var output = UnifiedOutput.FromPinnedPointer(outputPtr, outputLength); + var upsertOptions = new UpsertOptions() { KeyHash = preparedParameters.KeyHash }; + _ = unifiedContext.Upsert((FixedSpanByteKey)preparedParameters.Key, valueObject, ref upsertOptions); + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Dispose(); } + static void UnifiedStoreRMW(PreparedParameters preparedParameters, TUnifiedContext unifiedContext, ref SessionParseState parseState, byte* outputPtr, int outputLength) + where TUnifiedContext : ITsavoriteContext + { + var curr = preparedParameters.PayloadPtr; + var unifiedInput = new UnifiedInput { parseState = parseState }; + _ = unifiedInput.DeserializeFrom(curr); + + // Call RMW with the reconstructed key & UnifiedInput + var output = UnifiedOutput.FromPinnedPointer(outputPtr, outputLength); + var rmwOptions = new RMWOptions { KeyHash = preparedParameters.KeyHash }; + var status = unifiedContext.RMW((FixedSpanByteKey)preparedParameters.Key, ref unifiedInput, ref output, ref rmwOptions); + if (status.IsPending) + StorageSession.CompletePendingForUnifiedStoreSession(ref status, ref output, ref unifiedContext); + + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Dispose(); + } + + static void UnifiedStoreDelete(PreparedParameters preparedParameters, TUnifiedContext unifiedContext, VectorManager vectorManager, StorageSession storageSession) + where TUnifiedContext : ITsavoriteContext + => unifiedContext.Delete((FixedSpanByteKey)preparedParameters.Key); + /// /// On recovery apply records with header.version greater than CurrentVersion. /// + /// /// /// /// /// /// /// - bool SkipRecord(bool inFuzzyRegion, byte* entryPtr, int length, bool asReplica) + bool SkipRecord(int sublogIdx, bool inFuzzyRegion, byte* entryPtr, int length, bool asReplica) { var header = *(AofHeader*)entryPtr; return (asReplica && inFuzzyRegion) ? // Buffer logic only for AOF version > 1 - BufferNewVersionRecord(header, entryPtr, length) : + BufferNewVersionRecord(sublogIdx, header, entryPtr, length) : IsOldVersionRecord(header); - bool BufferNewVersionRecord(AofHeader header, byte* entryPtr, int length) + bool BufferNewVersionRecord(int sublogIdx, AofHeader header, byte* entryPtr, int length) { if (IsNewVersionRecord(header)) { - aofReplayCoordinator.AddFuzzyRegionOperation(new ReadOnlySpan(entryPtr, length)); + aofReplayCoordinator.AddFuzzyRegionOperation(sublogIdx, new ReadOnlySpan(entryPtr, length)); return true; } return false; } bool IsOldVersionRecord(AofHeader header) - { - var storeType = ToAofStoreType(header.opType); + => header.storeVersion < storeWrapper.store.CurrentVersion; - return storeType switch - { - AofStoreType.MainStoreType => header.storeVersion < storeWrapper.store.CurrentVersion, - AofStoreType.ObjectStoreType => header.storeVersion < storeWrapper.objectStore.CurrentVersion, - AofStoreType.TxnType => header.storeVersion < storeWrapper.objectStore.CurrentVersion, - _ => throw new GarnetException($"Unexpected AOF header store type {storeType}"), - }; + bool IsNewVersionRecord(AofHeader header) + => header.storeVersion > storeWrapper.store.CurrentVersion; + } + + /// + /// Check if the calling parallel replay task should replay this entry + /// + /// + /// + /// Log address of the entry, used as ordering value for single-physical-log parallel replay mode + /// + /// + /// + public bool CanReplay(byte* ptr, int replayTaskIdx, long entryAddress, out long logAddressSequenceNumber) + { + var header = *(AofHeader*)ptr; + var replayHeaderType = header.HeaderType; + logAddressSequenceNumber = 0L; + switch (replayHeaderType) + { + // Single-physical-log + multi-replay: BasicHeader entries, use entry address for ordering + case AofHeaderType.BasicHeader: + logAddressSequenceNumber = entryAddress; + // Keyless entries (transactions, checkpoints, flush, stored procedures) are processed by all tasks + // because they may participate in barriers via ProcessSynchronizedOperation + if (!header.opType.HasKey()) + return true; + var basicCurr = AofHeader.SkipHeader(ptr); + var basicKey = PinnedSpanByte.FromLengthPrefixedPinnedPointer(basicCurr).ReadOnlySpan; + return replayTaskIdx == storeWrapper.appendOnlyFile.Log.GetReplayTaskIdx(basicKey); + // Multi-physical-log: ShardedHeader entries with embedded sequence number + case AofHeaderType.ShardedHeader: + var shardedHeader = *(AofShardedHeader*)ptr; + logAddressSequenceNumber = shardedHeader.sequenceNumber; + // Keyless entries are processed by task 0 only + if (!header.opType.HasKey()) + return replayTaskIdx == 0; + var curr = AofHeader.SkipHeader(ptr); + var key = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr).ReadOnlySpan; + return replayTaskIdx == storeWrapper.appendOnlyFile.Log.GetReplayTaskIdx(key); + // Single-physical-log + multi-replay: transaction header without sequence number + case AofHeaderType.SingleLogTransactionHeader: + var singleLogTxnHeader = *(AofSingleLogTransactionHeader*)ptr; + logAddressSequenceNumber = entryAddress; + var singleLogBitVector = BitVector.CopyFrom(new Span(singleLogTxnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + return singleLogBitVector.IsSet(replayTaskIdx); + // Multi-physical-log: transaction header with embedded sequence number + case AofHeaderType.ShardedLogTransactionHeader: + var txnHeader = *(AofShardedLogTransactionHeader*)ptr; + logAddressSequenceNumber = txnHeader.shardedHeader.sequenceNumber; + var bitVector = BitVector.CopyFrom(new Span(txnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + return bitVector.IsSet(replayTaskIdx); + default: + throw new GarnetException($"Replay header type {replayHeaderType} not supported!"); } + } - bool IsNewVersionRecord(AofHeader header) + /// + /// Calculates the index of the replay task associated with the specified AOF header pointer. + /// + /// A pointer to a byte array representing the AOF header. + /// The zero-based index of the replay task to which the entry should be assigned. Returns -1 if the header type + /// does not contain a key for task assignment. + /// Thrown when the AOF header type referenced by is not supported. + public int GetReplayTaskIdx(byte* ptr) + { + var header = *(AofHeader*)ptr; + var replayHeaderType = header.HeaderType; + switch (replayHeaderType) { - var storeType = ToAofStoreType(header.opType); - return storeType switch - { - AofStoreType.MainStoreType => header.storeVersion > storeWrapper.store.CurrentVersion, - AofStoreType.ObjectStoreType => header.storeVersion > storeWrapper.objectStore.CurrentVersion, - AofStoreType.TxnType => header.storeVersion > storeWrapper.objectStore.CurrentVersion, - _ => throw new GarnetException($"Unknown AOF header store type {storeType}"), - }; + // Single-physical-log + multi-replay: BasicHeader entries + case AofHeaderType.BasicHeader: + var basicCurr = AofHeader.SkipHeader(ptr); + var basicKey = PinnedSpanByte.FromLengthPrefixedPinnedPointer(basicCurr).ReadOnlySpan; + return storeWrapper.appendOnlyFile.Log.GetReplayTaskIdx(basicKey); + // Multi-physical-log: ShardedHeader entries + case AofHeaderType.ShardedHeader: + var curr = AofHeader.SkipHeader(ptr); + var key = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr).ReadOnlySpan; + return storeWrapper.appendOnlyFile.Log.GetReplayTaskIdx(key); + // Transaction headers (both types) don't have a single key for task assignment + case AofHeaderType.ShardedLogTransactionHeader: + case AofHeaderType.SingleLogTransactionHeader: + return -1; + default: + throw new GarnetException($"Replay header type {replayHeaderType} not supported!"); } + } - static AofStoreType ToAofStoreType(AofEntryType type) + /// + /// Determines whether the specified log entry should be skipped during replay based on its sequence number or address. + /// + /// A pointer to the start of the log entry header in memory. + /// The sequence number/address threshold. Entries beyond this value will be skipped. + /// Specify -1 to skip all entries. + /// Log address of the entry, used for single-physical-log mode. + /// When this method returns, contains the sequence number/address of the current log entry, or -1 if unavailable. + /// true if the log entry should be skipped; otherwise, false. + /// Thrown if the log entry header type is not supported. + public bool SkipReplay(byte* ptr, long untilSequenceNumber, long logAddressSequenceNumber, out long sequenceNumber) + { + sequenceNumber = -1; + if (untilSequenceNumber == -1) + return true; + var header = *(AofHeader*)ptr; + var replayHeaderType = header.HeaderType; + switch (replayHeaderType) { - return type switch - { - AofEntryType.StoreUpsert or AofEntryType.StoreRMW or AofEntryType.StoreDelete => AofStoreType.MainStoreType, - AofEntryType.ObjectStoreUpsert or AofEntryType.ObjectStoreRMW or AofEntryType.ObjectStoreDelete => AofStoreType.ObjectStoreType, - AofEntryType.TxnStart or AofEntryType.TxnCommit or AofEntryType.TxnAbort or AofEntryType.StoredProcedure => AofStoreType.TxnType, - AofEntryType.CheckpointStartCommit or AofEntryType.ObjectStoreStreamingCheckpointStartCommit => AofStoreType.CheckpointType, - AofEntryType.CheckpointEndCommit or AofEntryType.MainStoreStreamingCheckpointEndCommit or AofEntryType.ObjectStoreStreamingCheckpointEndCommit => AofStoreType.CheckpointType, - AofEntryType.FlushAll or AofEntryType.FlushDb => AofStoreType.FlushDbType, - _ => throw new GarnetException($"Conversion to AofStoreType not possible for {type}"), - }; + // Single-physical-log + multi-replay: use entry address + case AofHeaderType.BasicHeader: + case AofHeaderType.SingleLogTransactionHeader: + sequenceNumber = logAddressSequenceNumber; + return logAddressSequenceNumber > untilSequenceNumber; + // Multi-physical-log: use embedded sequence number + case AofHeaderType.ShardedHeader: + var shardedHeader = *(AofShardedHeader*)ptr; + sequenceNumber = shardedHeader.sequenceNumber; + return shardedHeader.sequenceNumber > untilSequenceNumber; + case AofHeaderType.ShardedLogTransactionHeader: + var txnHeader = *(AofShardedLogTransactionHeader*)ptr; + sequenceNumber = txnHeader.shardedHeader.sequenceNumber; + return txnHeader.shardedHeader.sequenceNumber > untilSequenceNumber; + default: + throw new GarnetException($"Replay header type {replayHeaderType} not supported!"); } } } diff --git a/libs/server/AOF/GarnetAppendOnlyFile.cs b/libs/server/AOF/GarnetAppendOnlyFile.cs new file mode 100644 index 00000000000..ce7967d89c8 --- /dev/null +++ b/libs/server/AOF/GarnetAppendOnlyFile.cs @@ -0,0 +1,241 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + public sealed class GarnetAppendOnlyFile + { + const long kFirstValidAofAddress = 64; + + /// + /// Calculates the total size, in bytes, of the log between the beginning and tail addresses. + /// + public long TotalSize() => Log.TailAddress.AggregateDiff(Log.BeginAddress); + + /// + /// Ensures prefix-consistent reads when the Garnet instance + /// operates with multiple physical sublogs. + /// + public ReadConsistencyManager readConsistencyManager = null; + + /// + /// Used to generate monotonically increasing sequence numbers for each enqueue operation + /// when the Garnet instance operates with multiple physical sublogs. + /// + public SequenceNumberGenerator seqNumGen = null; + + /// + /// Provides an interface for managing and interacting with physical sublog instances. + /// + public GarnetLog Log { get; private set; } + + public readonly GarnetServerOptions serverOptions; + + public long HeaderSize => Log.HeaderSize; + + public readonly AofAddress InvalidAofAddress; + + public readonly AofAddress MaxAofAddress; + + public readonly ILogger logger; + + /// + /// Calculate virtual sublog index provided physical sublog index and replay task index + /// + /// + /// + /// + public int GetVirtualSublogIdx(int sublogIdx, int replayIdx) + => (sublogIdx * serverOptions.AofReplayTaskCount) + replayIdx; + + /// + /// Garnet append only file constructor + /// + /// + /// + /// + public GarnetAppendOnlyFile(GarnetServerOptions serverOptions, TsavoriteLogSettings[] logSettings, ILogger logger = null) + { + this.serverOptions = serverOptions; + InvalidAofAddress = AofAddress.Create(length: serverOptions.AofPhysicalSublogCount, value: -1); + MaxAofAddress = AofAddress.Create(length: serverOptions.AofPhysicalSublogCount, value: long.MaxValue); + CreateOrUpdateKeySequenceManager(); + // Only create sequence number generator for multi-physical-log (sharded) mode. + // Single-physical-log + multi-replay uses log addresses instead. + if (serverOptions.AofPhysicalSublogCount > 1) + seqNumGen = new SequenceNumberGenerator(0); + this.logger = logger; + Log = new(this, serverOptions, logSettings, logger); + } + + /// + /// Dispose append only file + /// + public void Dispose() => Log.Dispose(); + + /// + /// Get a sequence number that is strictly greater than any sequence number assigned to records + /// at or below the currently observed TailAddress. + /// + /// + /// Correctness relies on ordering: the caller must read TailAddress BEFORE calling this method. + /// On the enqueue path, the sequence number is captured BEFORE the Enqueue (which advances TailAddress). + /// Therefore, any fresh call made after + /// observing a TailAddress is guaranteed to return a value strictly greater than the sequence + /// numbers of all records at or below that TailAddress. + /// + /// A sequence number strictly greater than those of all records up to the last observed tail. + public long GetLargerThanMaximumSequenceNumber() + => seqNumGen.GetSequenceNumber() + 1; + + /// + /// Create or update existing timestamp manager + /// NOTE: We need to create a new version for consistency manager in order for running sessions to update their context on the next read + /// + public void CreateOrUpdateKeySequenceManager() + { + // Create manager only if sharded log is enabled + if (!serverOptions.MultiLogEnabled) return; + var currentVersion = readConsistencyManager?.CurrentVersion ?? 0L; + var _readConsistencyManager = new ReadConsistencyManager(currentVersion + 1, this, serverOptions); + _ = Interlocked.CompareExchange(ref readConsistencyManager, _readConsistencyManager, readConsistencyManager); + } + + /// + /// Reset sequence number generator. + /// NOTE: We need to update starting offset when recovering or failing over to ensure time moves forward. + /// Only applicable for multi-physical-log (sharded) mode. + /// + public void ResetSequenceNumberGenerator() + { + // Only reset for multi-physical-log mode; single-physical-log uses log addresses + if (serverOptions.AofPhysicalSublogCount <= 1) + return; + var physicalSublogMaxReplayedSequenceNumber = readConsistencyManager.GetPhysicalSublogMaxReplayedSequenceNumber(); + var start = physicalSublogMaxReplayedSequenceNumber.Max(); + var newSeqNumGen = new SequenceNumberGenerator(start); + _ = Interlocked.CompareExchange(ref seqNumGen, newSeqNumGen, seqNumGen); + } + + /// + /// Compute AOF sync replay address at recovery + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public ulong ComputeAofSyncReplayAddress( + bool recoverFromRemote, + bool sameMainStoreCheckpointHistory, + bool sameHistory2, + in AofAddress replicationOffset2, + in AofAddress replicaAofBeginAddress, + in AofAddress replicaAofTailAddress, + in AofAddress beginAddress, + ref AofAddress checkpointAofBeginAddress) + { + var replayAOFMap = 0UL; + for (var sublogIdx = 0; sublogIdx < serverOptions.AofPhysicalSublogCount; sublogIdx++) + ComputeAofSubloSyncReplayAddress(sublogIdx, ref replayAOFMap, recoverFromRemote, sameMainStoreCheckpointHistory, sameHistory2, replicationOffset2, replicaAofBeginAddress, replicaAofTailAddress, beginAddress, ref checkpointAofBeginAddress); + + return replayAOFMap; + + void ComputeAofSubloSyncReplayAddress( + int sublogIdx, + ref ulong replayAOFMap, + bool recoverFromRemote, + bool sameMainStoreCheckpointHistory, + bool sameHistory2, + in AofAddress replicationOffset2, + in AofAddress replicaAofBeginAddress, + in AofAddress replicaAofTailAddress, + in AofAddress beginAddress, + ref AofAddress checkpointAofBeginAddress) + { + if (!recoverFromRemote) + { + if (replicaAofBeginAddress[sublogIdx] > kFirstValidAofAddress && replicaAofBeginAddress[sublogIdx] > checkpointAofBeginAddress[sublogIdx]) + { + logger?.LogInformation( + "ReplicaSyncSession: replicaAofBeginAddress {replicaAofBeginAddress} > PrimaryCheckpointRecoveredReplicationOffset {RecoveredReplicationOffset}, cannot use remote AOF", + replicaAofBeginAddress[sublogIdx], checkpointAofBeginAddress[sublogIdx]); + } + else + { + // Tail address cannot be behind the recovered address since above we checked replicaAofBeginAddress and it appears after RecoveredReplicationOffset + // unless we are performing MainMemoryReplication + // TODO: shouldn't we use the remote cEntry's tail address here since replica will recover to that? + if (replicaAofTailAddress[sublogIdx] < checkpointAofBeginAddress[sublogIdx] && !serverOptions.FastAofTruncate) + { + logger?.LogCritical("ReplicaSyncSession replicaAofTail {replicaAofTailAddress} < canServeFromAofAddress {RecoveredReplicationOffset}", replicaAofTailAddress, checkpointAofBeginAddress); + throw new Exception($"ReplicaSyncSession replicaAofTail {replicaAofTailAddress} < canServeFromAofAddress {checkpointAofBeginAddress}"); + } + + // If we are behind this primary we need to decide until where to replay + var replayUntilAddress = replicaAofTailAddress; + // Replica tail is further ahead than committed address of primary + if (Log.CommittedUntilAddress[sublogIdx] < replayUntilAddress[sublogIdx]) + replayUntilAddress[sublogIdx] = Log.CommittedUntilAddress[sublogIdx]; + + // Replay only if records not included in checkpoint + if (replayUntilAddress[sublogIdx] > checkpointAofBeginAddress[sublogIdx]) + { + logger?.LogInformation("ReplicaSyncSession: have to replay remote AOF from {beginAddress} until {untilAddress}", checkpointAofBeginAddress[sublogIdx], replayUntilAddress); + replayAOFMap |= 1UL << sublogIdx; + // Bound replayUntilAddress to ReplicationOffset2 to avoid replaying divergent history only if connecting replica was attached to old primary + if (sameHistory2 && replayUntilAddress[sublogIdx] > replicationOffset2[sublogIdx]) + replayUntilAddress[sublogIdx] = replicationOffset2[sublogIdx]; + checkpointAofBeginAddress = replayUntilAddress; + } + + if (!sameMainStoreCheckpointHistory) + { + // If we are not in the same checkpoint history, we need to stream the AOF from the primary's beginning address + checkpointAofBeginAddress[sublogIdx] = beginAddress[sublogIdx]; + replayAOFMap &= ~(1UL << sublogIdx); + logger?.LogInformation("ReplicaSyncSession: not in same checkpoint history, will replay from beginning address {checkpointAofBeginAddress}", checkpointAofBeginAddress); + } + } + } + } + } + + /// + /// Perform a data loss check at recovery + /// + /// + /// + /// + /// + public void DataLossCheck(bool possibleAofDataLoss, AofAddress syncFromAofAddress, ILogger logger = null) + { + var beginAddress = Log.BeginAddress; + var anyLesser = syncFromAofAddress.AnyLesser(beginAddress); + + if (anyLesser) + { + if (!possibleAofDataLoss) + { + logger?.LogError("syncFromAofAddress: {syncFromAofAddress} < beginAofAddress: {storeWrapper.appendOnlyFile.BeginAddress}", syncFromAofAddress, beginAddress); + throw new Exception("Failed syncing because replica requested truncated AOF address"); + } + else + { + logger?.LogWarning("AOF truncated, unsafe attach: syncFromAofAddress: {syncFromAofAddress} < beginAofAddress: {storeWrapper.appendOnlyFile.BeginAddress}", syncFromAofAddress, beginAddress); + } + } + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/GarnetLog.cs b/libs/server/AOF/GarnetLog.cs new file mode 100644 index 00000000000..cd700ee16c5 --- /dev/null +++ b/libs/server/AOF/GarnetLog.cs @@ -0,0 +1,1030 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Provides a unified interface for managing append-only logs in both single-log and sharded-log modes, supporting + /// operations such as recovery, commit, truncation, and scanning across all sublogs as configured by the server + /// options. + /// + public sealed class GarnetLog + { + readonly GarnetAppendOnlyFile appendOnlyFile; + readonly GarnetServerOptions serverOptions; + readonly SingleLog singleLog; + readonly ShardedLog shardedLog; + readonly Func cookieGeneratorCallback; + readonly bool usingSingleLog; + readonly bool usingSinglePhysicalLog; + readonly int physicalSublogCount; + readonly int replayTaskCount; + + public static unsafe long GetSequenceNumberFromCookie(byte[] cookie) + { + fixed (byte* ptr = cookie) + { + return *(long*)ptr; + } + } + + /// + /// Initializes a new GarnetLog instance with the specified configuration. + /// + /// Append only file instance. + /// Server configuration determining log mode and sharding. + /// Settings for the underlying log(s). + /// Optional logger for recording events. + public GarnetLog(GarnetAppendOnlyFile appendOnlyFile, GarnetServerOptions serverOptions, TsavoriteLogSettings[] logSettings, ILogger logger = null) + { + this.appendOnlyFile = appendOnlyFile; + this.cookieGeneratorCallback = () => + { + unsafe + { + var cookie = stackalloc byte[8]; + *(long*)cookie = appendOnlyFile.seqNumGen.GetSequenceNumber(); + return new Span(cookie, 8).ToArray(); + } + }; + + this.serverOptions = serverOptions; + this.usingSingleLog = serverOptions.AofPhysicalSublogCount == 1 && serverOptions.AofReplayTaskCount == 1; + this.usingSinglePhysicalLog = serverOptions.AofPhysicalSublogCount == 1; + + if (usingSinglePhysicalLog) + this.singleLog = new SingleLog(logSettings[0], logger); + else + { + this.shardedLog = new ShardedLog(serverOptions.AofPhysicalSublogCount, logSettings, logger: logger); + } + + physicalSublogCount = serverOptions.AofPhysicalSublogCount; + replayTaskCount = serverOptions.AofReplayTaskCount; + } + + public TsavoriteLog SingleLog => singleLog.log; + public long HeaderSize => singleLog != null ? singleLog.HeaderSize : shardedLog.HeaderSize; + public int Size => singleLog != null ? 1 : shardedLog.Length; + public int ReplayTaskCount => serverOptions.AofReplayTaskCount; + + /// + /// Hash function used for sharded-log + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long HASH(ReadOnlySpan key) + => GarnetKeyComparer.StaticGetHashCode64((FixedSpanByteKey)key); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetPhysicalSublogIdx(long hash) => (int)((ulong)hash % (uint)physicalSublogCount); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetReplayTaskIdx(long hash) => (int)(((ulong)hash / (uint)physicalSublogCount) % (uint)replayTaskCount); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetVirtualSublogIdx(long hash) => GetPhysicalSublogIdx(hash) * replayTaskCount + GetReplayTaskIdx(hash); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetPhysicalSublogIdx(ReadOnlySpan key) => GetPhysicalSublogIdx(HASH(key)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetReplayTaskIdx(ReadOnlySpan key) => GetReplayTaskIdx(HASH(key)); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetVirtualSublogIdx(ReadOnlySpan key) => GetVirtualSublogIdx(HASH(key)); + + public AofAddress BeginAddress + { + get + { + if (singleLog != null) + return singleLog.BeginAddress; + return shardedLog.BeginAddress; + } + } + + public AofAddress TailAddress + { + get + { + if (singleLog != null) + return singleLog.TailAddress; + return shardedLog.TailAddress; + } + } + + public AofAddress CommittedUntilAddress + { + get + { + if (singleLog != null) + return singleLog.CommittedUntilAddress; + return shardedLog.CommittedUntilAddress; + } + } + + public AofAddress CommittedBeginAddress + { + get + { + if (singleLog != null) + return singleLog.CommittedBeginAddress; + return shardedLog.CommittedBeginAddress; + } + } + + public AofAddress FlushedUntilAddress + { + get + { + if (singleLog != null) + return singleLog.FlushedUntilAddress; + return shardedLog.FlushedUntilAddress; + } + } + + public AofAddress MaxMemorySizeBytes + { + get + { + if (singleLog != null) + return singleLog.MaxMemorySizeBytes; + return shardedLog.MaxMemorySizeBytes; + } + } + + public AofAddress MemorySizeBytes + { + get + { + if (singleLog != null) + return singleLog.MemorySizeBytes; + return shardedLog.MemorySizeBytes; + } + } + + public void Recover() + { + if (singleLog != null) + singleLog.Recover(); + else + shardedLog.Recover(); + } + + public bool RecoverLatestSequenceNumber(out long recoverUntilSequenceNumber) + { + recoverUntilSequenceNumber = -1; + if (serverOptions.AofPhysicalSublogCount == 1) + return true; + var sublogCount = shardedLog.sublog.Length; + for (var physicalSublogIdx = 0; physicalSublogIdx < sublogCount; physicalSublogIdx++) + { + var physicalSublog = shardedLog.sublog[physicalSublogIdx]; + var cookie = physicalSublog.RecoveredCookie; + if (cookie == null) + return false; + var latestSequenceNumber = GetSequenceNumberFromCookie(cookie); + recoverUntilSequenceNumber = recoverUntilSequenceNumber == -1 ? latestSequenceNumber : Math.Min(recoverUntilSequenceNumber, latestSequenceNumber); + } + return true; + } + + public void Reset() + { + if (singleLog != null) + singleLog.Reset(); + else + shardedLog.Reset(); + } + + public void Dispose() + { + if (singleLog != null) + singleLog.Dispose(); + else + shardedLog.Dispose(); + } + + /// + /// Get a bitmap having all bits set according to the total number of sublogs being used + /// + /// + public ulong AllLogsBitmask() => (ulong)((1L << Size) - 1); + + /// + /// Lock sublogs for enqueue operation (bits indicate sublogIdx) + /// NOTE: Slow; should be used sparingly + /// + /// + public void LockSublogs(ulong logAccessBitmap) + { + Debug.Assert(serverOptions != null); + Debug.Assert(BitOperations.PopCount(logAccessBitmap) <= shardedLog.Length); + shardedLog.LockSublogs(logAccessBitmap); + } + + /// + /// Unlock sublogs using the provided logAccessBitmap (bits indicate sublogIdx) + /// + /// + public void UnlockSublogs(ulong logAccessBitmap) + { + Debug.Assert(shardedLog != null); + Debug.Assert(BitOperations.PopCount(logAccessBitmap) <= shardedLog.Length); + shardedLog.UnlockSublogs(logAccessBitmap); + } + + /// + /// Get sublog instance indicated by the provided index + /// + /// + /// + public TsavoriteLog GetSubLog(int sublogIdx) + { + if (singleLog != null) + { + Debug.Assert(sublogIdx == 0); + return singleLog.log; + } + else + { + Debug.Assert(sublogIdx < shardedLog.Length); + return shardedLog.sublog[sublogIdx]; + } + } + + /// + /// Gets the tail address for a specific sublog without copying the entire AofAddress struct. + /// + /// Index of the physical sublog. + /// The tail address of the specified sublog. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetTailAddress(int sublogIdx) + { + if (singleLog != null) + { + Debug.Assert(sublogIdx == 0); + return singleLog.log.TailAddress; + } + else + { + Debug.Assert(sublogIdx < shardedLog.Length); + return shardedLog.sublog[sublogIdx].TailAddress; + } + } + + /// + /// Gets the begin address for a specific sublog without copying the entire AofAddress struct. + /// + /// Index of the physical sublog. + /// The begin address of the specified sublog. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetBeginAddress(int sublogIdx) + { + if (singleLog != null) + { + Debug.Assert(sublogIdx == 0); + return singleLog.log.BeginAddress; + } + else + { + Debug.Assert(sublogIdx < shardedLog.Length); + return shardedLog.sublog[sublogIdx].BeginAddress; + } + } + + /// + /// Set log shift tail callbacks + /// + /// + /// + public void SetLogShiftTailCallback(int sublogIdx, Action safeTailPageShiftCallback) + { + if (singleLog != null) + { + singleLog.log.SafeTailPageShiftCallback = safeTailPageShiftCallback; + } + else + { + shardedLog.sublog[sublogIdx].SafeTailPageShiftCallback = safeTailPageShiftCallback; + } + } + + /// + /// Scan sublog with the specified parameters + /// + /// Sublog index + /// Begin address for scan + /// End address for scan + /// Whether to recover named iterator from latest commit + /// Use single or double buffering + /// Whether we scan uncommitted data + /// Optional logger + /// TsavoriteLogScanIterator instance + public TsavoriteLogScanIterator Scan(int sublogIdx, long beginAddress, long endAddress, bool recover = true, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) + { + if (singleLog != null) + { + Debug.Assert(sublogIdx == 0); + return singleLog.log.Scan(beginAddress, endAddress, recover, scanBufferingMode, scanUncommitted, logger); + } + else + { + Debug.Assert(sublogIdx < shardedLog.Length); + return shardedLog.sublog[sublogIdx].Scan(beginAddress, endAddress, recover, scanBufferingMode, scanUncommitted, logger); + } + } + + /// + /// Scan single sublog with the specified parameters + /// + /// Sublog index + /// Begin address for scan + /// End address for scan + /// Whether to recover named iterator from latest commit + /// Use single or double buffering + /// Whether we scan uncommitted data + /// Optional logger + /// TsavoriteLogScanSingleIterator instance + public TsavoriteLogScanSingleIterator ScanSingle(int sublogIdx, long beginAddress, long endAddress, bool recover = true, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) + { + if (singleLog != null) + { + Debug.Assert(sublogIdx == 0); + return singleLog.log.ScanSingle(beginAddress, endAddress, recover, scanBufferingMode, scanUncommitted, logger); + } + else + { + Debug.Assert(sublogIdx < shardedLog.Length); + return shardedLog.sublog[sublogIdx].ScanSingle(beginAddress, endAddress, recover, scanBufferingMode, scanUncommitted, logger); + } + } + + /// + /// Get log page size bits + /// TODO: Is this initialized only once? Is it same across sublogs? + /// + /// + public int UnsafeGetLogPageSizeBits() + => GetSubLog(0).UnsafeGetLogPageSizeBits(); + + /// + /// UnsafeGetReadOnlyAddressAbove + /// + /// + /// + /// + /// + public long UnsafeGetReadOnlyAddressAbove(int physicalSublogIdx, long newTailAddress, int numPagesAbove) + => GetSubLog(0).UnsafeGetReadOnlyAddressAbove(newTailAddress, numPagesAbove); + + /// + /// Shifts the begin address of the specified sublog or single log to the given address. + /// + /// Index of the physical sublog to operate on when using a sharded log. + /// The address to which the begin address should be shifted. + /// If true, snaps the begin address to the start of the page containing the specified address. + /// If true, truncates the log up to the new begin address. + public void UnsafeShiftBeginAddress(int physicalSublogIdx, long untilAddress, bool snapToPageStart = false, bool truncateLog = false) + { + if (singleLog != null) + singleLog.log.UnsafeShiftBeginAddress(untilAddress, snapToPageStart: snapToPageStart, truncateLog: truncateLog); + else + shardedLog.sublog[physicalSublogIdx].UnsafeShiftBeginAddress(untilAddress, snapToPageStart: snapToPageStart, truncateLog: truncateLog); + } + + /// + /// Truncates the log up to the specified address, either on a single log or a sharded sublog. + /// + /// The index of the physical sublog to truncate when using sharded logs. + /// The address up to which the log should be truncated. + public void TruncateUntil(int physicalSublogIdx, long untilAddress) + { + if (singleLog != null) + singleLog.log.TruncateUntil(untilAddress); + else + shardedLog.sublog[physicalSublogIdx].TruncateUntil(untilAddress); + } + + /// + /// Safe initialize when FastAofTruncate is enabled + /// + /// + /// + /// + /// + public void SafeInitialize(int sublogIdx, long beginAddress, long committedUntilAddress, long lastCommitNum = 0) + { + if (singleLog != null) + singleLog.log.SafeInitialize(beginAddress, committedUntilAddress, lastCommitNum); + else + shardedLog.sublog[sublogIdx].SafeInitialize(beginAddress, committedUntilAddress, lastCommitNum); + } + + /// + /// Conditional initialize + /// + /// + public void InitializeIf(ref AofAddress recoveredSafeAofAddress) + { + if (singleLog != null) + { + if (TailAddress[0] < recoveredSafeAofAddress[0]) + singleLog.log.Initialize(TailAddress[0], recoveredSafeAofAddress[0]); + } + else + { + for (var i = 0; i < shardedLog.Length; i++) + if (TailAddress[i] < recoveredSafeAofAddress[i]) + shardedLog.sublog[i].Initialize(TailAddress[i], recoveredSafeAofAddress[i]); + } + } + + /// + /// Initialize + /// + /// + /// + /// + public void Initialize(in AofAddress beginAddress, in AofAddress committedUntilAddress, long lastCommitNum = 0) + { + if (singleLog != null) + { + Debug.Assert(beginAddress.Length == 1); + singleLog.log.Initialize(beginAddress[0], committedUntilAddress[0], lastCommitNum); + } + else + { + Debug.Assert(beginAddress.Length == shardedLog.Length); + for (var i = 0; i < shardedLog.Length; i++) + shardedLog.sublog[i].Initialize(beginAddress[i], committedUntilAddress[i], lastCommitNum); + } + } + + /// + /// Commit all physical sublogs + /// + /// + /// + public void Commit(bool spinWait = false, byte[] cookie = null) + { + if (singleLog != null) + { + singleLog.log.Commit(spinWait, cookie); + } + else + { + var _cookie = cookieGeneratorCallback(); + for (var i = 0; i < shardedLog.Length; i++) + shardedLog.sublog[i].Commit(spinWait, cookie: _cookie); + } + } + + /// + /// Blocks the calling thread until the log is committed up to the specified address or commit number. + /// + /// The address up to which to wait for the commit. Defaults to 0. + /// The commit number up to which to wait. Defaults to -1. + public void WaitForCommit(long untilAddress = 0, long commitNum = -1) + { + if (singleLog != null) + { + singleLog.log.WaitForCommit(untilAddress, commitNum); + } + else + { + for (var i = 0; i < shardedLog.Length; i++) + shardedLog.sublog[i].WaitForCommit(untilAddress, commitNum); + } + } + + /// + /// Commit async all physical sublogs + /// + /// + /// + /// + public async ValueTask CommitAsync(byte[] cookie = null, CancellationToken token = default) + { + if (singleLog != null) + { + // Optimization for single log case + await singleLog.log.CommitAsync(cookie: cookie, token).ConfigureAwait(false); + } + else + { + var _cookie = cookieGeneratorCallback(); + // Create tasks for all sublogs + var tasks = new Task[shardedLog.Length]; + for (var i = 0; i < shardedLog.Length; i++) + tasks[i] = shardedLog.sublog[i].CommitAsync(token: token, cookie: _cookie).AsTask(); + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + } + + /// + /// Wait for commit asycn of all sublogs + /// + /// + /// + /// + /// + public async ValueTask WaitForCommitAsync(long untilAddress = 0, long commitNum = -1, CancellationToken token = default) + { + if (singleLog != null) + { + // Optimization for single log case + await singleLog.log.WaitForCommitAsync(untilAddress, commitNum, token).ConfigureAwait(false); + } + else + { + // Create tasks for all sublogs + var tasks = new Task[shardedLog.Length]; + for (var i = 0; i < shardedLog.Length; i++) + tasks[i] = shardedLog.sublog[i].WaitForCommitAsync(untilAddress, commitNum, token).AsTask(); + + await Task.WhenAll(tasks).ConfigureAwait(false); + } + } + + /// + /// Shift begin address of all physical sublogs + /// + /// + /// + /// + public void UnsafeShiftBeginAddress(AofAddress untilAddress, bool snapToPageStart = false, bool truncateLog = false) + { + if (singleLog != null) + { + singleLog.log.UnsafeShiftBeginAddress(untilAddress[0], snapToPageStart, truncateLog); + } + else + { + for (var i = 0; i < shardedLog.Length; i++) + shardedLog.sublog[i].UnsafeShiftBeginAddress(untilAddress[i], snapToPageStart, truncateLog); + } + } + + /// + /// Truncate until address for all physical sublogs + /// + /// + public void TruncateUntil(AofAddress untilAddress) + { + if (singleLog != null) + { + singleLog.log.TruncateUntil(untilAddress[0]); + } + else + { + for (var i = 0; i < shardedLog.Length; i++) + shardedLog.sublog[i].TruncateUntil(untilAddress[i]); + } + } + + internal void Enqueue(AofEntryType opType, long version, int sessionId, ReadOnlySpan key, ReadOnlySpan value, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) + where TInput : IStoreInput + where TEpochAccessor : IEpochAccessor + { + // Single physical log (covers both single-log and single-physical-log + multi-replay) + // Uses BasicHeader — log addresses provide ordering for multi-replay consistency + if (usingSinglePhysicalLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId + }; + + singleLog.log.Enqueue( + header, + key, + value, + ref input, + epochAccessor, + out logicalAddress); + } + // Multi physical sublogs and multi-replay support + else + { + var shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber() + }; + + var physicalSublogIdx = GetPhysicalSublogIdx(key); + shardedLog.sublog[physicalSublogIdx].Enqueue( + shardedHeader, + key, + value, + ref input, + epochAccessor, + out logicalAddress); + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal void Enqueue(AofEntryType opType, long version, int sessionId, ReadOnlySpan key, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) + where TInput : IStoreInput + where TEpochAccessor : IEpochAccessor + { + // Single physical log (covers both single-log and single-physical-log + multi-replay) + if (usingSinglePhysicalLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId + }; + + singleLog.log.Enqueue( + header, + key, + ref input, + epochAccessor, + out logicalAddress); + } + // Multi physical sublogs and multi-replay support + else + { + var shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber() + }; + + var physicalSublogIdx = GetPhysicalSublogIdx(key); + shardedLog.sublog[physicalSublogIdx].Enqueue( + shardedHeader, + key, + ref input, + epochAccessor, + out logicalAddress); + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal void Enqueue(AofEntryType opType, long version, int sessionId, ReadOnlySpan key, ReadOnlySpan value, TEpochAccessor epochAccessor, out long logicalAddress) + where TEpochAccessor : IEpochAccessor + { + // Single physical log (covers both single-log and single-physical-log + multi-replay) + if (usingSinglePhysicalLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId, + }; + + singleLog.log.Enqueue( + header, + key, + value, + epochAccessor, + out logicalAddress); + } + // Multi physical sublogs and multi-replay support + else + { + var shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber() + }; + + var physicalSublogIdx = GetPhysicalSublogIdx(key); + shardedLog.sublog[physicalSublogIdx].Enqueue( + shardedHeader, + key, + value, + epochAccessor, + out logicalAddress); + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal unsafe void EnqueueStoredProc(AofEntryType opType, byte procedureId, long txnVersion, int sessionId, ref CustomProcedureInput procInput, CustomTransactionProcedure proc) + { + if (usingSingleLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + procedureId = procedureId, + storeVersion = txnVersion, + sessionID = sessionId, + }; + singleLog.log.Enqueue(header, ref procInput, out _); + } + else if (usingSinglePhysicalLog) + { + // Single physical log + multi-replay: use lightweight header without sequence number + var singleLogTxnHeader = new AofSingleLogTransactionHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.SingleLogTransactionHeader, + opType = opType, + procedureId = procedureId, + storeVersion = txnVersion, + sessionID = sessionId, + }, + participantCount = (short)proc.virtualSublogParticipantCount + }; + + proc.replayTaskAccessVector[0].CopyTo( + new Span(singleLogTxnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + singleLog.log.Enqueue(singleLogTxnHeader, ref procInput, out _); + } + else + { + var txnHeader = new AofShardedLogTransactionHeader + { + shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedLogTransactionHeader, + opType = opType, + procedureId = procedureId, + storeVersion = txnVersion, + sessionID = sessionId, + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber(), + }, + participantCount = (short)proc.virtualSublogParticipantCount + }; + + try + { + if (serverOptions.AofPhysicalSublogCount > 1) + LockSublogs(proc.physicalSublogAccessVector); + var _physicalSublogAccessVector = proc.physicalSublogAccessVector; + while (_physicalSublogAccessVector > 0) + { + var physicalSublogIdx = _physicalSublogAccessVector.GetNextOffset(); + proc.replayTaskAccessVector[physicalSublogIdx].CopyTo( + new Span(txnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + shardedLog.sublog[physicalSublogIdx].Enqueue(txnHeader, ref procInput, out _); + } + } + finally + { + if (serverOptions.AofPhysicalSublogCount > 1) + UnlockSublogs(proc.physicalSublogAccessVector); + } + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal unsafe void EnqueueTxn(AofEntryType opType, long txnVersion, int sessionId, ulong physicalSublogAccessVector, BitVector[] virtualSublogAccessVector, int virtualSublogParticipantCount) + { + if (usingSingleLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = txnVersion, + sessionID = sessionId, + }; + appendOnlyFile.Log.SingleLog.Enqueue(header, out _); + } + else if (usingSinglePhysicalLog) + { + // Single physical log + multi-replay: use lightweight header without sequence number + var singleLogTxnHeader = new AofSingleLogTransactionHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.SingleLogTransactionHeader, + opType = opType, + storeVersion = txnVersion, + sessionID = sessionId, + }, + participantCount = (short)virtualSublogParticipantCount + }; + + virtualSublogAccessVector[0].CopyTo(new Span(singleLogTxnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + singleLog.log.Enqueue(singleLogTxnHeader, out _); + } + else + { + var txnHeader = new AofShardedLogTransactionHeader + { + shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedLogTransactionHeader, + opType = opType, + storeVersion = txnVersion, + sessionID = sessionId, + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber(), + }, + participantCount = (short)virtualSublogParticipantCount + }; + + try + { + if (serverOptions.AofPhysicalSublogCount > 1) + LockSublogs(physicalSublogAccessVector); + var _physicalSublogAccessVector = physicalSublogAccessVector; + while (_physicalSublogAccessVector > 0) + { + var physicalSublogIdx = _physicalSublogAccessVector.GetNextOffset(); + virtualSublogAccessVector[physicalSublogIdx].CopyTo(new Span(txnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes)); + shardedLog.sublog[physicalSublogIdx].Enqueue(txnHeader, out _); + } + } + finally + { + if (serverOptions.AofPhysicalSublogCount > 1) + UnlockSublogs(physicalSublogAccessVector); + } + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal void Enqueue(AofEntryType opType, long version, int sessionId, ReadOnlySpan key, ref TInput input, out long logicalAddress) + where TInput : IStoreInput + { + // Single physical log (covers both single-log and single-physical-log + multi-replay) + if (usingSinglePhysicalLog) + { + var header = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId, + }; + + singleLog.log.Enqueue( + header, + key, + ref input, + out logicalAddress); + } + // Multi physical sublogs and multi-replay support + else + { + var shardedHeader = new AofShardedHeader + { + basicHeader = new AofHeader + { + HeaderType = AofHeaderType.ShardedHeader, + opType = opType, + storeVersion = version, + sessionID = sessionId, + }, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber() + }; + + var physicalSublogIdx = GetPhysicalSublogIdx(key); + shardedLog.sublog[physicalSublogIdx].Enqueue( + shardedHeader, + key, + ref input, + out logicalAddress); + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + /// + /// Enqueues an entry to all physical sublogs (broadcast). Used for markers that must + /// be visible to all replay tasks (e.g., database commit, safe-flush, checkpoint). + /// + private unsafe void EnqueueBroadcastEntry(AofHeader basicHeader) + { + if (usingSingleLog) + { + singleLog.log.Enqueue(basicHeader, out _); + } + else if (usingSinglePhysicalLog) + { + // Single physical log + multi-replay: use lightweight header without sequence number + basicHeader.HeaderType = AofHeaderType.SingleLogTransactionHeader; + var singleLogTxnHeader = new AofSingleLogTransactionHeader + { + basicHeader = basicHeader, + participantCount = (short)appendOnlyFile.serverOptions.AofVirtualSublogCount + }; + new Span(singleLogTxnHeader.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes).Fill(0xFF); + + singleLog.log.Enqueue(singleLogTxnHeader, out _); + } + else + { + basicHeader.HeaderType = AofHeaderType.ShardedLogTransactionHeader; + var header = new AofShardedLogTransactionHeader + { + shardedHeader = new AofShardedHeader + { + basicHeader = basicHeader, + sequenceNumber = appendOnlyFile.seqNumGen.GetSequenceNumber() + }, + participantCount = (short)appendOnlyFile.serverOptions.AofVirtualSublogCount + }; + new Span(header.replayTaskAccessVector, AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes).Fill(0xFF); + + var physicalSublogAccessVector = AllLogsBitmask(); + try + { + if (serverOptions.AofPhysicalSublogCount > 1) + LockSublogs(physicalSublogAccessVector); + var _physicalSublogAccessVector = physicalSublogAccessVector; + + while (_physicalSublogAccessVector > 0) + { + var physicalSublogIdx = _physicalSublogAccessVector.GetNextOffset(); + shardedLog.sublog[physicalSublogIdx].Enqueue(header, out _); + } + } + finally + { + if (serverOptions.AofPhysicalSublogCount > 1) + UnlockSublogs(physicalSublogAccessVector); + } + + if (serverOptions.AofAutoCommit) + Commit(); + } + } + + internal void EnqueueDatabaseCommit(AofEntryType opType, long version) + { + var basicHeader = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = version, + sessionID = -1 + }; + EnqueueBroadcastEntry(basicHeader); + } + + internal void EnqueueSafeFlushAOF(AofEntryType opType, bool unsafeTruncateLog, int dbId) + { + var basicHeader = new AofHeader + { + HeaderType = AofHeaderType.BasicHeader, + opType = opType, + storeVersion = 0, + sessionID = -1, + UnsafeTruncateLog = unsafeTruncateLog, + databaseId = (byte)dbId + }; + EnqueueBroadcastEntry(basicHeader); + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReadConsistency/CustomProcedureKeyHashCollection.cs b/libs/server/AOF/ReadConsistency/CustomProcedureKeyHashCollection.cs new file mode 100644 index 00000000000..b03c825025e --- /dev/null +++ b/libs/server/AOF/ReadConsistency/CustomProcedureKeyHashCollection.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace Garnet.server +{ + /// + /// Used to track the hashes of the keys associated with a given proc in order to update their timestamps in the ReplicaTimestampTracker + /// + /// + public class CustomProcedureKeyHashCollection(GarnetAppendOnlyFile appendOnlyFile) + { + readonly GarnetAppendOnlyFile appendOnlyFile = appendOnlyFile; + readonly List hashes = []; + + /// + /// Add hash from key to track + /// + /// + public void AddHash(long hash) + => hashes.Add(hash); + + /// + /// Update sequenceNumber for all keys in the collection + /// + /// + public void UpdateSequenceNumber(long sequenceNumber) + { + foreach (var hash in hashes) + appendOnlyFile.readConsistencyManager.UpdateVirtualSublogKeySequenceNumber(hash, sequenceNumber); + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReadConsistency/ReadConsistencyManager.cs b/libs/server/AOF/ReadConsistency/ReadConsistencyManager.cs new file mode 100644 index 00000000000..420bb5036a0 --- /dev/null +++ b/libs/server/AOF/ReadConsistency/ReadConsistencyManager.cs @@ -0,0 +1,228 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Garnet.server +{ + /// + /// Manages read consistency for append-only file operations, tracking sequence numbers and ensuring consistent + /// reads across virtual sublogs and keys. + /// + /// + /// + /// + public class ReadConsistencyManager(long currentVersion, GarnetAppendOnlyFile appendOnlyFile, GarnetServerOptions serverOptions) + { + /// + /// Read consistency manager version. + /// + public long CurrentVersion { get; private set; } = currentVersion; + readonly GarnetServerOptions serverOptions = serverOptions; + + readonly VirtualSublogReplayState[] vsrs = [.. Enumerable.Range(0, serverOptions.AofVirtualSublogCount).Select(_ => new VirtualSublogReplayState())]; + + /// + /// Get sequence number for provided key. + /// + /// + /// + /// + public long GetKeySequenceNumber(ReadOnlySpan key, bool frontier = false) + { + var hash = GarnetLog.HASH(key); + return frontier ? GetSublogFrontierSequenceNumber(hash) : GetKeySequenceNumber(hash); + } + + /// + /// Get snapshot of maximum replayed timestamp for all physical sublogs + /// + /// + public AofAddress GetPhysicalSublogMaxReplayedSequenceNumber() + { + var physicalSublogCount = serverOptions.AofPhysicalSublogCount; + var replayTaskCount = serverOptions.AofReplayTaskCount; + var maxKeySeqNumVector = AofAddress.Create(physicalSublogCount, 0); + for (var physicalSublogIdx = 0; physicalSublogIdx < physicalSublogCount; physicalSublogIdx++) + { + for (var rt = 0; rt < replayTaskCount; rt++) + maxKeySeqNumVector[physicalSublogIdx] = Math.Max(maxKeySeqNumVector[physicalSublogIdx], vsrs[appendOnlyFile.GetVirtualSublogIdx(physicalSublogIdx, rt)].Max); + } + return maxKeySeqNumVector; + } + + /// + /// Get frontier sequence number for provided hash + /// NOTE: Frontier sequence number is maximum sequence number between key specific sequence number and maximum observed sublog sequence number + /// + /// + /// + long GetSublogFrontierSequenceNumber(long keyHash) + => vsrs[appendOnlyFile.Log.GetVirtualSublogIdx(keyHash)].GetFrontierSequenceNumber(keyHash); + + /// + /// Get key specific sequence number for provided hash + /// + /// + /// + long GetKeySequenceNumber(long keyHash) + => vsrs[appendOnlyFile.Log.GetVirtualSublogIdx(keyHash)].GetKeySequenceNumber(keyHash); + + /// + /// Update physical sublog max sequence number + /// + /// + /// + public void UpdatePhysicalSublogMaxSequenceNumber(int physicalSublogIdx, long sequenceNumber) + { + var replayTaskCount = serverOptions.AofReplayTaskCount; + // Update virtual sublog maximum value for all virtual sublogs + for (var rt = 0; rt < replayTaskCount; rt++) + vsrs[appendOnlyFile.GetVirtualSublogIdx(physicalSublogIdx, rt)].UpdateMaxSequenceNumber(sequenceNumber); + } + + /// + /// Update max sequence number of virtual sublog associated with the specified virtual sublogIdx. + /// + /// + /// + public void UpdateVirtualSublogMaxSequenceNumber(int virtualSublogIdx, long sequenceNumber) + => vsrs[virtualSublogIdx].UpdateMaxSequenceNumber(sequenceNumber); + + /// + /// Update key sequence number of virtual sublog associated with the specified virtual sublogIdx. + /// + /// + /// + /// + public void UpdateVirtualSublogKeySequenceNumber(int virtualSublogIdx, long keyHash, long sequenceNumber) + => vsrs[virtualSublogIdx].UpdateKeySequenceNumber(keyHash, sequenceNumber); + + /// + /// Update key sequence number of virtual sublog associated with the specified keyHash. + /// + /// + /// + public void UpdateVirtualSublogKeySequenceNumber(long keyHash, long sequenceNumber) + => vsrs[appendOnlyFile.Log.GetVirtualSublogIdx(keyHash)].UpdateKeySequenceNumber(keyHash, sequenceNumber); + + /// + /// Ensures that the specified replica read session context is synchronized with the current session version. + /// + /// A reference to the session context to check and update. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void CheckConsistencyManagerVersion(ref ReplicaReadSessionContext replicaReadSessionContext) + { + // If first time calling or version has been bumped reset read context + // NOTE: Version changes every time replica is reset and a attached to a new primary. + // When a batch of read commands executes, it all happens under epoch protection, hence version change will not affect read prefix consistency + if (replicaReadSessionContext.sessionVersion == -1 || replicaReadSessionContext.sessionVersion != CurrentVersion) + { + replicaReadSessionContext.sessionVersion = CurrentVersion; + replicaReadSessionContext.lastVirtualSublogIdx = -1; + replicaReadSessionContext.maximumSessionSequenceNumber = 0; + } + } + + /// + /// Verify key freshness before allowing reads. + /// + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void VerifyKeyFreshness(long keyHash, ref ReplicaReadSessionContext replicaReadSessionContext, CancellationToken ct) + { + var virtualSublogIdx = appendOnlyFile.Log.GetVirtualSublogIdx(keyHash); + + // Here we have to wait for replay to catch up + // Don't have to wait if reading from same sublog or maximumSessionTimestamp is behind the sublog frontier timestamp + if (replicaReadSessionContext.lastVirtualSublogIdx != -1 && replicaReadSessionContext.lastVirtualSublogIdx != virtualSublogIdx) + { + // Optimistic check without lock + while (replicaReadSessionContext.maximumSessionSequenceNumber >= GetSublogFrontierSequenceNumber(keyHash)) + { + vsrs[virtualSublogIdx].WaitForSequenceNumber( + keyHash, + replicaReadSessionContext.maximumSessionSequenceNumber, + ct); + } + } + + // Store for future update + replicaReadSessionContext.lastVirtualSublogIdx = (short)virtualSublogIdx; + replicaReadSessionContext.lastHash = keyHash; + } + + /// + /// This method implements part of the consistent read protocol for a single key when shared AOF is enabled. + /// NOTE: + /// This method waits until the log sequence number of the associated key is lesser or equal than the maximum session log sequence number. + /// It executes before store.Read is processed to ensure that the log sequence number of the associated key is ahead of the last read in accordance to the consistent read protocol + /// The replica read context is updated () after the actual store.Read call to ensure that we don't underestimate the true log sequence number. + /// + /// + /// + /// + public void BeforeConsistentReadKey(long hash, ref ReplicaReadSessionContext replicaReadSessionContext, CancellationToken ct) + { + // Check version + CheckConsistencyManagerVersion(ref replicaReadSessionContext); + + // Verify key freshness + VerifyKeyFreshness(hash, ref replicaReadSessionContext, ct); + } + + /// + /// This method implements part of the consistent read protocol for a single key when shared AOF is enabled. + /// NOTE: + /// This method is used to update the log sequence number after store.Read was processed. + /// This is done to ensure that the log sequence number tracked by the ReadConsistencyManager is an overestimate of the actual sequence number since + /// we cannot be certain at prepare phase what is the actual sequence number. + /// + /// + public void AfterConsistentReadKey(ref ReplicaReadSessionContext replicaReadSessionContext) + { + replicaReadSessionContext.maximumSessionSequenceNumber = Math.Max( + replicaReadSessionContext.maximumSessionSequenceNumber, GetKeySequenceNumber(replicaReadSessionContext.lastHash)); + } + + /// + /// Verify key freshness and keep track hash and maximum session sequence number to check for updates after batch read. + /// + /// + /// + /// + /// + public void BeforeConsistentReadKeyBatch(ReadOnlySpan key, ref ReplicaReadSessionContext batchReadContext, CancellationToken ct, out long hash) + { + // Verify key freshness + hash = GarnetLog.HASH(key); + VerifyKeyFreshness(hash, ref batchReadContext, ct); + + // Keep track of max sequence number to check for updates after batch read. + batchReadContext.maximumSessionSequenceNumber = Math.Max( + batchReadContext.maximumSessionSequenceNumber, GetKeySequenceNumber(batchReadContext.lastHash)); + } + + /// + /// Validate that key sequence number has not progressed beyond the snapshot used for batch key read. + /// + /// + /// + /// + public bool AfterConsistentReadKeyBatch(long hash, ref ReplicaReadSessionContext batchReadContext) + { + var keySequenceNumber = GetKeySequenceNumber(hash); + var mSSN = batchReadContext.maximumSessionSequenceNumber; + // NOTE: Read key batch is prefix consistent at boundary because maximumSessionSequenceNumber (mSSN) == maxof(batch key sequence numbers) + // and freshness check would have prevented boundary read of the corresponding key. + // In other words, T_k (timestamp of key k) < T_f (frontier timestamp where read was allowed to proceed) and because mSSN == max of all T_k in the batch + // mSSN < T_f, hence time has advanced beyond the point where it is safe to read. + return keySequenceNumber <= mSSN; + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReadConsistency/ReplicaReadSessionContext.cs b/libs/server/AOF/ReadConsistency/ReplicaReadSessionContext.cs new file mode 100644 index 00000000000..01543b41bca --- /dev/null +++ b/libs/server/AOF/ReadConsistency/ReplicaReadSessionContext.cs @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + [StructLayout(LayoutKind.Explicit, Size = 26)] + public struct ReplicaReadSessionContext + { + /// + /// Session version + /// + [FieldOffset(0)] + public long sessionVersion; + + /// + /// Maximum session sequence number established from all keys read so far + /// + [FieldOffset(8)] + public long maximumSessionSequenceNumber; + + /// + /// Last read hash + /// + [FieldOffset(16)] + public long lastHash; + + /// + /// Last read sublogIdx + /// + [FieldOffset(24)] + public short lastVirtualSublogIdx; + } + + public class ReadSessionState : IDisposable + { + /// + /// GarnetAppendOnlyFile instance + /// + readonly GarnetAppendOnlyFile appendOnlyFile; + + /// + /// Gets the configuration options for the Garnet server. + /// + readonly GarnetServerOptions serverOptions; + + /// + /// Replica read context used with sharded log + /// + ReplicaReadSessionContext replicaReadContext; + + /// + /// Read context for batch reads. Used to track max sequence number of all keys involved in the read. + /// + ReplicaReadSessionContext batchReadContext; + + /// + /// A cancellation token source used to signal cancellation for consistent read operations. + /// + readonly CancellationTokenSource consistentReadCts; + + /// + /// Timeout cancellation token source. + /// + CancellationTokenSource timeoutCts; + + /// + /// Consistent read in progress lock + /// + SingleWriterMultiReaderLock inProgress; + + /// + /// Array of key hashes used for consistent read key batch. + /// + long[] keyHashCache = null; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static int GetPowerOfTwoSize(int value) + => value <= 1 ? 1 : (int)BitOperations.RoundUpToPowerOf2((uint)value); + + void ExpandKeyHashCache(int keyCount) + { + var newSize = GetPowerOfTwoSize(keyCount); + keyHashCache = GC.AllocateArray(newSize, pinned: true); + } + + void ShrinkKeyHashCache(int keyCount) + { + var newSize = GetPowerOfTwoSize(keyCount); + keyHashCache = GC.AllocateArray(newSize, pinned: true); + } + + /// + /// Read session state constructor + /// + /// + /// + public ReadSessionState(GarnetAppendOnlyFile appendOnlyFile, GarnetServerOptions serverOptions) + { + this.appendOnlyFile = appendOnlyFile; + this.serverOptions = serverOptions; + replicaReadContext = new() { sessionVersion = -1, maximumSessionSequenceNumber = 0, lastVirtualSublogIdx = -1 }; + consistentReadCts = new(); + timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(consistentReadCts.Token); + timeoutCts.CancelAfter(serverOptions.ReplicaSyncTimeout); + } + + /// + /// Releases all resources used by the current instance of the class. + /// + public void Dispose() + { + consistentReadCts.Cancel(); + timeoutCts.Cancel(); + inProgress.WriteLock(); + consistentReadCts.Dispose(); + timeoutCts.Dispose(); + } + + void ResetTimeoutCts() + { + if (timeoutCts.TryReset()) + { + timeoutCts.CancelAfter(serverOptions.ReplicaSyncTimeout); + } + else + { + // TryReset failed (too many resets), recreate the CTS + timeoutCts?.Dispose(); + timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(consistentReadCts.Token); + timeoutCts.CancelAfter(serverOptions.ReplicaSyncTimeout); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void BeforeConsistentReadKeyCallback(long hash) + { + if (!inProgress.TryReadLock()) + throw new GarnetException($"Failed to acquire inProgress lock at {nameof(BeforeConsistentReadKeyCallback)}"); + try + { + ResetTimeoutCts(); + appendOnlyFile.readConsistencyManager.BeforeConsistentReadKey(hash & long.MaxValue, ref replicaReadContext, timeoutCts.Token); + } + finally + { + inProgress.ReadUnlock(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AfterConsistentReadKeyCallback() + => appendOnlyFile.readConsistencyManager.AfterConsistentReadKey(ref replicaReadContext); + + /// + /// Initialize context for read key batch. + /// + /// + public void BeforeConsistentReadKeyBatch(ReadOnlySpan parameters) + { + if (!inProgress.TryReadLock()) + throw new GarnetException($"Failed to acquire inProgress lock at {nameof(BeforeConsistentReadKeyCallback)}"); + try + { + var keyCount = parameters.Length; + var consistencyManager = appendOnlyFile.readConsistencyManager; + // First check if version of consistency mananger has changed + appendOnlyFile.readConsistencyManager.CheckConsistencyManagerVersion(ref replicaReadContext); + + // Allocate array to cache key hashes for batch read + if (keyHashCache == null || keyCount > keyHashCache.Length) + ExpandKeyHashCache(keyCount); + else if ((keyCount << 2) < keyHashCache.Length) + ShrinkKeyHashCache(keyCount); + + // NOTE: this context is a copy used to emulate standalone reads. + // The actual update of the session max will happen after the read succeeds. + batchReadContext = replicaReadContext; + for (var i = 0; i < parameters.Length; i++) + { + var key = parameters[i]; + ResetTimeoutCts(); + consistencyManager.BeforeConsistentReadKeyBatch(key.ReadOnlySpan, ref batchReadContext, timeoutCts.Token, out var hash); + keyHashCache[i] = hash; + } + } + finally + { + inProgress.ReadUnlock(); + } + } + + /// + /// Validate keys have not changed after reading a key batch. + /// + /// + /// + public bool AfterConsistentReadKeyBatch(int keyCount) + { + var consistencyManager = appendOnlyFile.readConsistencyManager; + for (var i = 0; i < keyCount; i++) + { + var hash = keyHashCache[i]; + if (!consistencyManager.AfterConsistentReadKeyBatch(hash, ref batchReadContext)) + return false; + } + + // Propagate batch context back to session context to maintain prefix consistency + // for subsequent single-key reads across different sublogs. + replicaReadContext.maximumSessionSequenceNumber = batchReadContext.maximumSessionSequenceNumber; + replicaReadContext.lastVirtualSublogIdx = batchReadContext.lastVirtualSublogIdx; + replicaReadContext.lastHash = batchReadContext.lastHash; + + return true; + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReadConsistency/VirtualSublogReplayState.cs b/libs/server/AOF/ReadConsistency/VirtualSublogReplayState.cs new file mode 100644 index 00000000000..7baa2b148fd --- /dev/null +++ b/libs/server/AOF/ReadConsistency/VirtualSublogReplayState.cs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using Tsavorite.core; + +namespace Garnet.server +{ + internal struct VirtualSublogReplayState + { + const int SketchSlotSize = 1 << 15; + const int SketchSlotMask = SketchSlotSize - 1; + + readonly long[] sketch = new long[SketchSlotSize]; + long sketchMaxValue; + readonly object @lock = new(); + readonly SemaphoreSlim updateSignal = new(0); + int waiterCount; + + public readonly long Max => sketchMaxValue; + + public VirtualSublogReplayState() + { + var size = SketchSlotSize; + if ((size & (size - 1)) != 0) + throw new InvalidOperationException($"Size ({SketchSlotSize}) must be a power of 2"); + Array.Clear(sketch); + sketchMaxValue = 0; + } + + /// + /// Gets the current frontier sequence number associated with the specified hash value. + /// + /// The hash value for which to retrieve the frontier sequence number. + /// The frontier sequence number corresponding to the specified hash value. + public readonly long GetFrontierSequenceNumber(long hash) + => Math.Max(sketch[hash & SketchSlotMask], sketchMaxValue); + + /// + /// Gets the sequence number associated with the specified hash key. + /// + /// The hash value for which to retrieve the sequence number. + /// The sequence number corresponding to the given hash key. + public readonly long GetKeySequenceNumber(long hash) + => sketch[hash & SketchSlotMask]; + + /// + /// Updates the maximum observed sequence number. + /// + /// Updates are thread-safe and guaranteed to be monotonically increasing. + /// The sequence number to compare against the current maximum. + public void UpdateMaxSequenceNumber(long sequenceNumber) + { + _ = Utility.MonotonicUpdate(ref sketchMaxValue, sequenceNumber, out _); + SignalAdvanceTime(); + } + + /// + /// Updates the sequence number associated with the specified key hash. + /// + /// Updates are thread-safe and guaranteed to be monotonically increasing. + /// The hash value identifying the key whose sequence number is to be updated. + /// The new sequence number to associate with the specified key hash. Must be greater than or equal to the + /// current value to have an effect. + public void UpdateKeySequenceNumber(long hash, long sequenceNumber) + { + _ = Utility.MonotonicUpdate(ref sketch[hash & SketchSlotMask], sequenceNumber, out _); + _ = Utility.MonotonicUpdate(ref sketchMaxValue, sequenceNumber, out _); + SignalAdvanceTime(); + } + + /// + /// Signals that time should advance, allowing any awaiting operations to proceed. + /// + void SignalAdvanceTime() + { + var releaseCount = 0; + if (Volatile.Read(ref waiterCount) == 0) + return; + + lock (@lock) + { + releaseCount = waiterCount; + } + + if (releaseCount > 0) + updateSignal.Release(releaseCount); + } + + /// + /// Waits until the session's frontier sequence number for the specified hash reaches or exceeds + /// the given maximum sequence number. + /// + /// The hash value identifying the session whose sequence number is being monitored. + /// The target sequence number to wait for. + /// A cancellation token that can be used to cancel the wait operation. + public void WaitForSequenceNumber(long hash, long maximumSessionSequenceNumber, CancellationToken ct) + { + while (true) + { + lock (@lock) + { + if (maximumSessionSequenceNumber < GetFrontierSequenceNumber(hash)) + return; + + waiterCount++; + } + + try + { + updateSignal.Wait(ct); + } + finally + { + lock (@lock) + { + waiterCount--; + } + } + } + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/Recover/AofRecover.cs b/libs/server/AOF/Recover/AofRecover.cs new file mode 100644 index 00000000000..fb81bc14551 --- /dev/null +++ b/libs/server/AOF/Recover/AofRecover.cs @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Garnet.common; +using Microsoft.Extensions.Logging; + +namespace Garnet.server +{ + public sealed partial class AofProcessor + { + /// + /// Performs recovery of the append-only file (AOF) for the specified database up to the given address, + /// replaying records and logging recovery statistics. + /// + /// The database instance for which AOF recovery is performed. + /// The address up to which the AOF should be recovered. + /// The address up to which the AOF has been successfully recovered. + public AofAddress Recover(GarnetDatabase db, AofAddress untilAddress) + { + Stopwatch swatch = new(); + swatch.Start(); + var total_number_of_replayed_records = 0L; + try + { + storeWrapper.appendOnlyFile.CreateOrUpdateKeySequenceManager(); + logger?.LogInformation("Begin AOF recovery for DB ID: {id}", db.Id); + return RecoverReplayDriver(db, untilAddress); + } + finally + { + storeWrapper.appendOnlyFile.ResetSequenceNumberGenerator(); + var seconds = swatch.ElapsedMilliseconds / 1000.0; + var aofSize = db.AppendOnlyFile.TotalSize(); + var recordsPerSec = total_number_of_replayed_records / seconds; + var GiBperSecs = aofSize / seconds / 1_000_000_000; + + logger?.LogInformation("AOF Recovery in {seconds} secs", seconds); + logger?.LogInformation("Total number of replayed records {total_number_of_replayed_records:N0}", total_number_of_replayed_records); + logger?.LogInformation("Throughput {recordsPerSec:N2} records/sec", recordsPerSec); + logger?.LogInformation("AOF Recovery size {aofSize:N0}", aofSize); + logger?.LogInformation("AOF Recovery throughput {GiBperSecs:N2} GiB/secs", GiBperSecs); + } + + AofAddress RecoverReplayDriver(GarnetDatabase db, AofAddress untilAddress) + { + // Begin replay for specified database + logger?.LogInformation("Begin AOF replay for DB ID: {id}", db.Id); + try + { + // Fetch the database AOF and update the current database context for the processor + var appendOnlyFile = db.AppendOnlyFile; + SwitchActiveDatabaseContext(db); + + // Set the tail address for replay recovery to the tail address of the AOF if none specified + untilAddress.SetValueIf(appendOnlyFile.Log.TailAddress, -1); + + var recordsReplayed = storeWrapper.serverOptions.MultiLogEnabled ? MultiLogRecover(appendOnlyFile, db.Id, untilAddress) : SingleLogRecover(appendOnlyFile, db.Id, 0, untilAddress); + + _ = Interlocked.Add(ref total_number_of_replayed_records, recordsReplayed); + return untilAddress; + } + catch (Exception ex) + { + logger?.LogError(ex, "An error occurred AofProcessor.RecoverReplay"); + + if (storeWrapper.serverOptions.FailOnRecoveryError) + throw; + } + + return AofAddress.Create(storeWrapper.serverOptions.AofPhysicalSublogCount, -1); + } + } + + /// + /// Replays and processes records from a specified physical sublog of the append-only file up to a given address + /// for a particular database. + /// + /// The append-only file containing the sublog to replay. + /// The identifier of the database for which the sublog is being replayed. + /// The index of the physical sublog to process. + /// The address up to which records should be replayed in the sublog. + /// A task representing the asynchronous operation, containing the number of records replayed. + private long SingleLogRecover(GarnetAppendOnlyFile appendOnlyFile, int dbId, int physicalSublogIdx, AofAddress untilAddress) + { + var count = 0L; + var beginAddress = appendOnlyFile.Log.BeginAddress; + using var scan = appendOnlyFile.Log.Scan(physicalSublogIdx, beginAddress[physicalSublogIdx], untilAddress[physicalSublogIdx]); + + // Replay each AOF record in the current database context + while (scan.GetNext(MemoryPool.Shared, out var entry, out var length, out _, out var nextAofAddress)) + { + count++; + unsafe + { + fixed (byte* ptr = entry.Memory.Span) + ProcessAofRecordInternal(physicalSublogIdx, ptr, length, asReplica: false, out _); + entry.Dispose(); + } + + if (count % 100_000 == 0) + { + logger?.LogTrace("Completed AOF replay of {count} records, until AOF address {nextAofAddress} (DB ID: {id})", count, nextAofAddress, dbId); + } + } + + logger?.LogInformation("Completed full AOF sublog {sublogIdx} replay of {count:N0} records (DB ID: {id})", physicalSublogIdx, count, dbId); + return count; + } + + /// + /// Recovers log records from the specified append-only file up to a given address. + /// + /// The append-only file from which log records are recovered. This file must be valid and accessible for + /// recovery to proceed. + /// The identifier of the database for which the log recovery is being performed. This must correspond to a + /// valid database context. + /// An address indicating the point up to which log records should be recovered. This must be within the valid + /// range of the log. + /// The total number of log records that were successfully replayed during the recovery process. + private long MultiLogRecover(GarnetAppendOnlyFile appendOnlyFile, int dbId, AofAddress untilAddress) + { + var recordsReplayed = 0L; + if (appendOnlyFile.Log.RecoverLatestSequenceNumber(out var recoverUntilSequenceNumber)) + { + var beginAddress = appendOnlyFile.Log.BeginAddress; + var recoverDrivers = new RecoverLogDriver[untilAddress.Length]; + for (var physicalSublogIdx = 0; physicalSublogIdx < untilAddress.Length; physicalSublogIdx++) + { + recoverDrivers[physicalSublogIdx] = new RecoverLogDriver( + this, + appendOnlyFile, + storeWrapper.serverOptions, + dbId, + physicalSublogIdx, + beginAddress[physicalSublogIdx], + untilAddress[physicalSublogIdx], + recoverUntilSequenceNumber, + logger); + } + + try + { + // TODO: Can we async this method rather than blocking? We're in recovery. + var recoveryTasks = Task.WhenAll([.. recoverDrivers.Select(driver => driver.CreateRecoverTaskAsync())]); + AsyncUtils.BlockingWait(recoveryTasks); + + recordsReplayed = recoverDrivers.Sum(driver => driver.ReplayedRecordCount); + } + finally + { + for (var physicalSublogIdx = 0; physicalSublogIdx < untilAddress.Length; physicalSublogIdx++) + recoverDrivers[physicalSublogIdx]?.Dispose(); + } + } + + return recordsReplayed; + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/Recover/RecoverLogDriver.cs b/libs/server/AOF/Recover/RecoverLogDriver.cs new file mode 100644 index 00000000000..79532874994 --- /dev/null +++ b/libs/server/AOF/Recover/RecoverLogDriver.cs @@ -0,0 +1,273 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Initializes a new instance of the RecoverLogDriver class for replaying a segment of an append-only file + /// for recovery. + /// + /// Processor responsible for handling append-only file operations. + /// The append-only file to be scanned for recovery. + /// Configuration options for the server. + /// Identifier of the database we are recovering. + /// Index of the physical sublog to scan. + /// Start address in the append-only file for recovery. + /// End address in the append-only file for recovery. + /// Replay all records with sequence number to ensure prefix consistent recovery. + /// Optional logger for diagnostic output. + internal sealed class RecoverLogDriver( + AofProcessor aofProcessor, + GarnetAppendOnlyFile appendOnlyFile, + GarnetServerOptions serverOptions, + int dbId, + int physicalSublogIdx, + long startAddress, + long untilAddress, + long untilSequenceNumber, + ILogger logger = null) : IBulkLogEntryConsumer, IDisposable + { + readonly int physicalSublogIdx = physicalSublogIdx; + readonly AofProcessor aofProcessor = aofProcessor; + readonly GarnetServerOptions serverOptions = serverOptions; + readonly GarnetAppendOnlyFile appendOnlyFile = appendOnlyFile; + readonly TsavoriteLogScanSingleIterator replayIterator = appendOnlyFile.Log.ScanSingle(physicalSublogIdx, startAddress, untilAddress, scanUncommitted: true, recover: false, logger: logger); + readonly TsavoriteLog physicalSublog = appendOnlyFile.Log.GetSubLog(physicalSublogIdx); + readonly CancellationTokenSource cts = new(); + readonly ILogger logger = logger; + readonly long startAddress = startAddress; + readonly long untilAddress = untilAddress; + readonly long untilSequenceNumber = untilSequenceNumber; + readonly int dbId = dbId; + readonly ReplayBatchContext replayBatchContext = new(serverOptions.AofReplayTaskCount); + Task[] replayTasks = null; + + /// + /// Gets the total number of records that have been replayed. + /// + public long ReplayedRecordCount { get; private set; } = 0; + + public void Dispose() + { + replayIterator?.Dispose(); + cts?.Dispose(); + } + + /// + /// Main consume method for recover driver. + /// + /// + /// + /// + /// + /// + public unsafe void Consume(byte* record, int recordLength, long currentAddress, long nextAddress, bool isProtected) + { + if (serverOptions.AofReplayTaskCount == 1) + { + // Recover/Replay on this consume thread + var ptr = record; + while (ptr < record + recordLength) + { + var entryLength = appendOnlyFile.HeaderSize; + var payloadLength = physicalSublog.UnsafeGetLength(ptr); + if (payloadLength > 0) + { + var entryPtr = ptr + entryLength; + var logAddressSequenceNumber = currentAddress + (ptr - record); + Debug.Assert(logAddressSequenceNumber > 0, "Entry log address must be positive"); + if (!aofProcessor.SkipReplay(entryPtr, untilSequenceNumber, logAddressSequenceNumber, out var sequenceNumber)) + { + aofProcessor.ProcessAofRecordInternal(physicalSublogIdx, entryPtr, payloadLength, true, out _, logAddressSequenceNumber); + } + else + { + // Sequence numbers are monotonically increasing — all subsequent entries will also exceed the threshold + logger?.LogTrace("Skipping entry replay {entrySequenceNumber} > {untilSequenceNumber}, stopping", sequenceNumber, untilSequenceNumber); + cts.Cancel(); + break; + } + entryLength += TsavoriteLog.UnsafeAlign(payloadLength); + } + else if (payloadLength < 0) + { + TsavoriteLogRecoveryInfo info = new(); + info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); + physicalSublog.UnsafeCommitMetadataOnly(info, isProtected); + entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); + } + ptr += entryLength; + + ReplayedRecordCount++; + if (ReplayedRecordCount % 100_000 == 0) + { + logger?.LogTrace("Completed AOF replay of {count} records, until AOF address {nextAofAddress} (DB ID: {id})", ReplayedRecordCount, untilAddress, dbId); + } + } + + // Completed replay + if (nextAddress == untilAddress) + cts.Cancel(); + } + else + { + // Wait for previous batch to complete before overwriting shared batch context + if (replayTasks != null) + { + replayBatchContext.LeaderFollowerBarrier.WaitCompleted(); + replayBatchContext.LeaderFollowerBarrier.Release(); + } + + CreateAndRunIntraPageParallelReplayTasks(); + + replayBatchContext.Record = record; + replayBatchContext.RecordLength = recordLength; + replayBatchContext.CurrentAddress = currentAddress; + replayBatchContext.NextAddress = nextAddress; + replayBatchContext.IsProtected = isProtected; + replayBatchContext.LeaderFollowerBarrier.SignalWorkReady(); + + // After the last batch, wait for workers and cancel to exit BulkConsumeAllAsync + if (nextAddress == untilAddress) + { + replayBatchContext.LeaderFollowerBarrier.WaitCompleted(); + replayBatchContext.LeaderFollowerBarrier.Release(); + cts.Cancel(); + } + } + } + + private void CreateAndRunIntraPageParallelReplayTasks() + => replayTasks ??= [.. Enumerable.Range(0, serverOptions.AofReplayTaskCount).Select(i => Task.Run(() => ContinuousBackgroundReplayAsync(i, physicalSublog)))]; + + internal async Task ContinuousBackgroundReplayAsync(int replayTaskIdx, TsavoriteLog replaySublog) + { + var virtualSublogIdx = appendOnlyFile.GetVirtualSublogIdx(physicalSublogIdx, replayTaskIdx); + while (!cts.Token.IsCancellationRequested) + { + try + { + await replayBatchContext.LeaderFollowerBarrier.WaitReadyWorkAsync(cancellationToken: cts.Token).ConfigureAwait(false); + } + catch (TaskCanceledException) when (cts.IsCancellationRequested) + { } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed at WaitAsync", nameof(ContinuousBackgroundReplayAsync)); + await cts.CancelAsync().ConfigureAwait(false); + break; + } + + try + { + unsafe + { + + var record = replayBatchContext.Record; + var recordLength = replayBatchContext.RecordLength; + var currentAddress = replayBatchContext.CurrentAddress; + var nextAddress = replayBatchContext.NextAddress; + var isProtected = replayBatchContext.IsProtected; + var ptr = record; + + var maxSequenceNumber = 0L; + + // logger?.LogError("[{sublogIdx},{replayIdx}] = {currentAddress} -> {nextAddress}", sublogIdx, replayIdx, currentAddress, nextAddress); + while (ptr < record + recordLength) + { + cts.Token.ThrowIfCancellationRequested(); + var entryLength = appendOnlyFile.HeaderSize; + var payloadLength = replaySublog.UnsafeGetLength(ptr); + if (payloadLength > 0) + { + var entryPtr = ptr + entryLength; + var logAddressSequenceNumber = currentAddress + (ptr - record); + Debug.Assert(logAddressSequenceNumber > 0, "Entry log address must be positive"); + // Check if entry is assigned for processing to this replay task and + // the sequence number is below the threshold to ensure prefix consistency + if (aofProcessor.CanReplay(entryPtr, replayTaskIdx, logAddressSequenceNumber, out var sequenceNumber)) + { + if (untilSequenceNumber != -1 && sequenceNumber > untilSequenceNumber) + { + // Sequence numbers are monotonically increasing — stop processing this batch + break; + } + aofProcessor.ProcessAofRecordInternal(virtualSublogIdx, entryPtr, payloadLength, true, out var isCheckpointStart, logAddressSequenceNumber); + maxSequenceNumber = Math.Max(sequenceNumber, maxSequenceNumber); + } + entryLength += TsavoriteLog.UnsafeAlign(payloadLength); + } + else if (payloadLength < 0) + { + // Only a single thread should commit metadata + if (replayTaskIdx == 0) + { + TsavoriteLogRecoveryInfo info = new(); + info.Initialize(new ReadOnlySpan(ptr + entryLength, -payloadLength)); + replaySublog.UnsafeCommitMetadataOnly(info, isProtected); + } + entryLength += TsavoriteLog.UnsafeAlign(-payloadLength); + } + ptr += entryLength; + } + + // Update max sequence number for this virtual sublog which is mapped + appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(virtualSublogIdx, maxSequenceNumber); + } + } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed at replaying", nameof(ContinuousBackgroundReplayAsync)); + await cts.CancelAsync().ConfigureAwait(false); + break; + } + finally + { + // Signal work completion after processing (skip if cancelled to avoid blocking on resetReady) + if (!cts.Token.IsCancellationRequested) + replayBatchContext.LeaderFollowerBarrier.SignalCompleted(); + } + } + } + + public void Throttle() { } + + /// + /// Starts a background task to replay and recover data until a specified address or when cancellation is requested. + /// + /// A Task representing the asynchronous recovery operation. + public Task CreateRecoverTaskAsync() + { + return Task.Run(async () => + { + try + { + if (startAddress == untilAddress) return; + logger?.LogInformation("Recover sublog [{physicalSublogIdx}] for addres range ({startAddress},{untilAddress})", physicalSublogIdx, startAddress, untilAddress); + while (!cts.IsCancellationRequested) + { + await replayIterator.BulkConsumeAllAsync( + this, + serverOptions.ReplicaSyncDelayMs, + maxChunkSize: 1 << 20, + cts.Token).ConfigureAwait(false); + + // Replay completed + if (replayIterator.NextAddress == untilAddress) + break; + } + } + catch (TaskCanceledException) when (cts.IsCancellationRequested) + { } + }); + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReplayBatchContext.cs b/libs/server/AOF/ReplayBatchContext.cs new file mode 100644 index 00000000000..d91034589d9 --- /dev/null +++ b/libs/server/AOF/ReplayBatchContext.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.common; + +namespace Garnet.server +{ + /// + /// Replay work item used with recover/replication replay. + /// + /// + public unsafe class ReplayBatchContext(int replayTasks) + { + /// + /// Record pointer. + /// + public byte* Record; + /// + /// Record length. + /// + public int RecordLength; + /// + /// Represents the current address value for a given TsavoriteLog page. + /// + public long CurrentAddress; + /// + /// Represents the next address value for a given TsavoriteLog page. + /// + public long NextAddress; + /// + /// Whether replay occurs under epoch protections. + /// + public bool IsProtected; + /// + /// Leader barrier to coordinate replication offset update. + /// + public LeaderFollowerBarrier LeaderFollowerBarrier = new(replayTasks); + } +} \ No newline at end of file diff --git a/libs/server/AOF/ReplayCoordinator/AofReplayContext.cs b/libs/server/AOF/ReplayCoordinator/AofReplayContext.cs index 7b5f142f121..adc7ea2cc1d 100644 --- a/libs/server/AOF/ReplayCoordinator/AofReplayContext.cs +++ b/libs/server/AOF/ReplayCoordinator/AofReplayContext.cs @@ -9,23 +9,27 @@ namespace Garnet.server { /// - /// AofReplayContext + /// Sublog replay buffer (one for each sublog) /// - public sealed class AofReplayContext + internal sealed class AofReplayContext { public readonly List fuzzyRegionOps = []; public readonly Queue txnGroupBuffer = []; public readonly Dictionary activeTxns = []; - public readonly RawStringInput storeInput; - public readonly ObjectInput objectStoreInput; + internal readonly RespServerSession respServerSession; + public CustomProcedureInput customProcInput; - public readonly SessionParseState parseState; + public SessionParseState parseState; public readonly byte[] objectOutputBuffer; public MemoryResult output; + public StringBasicContext StringBasicContext => respServerSession.storageSession.stringBasicContext; + public ObjectBasicContext ObjectBasicContext => respServerSession.storageSession.objectBasicContext.Session == null ? default : respServerSession.storageSession.objectBasicContext.Session.BasicContext; + public UnifiedBasicContext UnifiedBasicContext => respServerSession.storageSession.unifiedBasicContext; + /// /// Fuzzy region of AOF is the region between the checkpoint start and end commit markers. /// This regions can contain entries in both (v) and (v+1) versions. The processing logic is: @@ -37,23 +41,37 @@ public sealed class AofReplayContext public bool inFuzzyRegion = false; /// - /// AofReplayContext constructor + /// AOF replay context constructor /// - public AofReplayContext() + public AofReplayContext(RespServerSession respServerSession) { + this.respServerSession = respServerSession; parseState.Initialize(); - storeInput.parseState = parseState; - objectStoreInput.parseState = parseState; customProcInput.parseState = parseState; objectOutputBuffer = GC.AllocateArray(BufferSizeUtils.ServerBufferSize(new MaxSizeSettings()), pinned: true); } + public void Dispose() + { + var databaseSessionsSnapshot = respServerSession.GetDatabaseSessionsSnapshot(); + foreach (var dbSession in databaseSessionsSnapshot) + { + dbSession.StorageSession.stringBasicContext.Session?.Dispose(); + dbSession.StorageSession.objectBasicContext.Session?.Dispose(); + } + respServerSession?.Dispose(); + output.MemoryOwner?.Dispose(); + } + /// /// Add transaction group to this replay buffer /// /// - public void AddTransactionGroup(int sessionID) - => activeTxns[sessionID] = new(); + /// + /// + /// Sequence number or entry address of the TxnStart entry + public void AddTransactionGroup(int sessionID, int sublogIdx, byte logAccessBitmap, long startSequenceNumber = 0) + => activeTxns[sessionID] = new(sublogIdx, logAccessBitmap, startSequenceNumber); /// /// Add transaction group to fuzzy region buffer diff --git a/libs/server/AOF/ReplayCoordinator/AofReplayCoordinator.cs b/libs/server/AOF/ReplayCoordinator/AofReplayCoordinator.cs index 556dff60dd8..0c1d34c6c5b 100644 --- a/libs/server/AOF/ReplayCoordinator/AofReplayCoordinator.cs +++ b/libs/server/AOF/ReplayCoordinator/AofReplayCoordinator.cs @@ -2,19 +2,49 @@ // Licensed under the MIT license. using System; +using System.Collections.Concurrent; using System.Diagnostics; -using System.Runtime.CompilerServices; +using System.Threading.Tasks; using Garnet.common; using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; + /// + /// Used to index barriers for coordinated replay operations in the AofReplayCoordinator + /// NOTE: Use only negative numbers because sessionIDs will be positive values + /// + internal enum LeaderBarrierType : int + { + CHECKPOINT = -1, + STREAMING_CHECKPOINT = -2, + FLUSH_DB = -3, + FLUSH_DB_ALL = -4, + CUSTOM_STORED_PROC = -5, + } + + struct BarrierKey : IEquatable + { + /// + /// Session Id + /// + public int SessionId; + + /// + /// Transaction Id + /// + public long txnId; - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; + public bool Equals(BarrierKey other) + => SessionId == other.SessionId && txnId == other.txnId; + + public override bool Equals(object obj) + => obj is BarrierKey other && Equals(other); + + public override int GetHashCode() + => HashCode.Combine(SessionId, txnId); + } public sealed unsafe partial class AofProcessor { @@ -27,18 +57,31 @@ public sealed unsafe partial class AofProcessor /// transactions and operations, ensuring consistency and correctness during AOF replay. The is designed to work with an to facilitate the /// replay of operations. + /// /// /// - public class AofReplayCoordinator(AofProcessor aofProcessor, ILogger logger = null) : IDisposable + public class AofReplayCoordinator(GarnetServerOptions serverOptions, AofProcessor aofProcessor, ILogger logger = null) : IDisposable { + readonly GarnetServerOptions serverOptions = serverOptions; + readonly ConcurrentDictionary leaderBarriers = []; readonly AofProcessor aofProcessor = aofProcessor; - readonly AofReplayContext aofReplayContext = InitializeReplayContext(); - public AofReplayContext GetReplayContext() => aofReplayContext; + readonly AofReplayContext[] aofReplayContext = InitializeReplayContext(serverOptions.AofVirtualSublogCount, aofProcessor); + SingleWriterMultiReaderLock disposed = new(); readonly ILogger logger = logger; - internal static AofReplayContext InitializeReplayContext() + /// + /// Replay context for replay subtask + /// + /// + /// + internal AofReplayContext GetReplayContext(int sublogIdx) => aofReplayContext[sublogIdx]; + + internal static AofReplayContext[] InitializeReplayContext(int AofVirtualSublogCount, AofProcessor aofProcessor) { - return new AofReplayContext(); + var virtualSublogReplayContext = new AofReplayContext[AofVirtualSublogCount]; + for (var i = 0; i < virtualSublogReplayContext.Length; i++) + virtualSublogReplayContext[i] = new(aofProcessor.ObtainServerSession()); + return virtualSublogReplayContext; } /// @@ -46,25 +89,30 @@ internal static AofReplayContext InitializeReplayContext() /// public void Dispose() { - aofReplayContext.output.MemoryOwner?.Dispose(); + if (!disposed.TryWriteLock()) return; + foreach (var replayContext in aofReplayContext) + replayContext.Dispose(); } /// /// Get fuzzy region buffer count /// + /// /// - internal int FuzzyRegionBufferCount() => aofReplayContext.fuzzyRegionOps.Count; + internal int FuzzyRegionBufferCount(int sublogIdx) => aofReplayContext[sublogIdx].fuzzyRegionOps.Count; /// /// Clear fuzzy region buffer /// - internal void ClearFuzzyRegionBuffer() => aofReplayContext.fuzzyRegionOps.Clear(); + /// + internal void ClearFuzzyRegionBuffer(int sublogIdx) => aofReplayContext[sublogIdx].fuzzyRegionOps.Clear(); /// /// Add single operation to fuzzy region buffer /// + /// /// - internal unsafe void AddFuzzyRegionOperation(ReadOnlySpan entry) => aofReplayContext.fuzzyRegionOps.Add(entry.ToArray()); + internal void AddFuzzyRegionOperation(int sublogIdx, ReadOnlySpan entry) => aofReplayContext[sublogIdx].fuzzyRegionOps.Add(entry.ToArray()); /// /// This method will perform one of the following @@ -73,17 +121,20 @@ public void Dispose() /// 3. TxnAbort: Clear corresponding sublog replay buffer. /// 4. Default: Add an operation to an existing transaction group /// + /// /// /// /// + /// /// Returns true if a txn operation was processed and added otherwise false /// - internal unsafe bool AddOrReplayTransactionOperation(byte* ptr, int length, bool asReplica) + internal bool AddOrReplayTransactionOperation(int virtualSublogIdx, byte* ptr, int length, bool asReplica, long logAddressSequenceNumber = 0) { var header = *(AofHeader*)ptr; - var replayContext = GetReplayContext(); - // First try to process this as an existing transaction - if (aofReplayContext.activeTxns.TryGetValue(header.sessionID, out var group)) + var replayContext = GetReplayContext(virtualSublogIdx); + // Process operation as part of a transaction if it belongs to the same sessionId and + // there is already a transaction group associated with it. + if (aofReplayContext[virtualSublogIdx].activeTxns.TryGetValue(header.sessionID, out var group)) { switch (header.opType) { @@ -91,6 +142,7 @@ internal unsafe bool AddOrReplayTransactionOperation(byte* ptr, int length, bool throw new GarnetException("No nested transactions expected"); case AofEntryType.TxnAbort: ClearSessionTxn(); + UpdateMaxSequenceNumberFromHeader(); break; case AofEntryType.TxnCommit: if (replayContext.inFuzzyRegion) @@ -98,12 +150,12 @@ internal unsafe bool AddOrReplayTransactionOperation(byte* ptr, int length, bool // If in fuzzy region we want to record the commit marker and // buffer the transaction group for later replay var commitMarker = new ReadOnlySpan(ptr, length); - aofReplayContext.AddToFuzzyRegionBuffer(group, commitMarker); + aofReplayContext[virtualSublogIdx].AddToFuzzyRegionBuffer(group, commitMarker); } else { // Otherwise process transaction group immediately - ProcessTransactionGroup(ptr, asReplica, group); + ProcessTransactionGroup(virtualSublogIdx, ptr, asReplica, group, logAddressSequenceNumber); } // We want to clear and remove in both cases to make space for next txn from session @@ -112,14 +164,14 @@ internal unsafe bool AddOrReplayTransactionOperation(byte* ptr, int length, bool case AofEntryType.StoredProcedure: throw new GarnetException($"Unexpected AOF header operation type {header.opType} within transaction"); default: - group.operations.Add(new ReadOnlySpan(ptr, length).ToArray()); + group.Operations.Add(new ReadOnlySpan(ptr, length).ToArray()); break; } void ClearSessionTxn() { - aofReplayContext.activeTxns[header.sessionID].Clear(); - _ = aofReplayContext.activeTxns.Remove(header.sessionID); + aofReplayContext[virtualSublogIdx].activeTxns[header.sessionID].Clear(); + _ = aofReplayContext[virtualSublogIdx].activeTxns.Remove(header.sessionID); } return true; @@ -129,13 +181,36 @@ void ClearSessionTxn() switch (header.opType) { case AofEntryType.TxnStart: - aofReplayContext.AddTransactionGroup(header.sessionID); + var headerType = header.HeaderType; + short logAccessCount = 0; + long startSeqNum = 0; + if (serverOptions.MultiLogEnabled) + { + switch (headerType) + { + case AofHeaderType.SingleLogTransactionHeader: + logAccessCount = (*(AofSingleLogTransactionHeader*)ptr).participantCount; + startSeqNum = logAddressSequenceNumber; + break; + case AofHeaderType.ShardedLogTransactionHeader: + logAccessCount = (*(AofShardedLogTransactionHeader*)ptr).participantCount; + startSeqNum = (*(AofShardedHeader*)ptr).sequenceNumber; + break; + default: + // BasicHeader from SL-era AOF: all replay tasks participate + logAccessCount = (short)serverOptions.AofReplayTaskCount; + startSeqNum = logAddressSequenceNumber; + break; + } + } + aofReplayContext[virtualSublogIdx].AddTransactionGroup(header.sessionID, virtualSublogIdx, (byte)logAccessCount, startSeqNum); break; case AofEntryType.TxnAbort: case AofEntryType.TxnCommit: // We encountered a transaction end without start - this could happen because we truncated the AOF // after a checkpoint, and the transaction belonged to the previous version. It can safely // be ignored. + UpdateMaxSequenceNumberFromHeader(); break; default: // Continue processing @@ -144,118 +219,221 @@ void ClearSessionTxn() // Processed this record successfully return true; + + // U + void UpdateMaxSequenceNumberFromHeader() + { + var headerType = (*(AofHeader*)ptr).HeaderType; + long sequenceNumber; + switch (headerType) + { + case AofHeaderType.BasicHeader: + case AofHeaderType.SingleLogTransactionHeader: + sequenceNumber = logAddressSequenceNumber; + break; + case AofHeaderType.ShardedHeader: + sequenceNumber = (*(AofShardedHeader*)ptr).sequenceNumber; + break; + case AofHeaderType.ShardedLogTransactionHeader: + sequenceNumber = (*(AofShardedLogTransactionHeader*)ptr).shardedHeader.sequenceNumber; + break; + default: + throw new GarnetException($"Unexpected header type {headerType}"); + } + aofProcessor.storeWrapper.appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(virtualSublogIdx, sequenceNumber); + } } /// /// Process fuzzy region operations if any /// + /// /// /// - internal void ProcessFuzzyRegionOperations(long storeVersion, bool asReplica) + internal void ProcessFuzzyRegionOperations(int sublogIdx, long storeVersion, bool asReplica) { - var fuzzyRegionOps = aofReplayContext.fuzzyRegionOps; + var fuzzyRegionOps = aofReplayContext[sublogIdx].fuzzyRegionOps; if (fuzzyRegionOps.Count > 0) - logger?.LogInformation("Replaying {fuzzyRegionBufferCount} records from fuzzy region for checkpoint {newVersion}", fuzzyRegionOps.Count, storeVersion); + logger?.LogInformation("Replaying sublogIdx: {sublogIdx} - {fuzzyRegionBufferCount} records from fuzzy region for checkpoint {newVersion}", sublogIdx, fuzzyRegionOps.Count, storeVersion); + var replayContext = GetReplayContext(sublogIdx); foreach (var entry in fuzzyRegionOps) { fixed (byte* entryPtr = entry) - _ = aofProcessor.ReplayOp(aofProcessor.basicContext, aofProcessor.objectStoreBasicContext, entryPtr, entry.Length, asReplica); + { + var header = *(AofHeader*)entryPtr; + _ = aofProcessor.ReplayOpDispatch( + sublogIdx, + header, + replayContext, + replayContext.StringBasicContext, + replayContext.ObjectBasicContext, + replayContext.UnifiedBasicContext, + entryPtr, + entry.Length, + asReplica); + } } } /// /// Process fuzzy region transaction groups /// + /// /// /// - internal void ProcessFuzzyRegionTransactionGroup(byte* ptr, bool asReplica) + /// Log address of the commit entry + internal void ProcessFuzzyRegionTransactionGroup(int sublogIdx, byte* ptr, bool asReplica, long entryAddress = 0) { - Debug.Assert(aofReplayContext.txnGroupBuffer != null); + Debug.Assert(aofReplayContext[sublogIdx].txnGroupBuffer != null); // Process transaction groups in FIFO order - if (aofReplayContext.txnGroupBuffer.Count > 0) - { - var txnGroup = aofReplayContext.txnGroupBuffer.Dequeue(); - ProcessTransactionGroup(ptr, asReplica, txnGroup); - } + var txnGroup = aofReplayContext[sublogIdx].txnGroupBuffer.Dequeue(); + ProcessTransactionGroup(sublogIdx, ptr, asReplica, txnGroup, entryAddress); } /// /// Process provided transaction group /// + /// + /// /// /// - internal void ProcessTransactionGroup(byte* ptr, bool asReplica, TransactionGroup txnGroup) + /// Log address of the commit entry + internal void ProcessTransactionGroup(int sublogIdx, byte* ptr, bool asReplica, TransactionGroup txnGroup, long entryAddress = 0) { + var replayContext = GetReplayContext(sublogIdx); if (!asReplica) { // If recovering reads will not expose partial transactions so we can replay without locking. // Also we don't have to synchronize replay of sublogs because write ordering has been established at the time of enqueue. - ProcessTransactionGroupOperations(aofProcessor, aofProcessor.basicContext, aofProcessor.objectStoreBasicContext, txnGroup, asReplica); + ProcessTransactionGroupOperations( + aofProcessor, + replayContext.StringBasicContext, + replayContext.ObjectBasicContext, + replayContext.UnifiedBasicContext, + txnGroup, + asReplica, + entryAddress); } else { - var txnManager = aofProcessor.respServerSession.txnManager; + var txnManager = replayContext.respServerSession.txnManager; // Start by saving transaction keys for locking SaveTransactionGroupKeysToLock(txnManager, txnGroup); - // Start transaction - _ = txnManager.Run(internal_txn: true); + if (serverOptions.MultiLogEnabled) + { + var headerType = (AofHeaderType)(*(AofHeader*)ptr).HeaderType; + long commitSeqNum; + short partCount; + var sessionId = (*(AofHeader*)ptr).sessionID; - // Process in parallel transaction group - ProcessTransactionGroupOperations(aofProcessor, txnManager.LockableContext, txnManager.ObjectStoreLockableContext, txnGroup, asReplica); + if (headerType == AofHeaderType.SingleLogTransactionHeader) + { + commitSeqNum = entryAddress; + partCount = (*(AofSingleLogTransactionHeader*)ptr).participantCount; + } + else if (headerType == AofHeaderType.ShardedLogTransactionHeader) + { + var shardedHeader = *(AofShardedHeader*)ptr; + commitSeqNum = shardedHeader.sequenceNumber; + partCount = (*(AofShardedLogTransactionHeader*)ptr).participantCount; + } + else + { + // BasicHeader from SL-era AOF: all replay tasks participate + commitSeqNum = entryAddress; + partCount = (short)serverOptions.AofReplayTaskCount; + } + + // Acquire-barrier: synchronize all participants before locking using TxnStart sequence number + ProcessSynchronizedOperation( + sublogIdx, + txnGroup.StartSequenceNumber, + partCount, + sessionId, + null); - // NOTE: - // This txnManager instance is taken from a session with StoreWrapper(recordToAof=false). - // For this reason its internal appendOnlyFile instance is null. - // Hence this commit will not write into the replica's Aof file as it is required. - Debug.Assert(!txnManager.AofEnabled); + // Start transaction (acquires locks) + _ = txnManager.Run(internal_txn: true); + + // Process transaction group operations + ProcessTransactionGroupOperations( + aofProcessor, + txnManager.StringTransactionalContext, + txnManager.ObjectTransactionalContext, + txnManager.UnifiedTransactionalContext, + txnGroup, + asReplica, + entryAddress); + + // Release-barrier: synchronize all participants before committing using TxnCommit sequence number + ProcessSynchronizedOperation( + sublogIdx, + commitSeqNum, + partCount, + sessionId, + null); + } + else + { + // Single-log: no synchronization needed + _ = txnManager.Run(internal_txn: true); + + ProcessTransactionGroupOperations( + aofProcessor, + txnManager.StringTransactionalContext, + txnManager.ObjectTransactionalContext, + txnManager.UnifiedTransactionalContext, + txnGroup, + asReplica, + entryAddress); + } + + // Commit (NOTE: need to ensure that we do not write to log here) txnManager.Commit(true); } // Helper to iterate of transaction keys and add them to lockset - static unsafe void SaveTransactionGroupKeysToLock(TransactionManager txnManager, TransactionGroup txnGroup) + static void SaveTransactionGroupKeysToLock(TransactionManager txnManager, TransactionGroup txnGroup) { - foreach (var entry in txnGroup.operations) + foreach (var entry in txnGroup.Operations) { - ref var key = ref Unsafe.NullRef(); fixed (byte* entryPtr = entry) { - var header = *(AofHeader*)entryPtr; - var isObject = false; - switch (header.opType) - { - case AofEntryType.StoreUpsert: - case AofEntryType.StoreRMW: - case AofEntryType.StoreDelete: - key = ref Unsafe.AsRef(entryPtr + sizeof(AofHeader)); - isObject = false; - break; - case AofEntryType.ObjectStoreUpsert: - case AofEntryType.ObjectStoreRMW: - case AofEntryType.ObjectStoreDelete: - key = ref Unsafe.AsRef(entryPtr + sizeof(AofHeader)); - isObject = true; - break; - default: - throw new GarnetException($"Invalid replay operation {header.opType} within transaction"); - } - - // Add key to the lockset - txnManager.SaveKeyEntryToLock(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan()), isObject: isObject, LockType.Exclusive); + var curr = AofHeader.SkipHeader(entryPtr); + var key = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + txnManager.SaveKeyEntryToLock(key, LockType.Exclusive); } } } - // Process transaction - static void ProcessTransactionGroupOperations(AofProcessor aofProcessor, TContext context, TObjectContext objectContext, TransactionGroup txnGroup, bool asReplica) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext + // Process transaction + static void ProcessTransactionGroupOperations(AofProcessor aofProcessor, + TStringContext stringContext, TObjectContext objectContext, TUnifiedContext unifiedContext, + TransactionGroup txnGroup, bool asReplica, long entryAddress = 0) + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext { - foreach (var entry in txnGroup.operations) + var replayContext = aofProcessor.aofReplayCoordinator.GetReplayContext(txnGroup.VirtualSublogIdx); + foreach (var entry in txnGroup.Operations) { fixed (byte* entryPtr = entry) - _ = aofProcessor.ReplayOp(context, objectContext, entryPtr, entry.Length, asReplica: asReplica); + { + var header = *(AofHeader*)entryPtr; + _ = aofProcessor.ReplayOpDispatch( + txnGroup.VirtualSublogIdx, + header, + replayContext, + stringContext, + objectContext, + unifiedContext, + entryPtr, + entry.Length, + asReplica: asReplica, + logAddressSequenceNumber: entryAddress); + } } } } @@ -263,23 +441,164 @@ static void ProcessTransactionGroupOperations(AofProce /// /// Replay StoredProc wrapper for single and sharded logs /// + /// /// /// - internal void ReplayStoredProc(byte id, byte* ptr) + /// Log address of the entry, used for single-physical-log mode + internal void ReplayStoredProc(int sublogIdx, byte id, byte* ptr, long entryAddress = 0) { - StoredProcRunnerBase(id, ptr); + if (!serverOptions.MultiLogEnabled) + { + StoredProcRunnerBase(0, id, ptr, shardedLog: false, null); + } + else + { + var headerType = (AofHeaderType)(*(AofHeader*)ptr).HeaderType; + long sequenceNumber; + short participantCount; + int sessionId = (*(AofHeader*)ptr).sessionID; + + if (headerType == AofHeaderType.SingleLogTransactionHeader) + { + var singleLogHeader = *(AofSingleLogTransactionHeader*)ptr; + sequenceNumber = entryAddress; + participantCount = singleLogHeader.participantCount; + } + else if (headerType == AofHeaderType.ShardedLogTransactionHeader) + { + var shardedHeader = *(AofShardedHeader*)ptr; + sequenceNumber = shardedHeader.sequenceNumber; + participantCount = (*(AofShardedLogTransactionHeader*)ptr).participantCount; + } + else + { + // BasicHeader from SL-era AOF: all replay tasks participate + sequenceNumber = entryAddress; + participantCount = (short)serverOptions.AofReplayTaskCount; + } + + // Synchronized processing of stored proc operation + ProcessSynchronizedOperation( + sublogIdx, + sequenceNumber, + participantCount, + sessionId, + () => { StoredProcRunnerWrapper(sublogIdx, id, ptr, sequenceNumber); return Task.CompletedTask; } + ); + + // Wrapper for store proc runner used for multi-log synchronization + void StoredProcRunnerWrapper(int sublogIdx, byte id, byte* ptr, long seqNum) + { + // Initialize custom proc collection to keep track of hashes for keys for which their timestamp needs to be updated + CustomProcedureKeyHashCollection customProcKeyHashTracker = new(aofProcessor.storeWrapper.appendOnlyFile); + + // Update timestamps for associated keys + customProcKeyHashTracker?.UpdateSequenceNumber(seqNum); + + // Replay StoredProc + StoredProcRunnerBase(sublogIdx, id, ptr, shardedLog: true, customProcKeyHashTracker); + } + } // Based run stored proc method used of legacy single log implementation - void StoredProcRunnerBase(byte id, byte* ptr) + void StoredProcRunnerBase(int sublogIdx, byte id, byte* entryPtr, bool shardedLog, CustomProcedureKeyHashCollection customProcKeyHashTracker) { - var curr = ptr + sizeof(AofHeader); + var curr = AofHeader.SkipHeader(entryPtr); + var replayContext = aofReplayContext[sublogIdx]; // Reconstructing CustomProcedureInput - _ = aofReplayContext.customProcInput.DeserializeFrom(curr); + _ = replayContext.customProcInput.DeserializeFrom(curr); - // Run the stored procedure with the reconstructed input - var output = aofReplayContext.output; - _ = aofProcessor.respServerSession.RunTransactionProc(id, ref aofReplayContext.customProcInput, ref output, isReplaying: true); + // Run the stored procedure with the reconstructed input + var output = replayContext.output; + _ = replayContext.respServerSession.RunCustomTxnProcAtReplica(id, ref replayContext.customProcInput, ref output, isRecovering: true, customProcKeyHashTracker); + } + } + + /// + /// Unified method to process operations that require synchronization across sublogs + /// + /// SublogIdx + /// Sequence number or entry address for ordering + /// Number of participating replay tasks + /// Unique barrier ID for this operation type + /// The operation to execute + internal void ProcessSynchronizedOperation(int sublogIdx, long sequenceNumber, short participantCount, int barrierId, Func operation) + { + Debug.Assert(serverOptions.MultiLogEnabled); + + // Synchronize execution across sublogs + var leaderBarrier = GetBarrier(barrierId, sequenceNumber, participantCount); + var isLeader = leaderBarrier.TrySignalOrWait(out var signalException, serverOptions.ReplicaSyncTimeout); + Exception removeBarrierException = null; + + // We execute the synchronized operation iff + // 1. Task is the first that joined and + // 2. No exception was triggered or we allow data loss (see cref serverOptions.AllowDataLoss). + // In the event of an exception with the possibility of data loss we follow a best effort approach to guarantee + // the integrity of the replication stream + var execute = isLeader && (signalException == null || serverOptions.AllowDataLoss); + // Here either all participants joined or timeout exception happened + // We can guarantee only one leader since at least one replay task has entered this method. + + try + { + if (execute) + { + // Only one replay task will win and execute the following operation + if (operation != null) + { + var opTask = operation(); + + // No choice but to block here, cannot move off thread + AsyncUtils.BlockingWait(opTask); + } + } + } + finally + { + // The leader will always perform a cleanup + if (isLeader) + { + if (!TryRemoveBarrier(barrierId, sequenceNumber)) + removeBarrierException = new GarnetException($"RemoveBarrier failed when processing {barrierId}"); + + // Release participants if any + leaderBarrier.Release(); + } + } + + // Throw exception if data loss is not allowed and replay failed due to exception (possibly timeout) + if (signalException != null && serverOptions.AllowDataLoss) + throw signalException; + + // Need to always fail here otherwise next operations could not create a barrier if the last operation was not + // able to remove it. + if (removeBarrierException != null) + throw removeBarrierException; + + // Transaction replay consistency invariant: + // Updating the sequence number before the operation executes preserves prefix consistency — + // it signals that replay has reached this log position, matching the standalone operation model. + // Atomicity is currently preserved through coordinated locking (acquire-barrier before writes, + // release-barrier after commit), preventing readers from observing partial transaction state. + // Alternatively, atomicity could be preserved without locking by relying on the read protocol + // to re-read keys as they are updated; in that model, write-set replay must NOT advance the + // sequence number — only the commit marker should update it. + aofProcessor.storeWrapper.appendOnlyFile.readConsistencyManager.UpdateVirtualSublogMaxSequenceNumber(sublogIdx, sequenceNumber); + + // Get barrier helper + LeaderBarrier GetBarrier(int sessionId, long seqNum, short partCount) + { + var barrierID = new BarrierKey() { SessionId = sessionId, txnId = seqNum }; + return leaderBarriers.GetOrAdd(barrierID, _ => new LeaderBarrier(partCount)); + } + + // Remove barrier helper + bool TryRemoveBarrier(int sessionId, long seqNum) + { + var barrierID = new BarrierKey() { SessionId = sessionId, txnId = seqNum }; + return leaderBarriers.TryRemove(barrierID, out _); } } } diff --git a/libs/server/AOF/ReplayCoordinator/TransactionGroup.cs b/libs/server/AOF/ReplayCoordinator/TransactionGroup.cs index 25d9b0f6fff..69a89a719bc 100644 --- a/libs/server/AOF/ReplayCoordinator/TransactionGroup.cs +++ b/libs/server/AOF/ReplayCoordinator/TransactionGroup.cs @@ -6,18 +6,37 @@ namespace Garnet.server { /// - /// Transaction group contains list of operations associated with a given transaction + /// Transaction group contains logAccessMap and list of operations associated with this Txn /// - public class TransactionGroup + /// + /// + /// Sequence number or entry address of the TxnStart entry + public class TransactionGroup(int sublogIdx, byte logAccessMap, long startSequenceNumber = 0) { /// - /// Transaction operation buffer + /// Virtual sublog index associated with this transaction group. /// - public List operations = []; + public readonly int VirtualSublogIdx = sublogIdx; /// - /// Clear the underlying buffer that holds the individual transaction operations + /// Virtual sublog access count associated with this transaction group. /// - public void Clear() => operations.Clear(); + public readonly byte LogAccessCount = logAccessMap; + + /// + /// Sequence number or entry address of the TxnStart entry, used to key the acquire-barrier + /// so it is distinct from the release-barrier keyed on the TxnCommit entry. + /// + public readonly long StartSequenceNumber = startSequenceNumber; + + /// + /// Operations associated with this transaction group. + /// + public List Operations = []; + + /// + /// Clear the underlying buffer that holds the individual transaction operations. + /// + public void Clear() => Operations.Clear(); } } \ No newline at end of file diff --git a/libs/server/AOF/ShardedLog.cs b/libs/server/AOF/ShardedLog.cs new file mode 100644 index 00000000000..61822105b9a --- /dev/null +++ b/libs/server/AOF/ShardedLog.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Linq; +using System.Threading; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + public class ShardedLog + { + /// + /// Number of physical sublogs + /// + public int Length { get; private set; } + readonly TsavoriteLogSettings[] logSettings; + + /// + /// Physical sublog instances + /// + public readonly TsavoriteLog[] sublog; + + /// + /// Distinct locks per sublog instance + /// + public readonly SingleWriterMultiReaderLock[] logLocks; + + /// + /// Provides a sharded log abstraction that manages multiple physical sublogs for scalable and concurrent log + /// operations. + /// + /// ShardedLog enables partitioning of log data across multiple sublogs to improve throughput and + /// concurrency. Each sublog operates independently but can be coordinated through the provided locking and address + /// management methods. Thread safety is provided for sublog access via explicit locking mechanisms. The class is + /// intended for advanced scenarios where high-performance, partitioned logging is required. + /// The number of physical sublogs to create and manage. Must be a positive integer. + /// An array of settings used to configure each physical sublog. The length must match the value of + /// physicalSublogCount. + /// An optional logger instance used to record diagnostic or operational information for each sublog. If null, + /// logging is disabled. + public ShardedLog(int physicalSublogCount, TsavoriteLogSettings[] logSettings, ILogger logger = null) + { + Length = physicalSublogCount; + this.logSettings = logSettings; + this.sublog = new TsavoriteLog[Length]; + for (var i = 0; i < logSettings.Length; i++) + sublog[i] = new TsavoriteLog(logSettings[i], logger); + logLocks = [.. Enumerable.Range(0, physicalSublogCount).Select(_ => new SingleWriterMultiReaderLock())]; + } + + ulong lockMap = 0; + + public void LockSublogs(ulong logAccessBitmap) + { + SpinWait spinWait = default; + while (true) + { + var currentLockMap = lockMap; + + // Attempt CAS only if none of the requested bits are held + if ((currentLockMap & logAccessBitmap) == 0) + { + var newLockMap = currentLockMap | logAccessBitmap; + if (Interlocked.CompareExchange(ref lockMap, newLockMap, currentLockMap) == currentLockMap) + break; + } + + // Bits held by another caller or CAS failed — spin before retrying + spinWait.SpinOnce(); + } + } + + public void UnlockSublogs(ulong logAccessBitmap) + { + Debug.Assert((lockMap & logAccessBitmap) > 0); + logAccessBitmap = ~logAccessBitmap; + while (true) + { + Thread.Yield(); + var currentLockMap = lockMap; + var newLockMap = currentLockMap & logAccessBitmap; + if (Interlocked.CompareExchange(ref lockMap, newLockMap, currentLockMap) == currentLockMap) + break; + } + } + + public AofAddress BeginAddress + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].BeginAddress; + return result; + } + } + + public AofAddress TailAddress + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].TailAddress; + return result; + } + } + + public AofAddress CommittedUntilAddress + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].CommittedUntilAddress; + return result; + } + } + + public AofAddress CommittedBeginAddress + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].CommittedBeginAddress; + return result; + } + } + + public AofAddress FlushedUntilAddress + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].FlushedUntilAddress; + return result; + } + } + + public long HeaderSize => sublog[0].HeaderSize; + + public AofAddress MaxMemorySizeBytes + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].MaxMemorySizeBytes; + return result; + } + } + + public AofAddress MemorySizeBytes + { + get + { + var result = AofAddress.Create(Length, 0); + for (var i = 0; i < sublog.Length; i++) + result[i] = sublog[i].MemorySizeBytes; + return result; + } + } + + public void Recover() + { + foreach (var log in sublog) + log.Recover(); + } + + public void Reset() + { + foreach (var log in sublog) + log.Reset(); + } + + public void Dispose() + { + for (var i = 0; i < sublog.Length; i++) + { + logSettings[i].LogDevice.Dispose(); + sublog[i].Dispose(); + } + } + + public void Initialize(in AofAddress beginAddress, in AofAddress committedUntilAddress, long lastCommitNum = 0) + { + for (var i = 0; i < sublog.Length; i++) + sublog[i].Initialize(beginAddress[i], committedUntilAddress[i], lastCommitNum); + } + } +} \ No newline at end of file diff --git a/libs/server/AOF/SingleLog.cs b/libs/server/AOF/SingleLog.cs new file mode 100644 index 00000000000..bf3306522f4 --- /dev/null +++ b/libs/server/AOF/SingleLog.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Microsoft.Extensions.Logging; +using Tsavorite.core; + + +namespace Garnet.server +{ + /// + /// Provides a wrapper for a single Tsavorite log instance, exposing key log addresses and management operations. + /// + /// This class simplifies access to the core addresses and lifecycle operations of a Tsavorite + /// log. It is intended for scenarios where a single log instance is managed directly + /// + /// The settings used to configure the underlying Tsavorite log. Cannot be null. + /// An optional logger used for diagnostic and operational messages. If null, logging is disabled. + public class SingleLog(TsavoriteLogSettings logSettings, ILogger logger = null) + { + readonly TsavoriteLogSettings logSettings = logSettings; + /// + /// The underlying TsavoriteLog instance for direct log operations. + /// + public readonly TsavoriteLog log = new(logSettings, logger: logger); + + public long HeaderSize => log.HeaderSize; + + public AofAddress BeginAddress => AofAddress.Create(1, value: log.BeginAddress); + + public AofAddress TailAddress => AofAddress.Create(1, value: log.TailAddress); + + public AofAddress CommittedUntilAddress => AofAddress.Create(1, value: log.CommittedUntilAddress); + + public AofAddress CommittedBeginAddress => AofAddress.Create(1, value: log.CommittedBeginAddress); + + public AofAddress FlushedUntilAddress => AofAddress.Create(1, value: log.FlushedUntilAddress); + + public AofAddress MaxMemorySizeBytes => AofAddress.Create(1, value: log.MaxMemorySizeBytes); + + public AofAddress MemorySizeBytes => AofAddress.Create(1, value: log.MemorySizeBytes); + + public void Recover() => log.Recover(); + public void Reset() => log.Reset(); + + public void Dispose() + { + logSettings.LogDevice.Dispose(); + log.Dispose(); + } + } +} \ No newline at end of file diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 435212cc982..5afb0d1366b 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -8,462 +8,336 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - - // Example aliases: - // using BasicGarnetApi = GarnetApi, BasicContext>; - // using LockableGarnetApi = GarnetApi, LockableContext>; + // See TransactionManager.cs for aliases BasicGarnetApi and TransactionalGarnetApi /// /// Garnet API implementation /// - public partial struct GarnetApi : IGarnetApi, IGarnetWatchApi - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - where TVectorContext : ITsavoriteContext + public partial struct GarnetApi : IGarnetApi, IGarnetWatchApi + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext { readonly StorageSession storageSession; - TContext context; + TStringContext stringContext; TObjectContext objectContext; + TUnifiedContext unifiedContext; - internal GarnetApi(StorageSession storageSession, TContext context, TObjectContext objectContext) + internal GarnetApi(StorageSession storageSession, TStringContext stringContext, TObjectContext objectContext, TUnifiedContext unifiedContext) { this.storageSession = storageSession; - this.context = context; + this.stringContext = stringContext; this.objectContext = objectContext; + this.unifiedContext = unifiedContext; } #region WATCH /// - public void WATCH(ArgSlice key, StoreType type) - => storageSession.WATCH(key, type); - - /// - public void WATCH(byte[] key, StoreType type) + public void WATCH(PinnedSpanByte key, StoreType type) => storageSession.WATCH(key, type); #endregion #region GET /// - public GarnetStatus GET(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output) - { - var asSpanByte = key.SpanByte; - - return storageSession.GET(ref asSpanByte, ref input, ref output, ref context); - } + public GarnetStatus GET(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.GET(key, ref input, ref output, ref stringContext); /// - public GarnetStatus GET_WithPending(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, long ctx, out bool pending) - => storageSession.GET_WithPending(ref key, ref input, ref output, ctx, out pending, ref context); + public GarnetStatus GET_WithPending(PinnedSpanByte key, ref StringInput input, ref StringOutput output, long ctx, out bool pending) + => storageSession.GET_WithPending(key.ReadOnlySpan, ref input, ref output, ctx, out pending, ref stringContext); /// - public bool GET_CompletePending((GarnetStatus, SpanByteAndMemory)[] outputArr, bool wait = false) - => storageSession.GET_CompletePending(outputArr, wait, ref context); + public bool GET_CompletePending((GarnetStatus, StringOutput)[] outputArr, bool wait = false) + => storageSession.GET_CompletePending(outputArr, wait, ref stringContext); - public bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait) - => storageSession.GET_CompletePending(out completedOutputs, wait, ref context); + public bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait) + => storageSession.GET_CompletePending(out completedOutputs, wait, ref stringContext); /// - public unsafe GarnetStatus GETForMemoryResult(ArgSlice key, out MemoryResult value) - => storageSession.GET(key, out value, ref context); + public unsafe GarnetStatus GETForMemoryResult(PinnedSpanByte key, out MemoryResult value) + => storageSession.GET(key, out value, ref stringContext); /// - public unsafe GarnetStatus GET(ArgSlice key, out ArgSlice value) - { - return storageSession.GET(key, out value, ref context); - } + public unsafe GarnetStatus GET(PinnedSpanByte key, out PinnedSpanByte value) + => storageSession.GET(key, out value, ref stringContext); /// - public GarnetStatus GET(byte[] key, out GarnetObjectStoreOutput value) + public GarnetStatus GET(PinnedSpanByte key, out ObjectOutput value) => storageSession.GET(key, out value, ref objectContext); /// - public GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) + public GarnetStatus LCS(PinnedSpanByte key1, PinnedSpanByte key2, ref StringOutput output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) => storageSession.LCS(key1, key2, ref output, lenOnly, withIndices, withMatchLen, minMatchLen); #endregion #region GETEX /// - public GarnetStatus GETEX(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.GETEX(ref key, ref input, ref output, ref context); + public GarnetStatus GETEX(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.GETEX(key, ref input, ref output, ref stringContext); #endregion #region GETRANGE /// - public GarnetStatus GETRANGE(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.GETRANGE(ref key, ref input, ref output, ref context); + public GarnetStatus GETRANGE(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.GETRANGE(key, ref input, ref output, ref stringContext); #endregion - #region TTL - + #region SET /// - public GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - => storageSession.TTL(ref key, storeType, ref output, ref context, ref objectContext); + public GarnetStatus SET(PinnedSpanByte key, PinnedSpanByte value) + => storageSession.SET(key, value, ref stringContext); /// - public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - => storageSession.TTL(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true); - - #endregion - - #region EXPIRETIME + public GarnetStatus SET(PinnedSpanByte key, ref StringInput input, PinnedSpanByte value) + => storageSession.SET(key, ref input, value, ref stringContext); /// - public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext); + public GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input) + => storageSession.SET_Conditional(key, ref input, ref stringContext); /// - public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true); - - #endregion - - #region SET - - public GarnetStatus SET(ref SpanByte key, ref SpanByte value) - => storageSession.SET(ref key, ref value, ref context); + public GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.SET_Conditional(key, ref input, ref output, ref stringContext); /// - public GarnetStatus SET(ArgSlice key, ref RawStringInput input, ref SpanByte value) - { - var asSpanByte = key.SpanByte; - - return storageSession.SET(ref asSpanByte, ref input, ref value, ref context); - } + public GarnetStatus SET_ETagConditional(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.SET_Conditional(key, ref input, ref output, ref stringContext); /// - public GarnetStatus DEL_Conditional(ref SpanByte key, ref RawStringInput input) - => storageSession.DEL_Conditional(ref key, ref input, ref context); + public GarnetStatus DEL_ETagConditional(PinnedSpanByte key, ref StringInput input) + => storageSession.DEL_Conditional(key, ref input, ref stringContext); /// - public GarnetStatus SET_Conditional(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output) - { - var asSpanByte = key.SpanByte; - - return storageSession.SET_Conditional(ref asSpanByte, ref input, ref output, ref context); - } + public GarnetStatus SET(PinnedSpanByte key, Memory value) + => storageSession.SET(key, value, ref stringContext); /// - public GarnetStatus SET_Conditional(ArgSlice key, ref RawStringInput input) - { - var asSpanByte = key.SpanByte; - - return storageSession.SET_Conditional(ref asSpanByte, ref input, ref context); - } + public GarnetStatus SET(PinnedSpanByte key, IGarnetObject value) + => storageSession.SET(key, value, ref objectContext); /// - public GarnetStatus SET(ArgSlice key, Memory value) - { - return storageSession.SET(key, value, ref context); - } + public GarnetStatus SET(in TSourceLogRecord srcLogRecord) + where TSourceLogRecord : ISourceLogRecord + => storageSession.SET(in srcLogRecord, ref unifiedContext); /// - public GarnetStatus SET(ArgSlice key, ArgSlice value) - { - var asSpanByte = key.SpanByte; - var valSpanByte = value.SpanByte; - - return storageSession.SET(ref asSpanByte, ref valSpanByte, ref context); - } + public GarnetStatus SET(PinnedSpanByte key, ref UnifiedInput input, in TSourceLogRecord srcLogRecord) + where TSourceLogRecord : ISourceLogRecord + => storageSession.SET(key, ref input, in srcLogRecord, ref unifiedContext); - /// - public GarnetStatus SET(byte[] key, IGarnetObject value) - => storageSession.SET(key, value, ref objectContext); #endregion #region SETEX /// - public unsafe GarnetStatus SETEX(ArgSlice key, ArgSlice value, ArgSlice expiryMs) - => storageSession.SETEX(key, value, expiryMs, ref context); + public unsafe GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, PinnedSpanByte expiryMs) + => storageSession.SETEX(key, value, expiryMs, ref stringContext); /// - public GarnetStatus SETEX(ArgSlice key, ArgSlice value, TimeSpan expiry) - => storageSession.SETEX(key, value, expiry, ref context); + public GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, TimeSpan expiry) + => storageSession.SETEX(key, value, expiry, ref stringContext); #endregion #region SETRANGE /// - public GarnetStatus SETRANGE(ArgSlice key, ref RawStringInput input, ref ArgSlice output) - => storageSession.SETRANGE(key, ref input, ref output, ref context); + public GarnetStatus SETRANGE(PinnedSpanByte key, ref StringInput input, ref PinnedSpanByte output) + => storageSession.SETRANGE(key, ref input, ref output, ref stringContext); #endregion #region MSETNX /// - public GarnetStatus MSET_Conditional(ref RawStringInput input) => - storageSession.MSET_Conditional(ref input, ref context); + public GarnetStatus MSET_Conditional(ref StringInput input) => + storageSession.MSET_Conditional(ref input, ref stringContext); #endregion #region APPEND /// - public GarnetStatus APPEND(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.APPEND(ref key, ref input, ref output, ref context); + public GarnetStatus APPEND(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.APPEND(key, ref input, ref output, ref stringContext); /// - public GarnetStatus APPEND(ArgSlice key, ArgSlice value, ref ArgSlice output) - => storageSession.APPEND(key, value, ref output, ref context); + public GarnetStatus APPEND(PinnedSpanByte key, PinnedSpanByte value, ref PinnedSpanByte output) + => storageSession.APPEND(key, value, ref output, ref stringContext); #endregion #region RENAME /// - public GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, bool withEtag = false, StoreType storeType = StoreType.All) - => storageSession.RENAME(oldKey, newKey, storeType, withEtag); - - /// - public GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, bool withEtag = false, StoreType storeType = StoreType.All) - => storageSession.RENAMENX(oldKey, newKey, storeType, out result, withEtag); - #endregion - - #region EXISTS - /// - public GarnetStatus EXISTS(ArgSlice key, StoreType storeType = StoreType.All) - => storageSession.EXISTS(key, storeType, ref context, ref objectContext); - #endregion - - #region EXPIRE - /// - public unsafe GarnetStatus EXPIRE(ArgSlice key, ref RawStringInput input, out bool timeoutSet, StoreType storeType = StoreType.All) - => storageSession.EXPIRE(key, ref input, out timeoutSet, storeType, ref context, ref objectContext); - - /// - public unsafe GarnetStatus EXPIRE(ArgSlice key, ArgSlice expiryMs, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) - => storageSession.EXPIRE(key, expiryMs, out timeoutSet, storeType, expireOption, ref context, ref objectContext); - - /// - public GarnetStatus EXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) - => storageSession.EXPIRE(key, expiry, out timeoutSet, storeType, expireOption, ref context, ref objectContext); - #endregion - - #region EXPIREAT - - /// - public GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) - => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext); - - /// - public GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) - => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext, milliseconds: true); - - #endregion + public GarnetStatus RENAME(PinnedSpanByte oldKey, PinnedSpanByte newKey) + => storageSession.RENAME(oldKey, newKey); - #region PERSIST /// - public unsafe GarnetStatus PERSIST(ArgSlice key, StoreType storeType = StoreType.All) - => storageSession.PERSIST(key, storeType, ref context, ref objectContext); + public GarnetStatus RENAMENX(PinnedSpanByte oldKey, PinnedSpanByte newKey, out int result) + => storageSession.RENAMENX(oldKey, newKey, out result); #endregion #region Increment (INCR, INCRBY, DECR, DECRBY) /// - public GarnetStatus Increment(ArgSlice key, ref RawStringInput input, ref ArgSlice output) - => storageSession.Increment(key, ref input, ref output, ref context); + public GarnetStatus Increment(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.Increment(key, ref input, ref output, ref stringContext); /// - public GarnetStatus Increment(ArgSlice key, out long output, long incrementCount = 1) - => storageSession.Increment(key, out output, incrementCount, ref context); + public GarnetStatus Increment(PinnedSpanByte key, out long output, long incrementCount = 1) + => storageSession.Increment(key, out output, incrementCount, ref stringContext); /// - public GarnetStatus Decrement(ArgSlice key, out long output, long decrementCount = 1) + public GarnetStatus Decrement(PinnedSpanByte key, out long output, long decrementCount = 1) => Increment(key, out output, -decrementCount); /// - public GarnetStatus IncrementByFloat(ArgSlice key, ref ArgSlice output, double val) + public GarnetStatus IncrementByFloat(PinnedSpanByte key, ref StringOutput output, double val) { SessionParseState parseState = default; - var input = new RawStringInput(RespCommand.INCRBYFLOAT, ref parseState, BitConverter.DoubleToInt64Bits(val)); + var input = new StringInput(RespCommand.INCRBYFLOAT, ref parseState, BitConverter.DoubleToInt64Bits(val)); _ = Increment(key, ref input, ref output); - - if (output.Length != NumUtils.MaximumFormatDoubleLength + 1) - return GarnetStatus.OK; - - var errorFlag = (OperationError)output.Span[0]; - - switch (errorFlag) - { - case OperationError.INVALID_TYPE: - case OperationError.NAN_OR_INFINITY: - return GarnetStatus.WRONGTYPE; - default: - throw new GarnetException($"Invalid OperationError {errorFlag}"); - } + return GarnetStatus.OK; } /// - public GarnetStatus IncrementByFloat(ArgSlice key, out double output, double val) + public GarnetStatus IncrementByFloat(PinnedSpanByte key, out double output, double val) { Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatDoubleLength + 1]; - var _output = ArgSlice.FromPinnedSpan(outputBuffer); - var status = IncrementByFloat(key, ref _output, val); + var stringOutput = StringOutput.FromPinnedSpan(outputBuffer); + + _ = IncrementByFloat(key, ref stringOutput, val); - switch (status) + if (!stringOutput.HasError) + { + _ = NumUtils.TryReadDouble(stringOutput.SpanByteAndMemory.Span, out output); + } + else { - case GarnetStatus.OK: - _ = NumUtils.TryReadDouble(_output.ReadOnlySpan, out output); - break; - case GarnetStatus.WRONGTYPE: - default: - var errorFlag = (OperationError)_output.Span[0]; - output = errorFlag == OperationError.NAN_OR_INFINITY ? double.NaN : 0; - break; + output = (stringOutput.OutputFlags & StringOutputFlags.NaNOrInfinityError) != 0 ? double.NaN : 0; } - return status; + return GarnetStatus.OK; } #endregion - #region DELETE - /// - public GarnetStatus DELETE(ArgSlice key, StoreType storeType = StoreType.All) - => storageSession.DELETE(key, storeType, ref context, ref objectContext); - - /// - public GarnetStatus DELETE(ref SpanByte key, StoreType storeType = StoreType.All) - => storageSession.DELETE(ref key, storeType, ref context, ref objectContext); - - /// - public GarnetStatus DELETE(byte[] key, StoreType storeType = StoreType.All) - => storageSession.DELETE(key, storeType, ref context, ref objectContext); - #endregion - #region GETDEL /// - public GarnetStatus GETDEL(ref SpanByte key, ref SpanByteAndMemory output) - => storageSession.GETDEL(ref key, ref output, ref context); - - /// - public GarnetStatus GETDEL(ArgSlice key, ref SpanByteAndMemory output) - => storageSession.GETDEL(key, ref output, ref context); + public GarnetStatus GETDEL(PinnedSpanByte key, ref StringOutput output) + => storageSession.GETDEL(key, ref output, ref stringContext); #endregion - #region TYPE - + #region Advanced ops /// - public GarnetStatus GetKeyType(ArgSlice key, out string typeName) - => storageSession.GetKeyType(key, out typeName, ref context, ref objectContext); - - #endregion - - #region MEMORY + public GarnetStatus RMW_MainStore(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.RMW_MainStore(key.ReadOnlySpan, ref input, ref output, ref stringContext); /// - public GarnetStatus MemoryUsageForKey(ArgSlice key, out long memoryUsage, int samples = 0) - => storageSession.MemoryUsageForKey(key, out memoryUsage, ref context, ref objectContext, samples); - - #endregion + public GarnetStatus Read_MainStore(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref stringContext); - #region Advanced ops /// - public GarnetStatus RMW_MainStore(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.RMW_MainStore(ref key, ref input, ref output, ref context); + public GarnetStatus RMW_ObjectStore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.RMW_ObjectStore(key.ReadOnlySpan, ref input, ref output, ref objectContext); /// - public GarnetStatus Read_MainStore(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.Read_MainStore(ref key, ref input, ref output, ref context); + public GarnetStatus Read_ObjectStore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.Read_ObjectStore(key.ReadOnlySpan, ref input, ref output, ref objectContext); /// - public GarnetStatus RMW_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) - => storageSession.RMW_ObjectStore(ref key, ref input, ref output, ref objectContext); + public GarnetStatus RMW_UnifiedStore(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.RMW_UnifiedStore(key.ReadOnlySpan, ref input, ref output, ref unifiedContext); /// - public GarnetStatus Read_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) - => storageSession.Read_ObjectStore(ref key, ref input, ref output, ref objectContext); + public GarnetStatus Read_UnifiedStore(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key.ReadOnlySpan, ref input, ref output, ref unifiedContext); + /// public void ReadWithPrefetch(ref TBatch batch, long userContext = default) - where TBatch : IReadArgBatch + where TBatch : IReadArgBatch #if NET9_0_OR_GREATER , allows ref struct #endif - => storageSession.ReadWithPrefetch(ref batch, ref context, userContext); + => storageSession.ReadWithPrefetch(ref batch, ref stringContext, userContext); #endregion #region Bitmap Methods /// - public GarnetStatus StringSetBit(ArgSlice key, ArgSlice offset, bool bit, out bool previous) - => storageSession.StringSetBit(key, offset, bit, out previous, ref context); + public GarnetStatus StringSetBit(PinnedSpanByte key, PinnedSpanByte offset, bool bit, out bool previous) + => storageSession.StringSetBit(key, offset, bit, out previous, ref stringContext); /// - public GarnetStatus StringSetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.StringSetBit(ref key, ref input, ref output, ref context); + public GarnetStatus StringSetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.StringSetBit(key, ref input, ref output, ref stringContext); /// - public GarnetStatus StringGetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.StringGetBit(ref key, ref input, ref output, ref context); + public GarnetStatus StringGetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.StringGetBit(key, ref input, ref output, ref stringContext); /// - public GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, out bool bValue) - => storageSession.StringGetBit(key, offset, out bValue, ref context); + public GarnetStatus StringGetBit(PinnedSpanByte key, PinnedSpanByte offset, out bool bValue) + => storageSession.StringGetBit(key, offset, out bValue, ref stringContext); /// - public GarnetStatus StringBitCount(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.StringBitCount(ref key, ref input, ref output, ref context); + public GarnetStatus StringBitCount(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.StringBitCount(key, ref input, ref output, ref stringContext); /// - public GarnetStatus StringBitCount(ArgSlice key, long start, long end, out long result, bool useBitInterval = false) - => storageSession.StringBitCount(key, start, end, useBitInterval, out result, ref context); + public GarnetStatus StringBitCount(PinnedSpanByte key, long start, long end, out long result, bool useBitInterval = false) + => storageSession.StringBitCount(key, start, end, useBitInterval, out result, ref stringContext); /// - public GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOperation bitOp, out long result) + public GarnetStatus StringBitOperation(ref StringInput input, BitmapOperation bitOp, out long result) => storageSession.StringBitOperation(ref input, bitOp, out result); /// - public GarnetStatus StringBitOperation(BitmapOperation bitop, ArgSlice destinationKey, ArgSlice[] keys, out long result) + public GarnetStatus StringBitOperation(BitmapOperation bitop, PinnedSpanByte destinationKey, PinnedSpanByte[] keys, out long result) => storageSession.StringBitOperation(bitop, destinationKey, keys, out result); /// - public GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.StringBitPosition(ref key, ref input, ref output, ref context); + public GarnetStatus StringBitPosition(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.StringBitPosition(key, ref input, ref output, ref stringContext); /// - public GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) - => storageSession.StringBitField(ref key, ref input, secondaryCommand, ref output, ref context); + public GarnetStatus StringBitField(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output) + => storageSession.StringBitField(key, ref input, secondaryCommand, ref output, ref stringContext); /// - public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) - => storageSession.StringBitFieldReadOnly(ref key, ref input, secondaryCommand, ref output, ref context); + public GarnetStatus StringBitFieldReadOnly(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output) + => storageSession.StringBitFieldReadOnly(key, ref input, secondaryCommand, ref output, ref stringContext); /// - public GarnetStatus StringBitField(ArgSlice key, List commandArguments, out List result) - => storageSession.StringBitField(key, commandArguments, out result, ref context); + public GarnetStatus StringBitField(PinnedSpanByte key, List commandArguments, out List result) + => storageSession.StringBitField(key, commandArguments, out result, ref stringContext); #endregion #region HyperLogLog Methods /// - public GarnetStatus HyperLogLogAdd(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) - => storageSession.HyperLogLogAdd(ref key, ref input, ref output, ref context); + public GarnetStatus HyperLogLogAdd(PinnedSpanByte key, ref StringInput input, ref StringOutput output) + => storageSession.HyperLogLogAdd(key, ref input, ref output, ref stringContext); /// - public GarnetStatus HyperLogLogAdd(ArgSlice key, string[] elements, out bool updated) - => storageSession.HyperLogLogAdd(key, elements, out updated, ref context); + public GarnetStatus HyperLogLogAdd(PinnedSpanByte key, string[] elements, out bool updated) + => storageSession.HyperLogLogAdd(key, elements, out updated, ref stringContext); /// - public GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, out bool error) - => storageSession.HyperLogLogLength(ref input, out count, out error, ref context); + public GarnetStatus HyperLogLogLength(ref StringInput input, out long count, out bool error) + => storageSession.HyperLogLogLength(ref input, out count, out error, ref stringContext); /// - public GarnetStatus HyperLogLogLength(Span keys, out long count) - => storageSession.HyperLogLogLength(keys, out count, ref context); + public GarnetStatus HyperLogLogLength(Span keys, out long count) + => storageSession.HyperLogLogLength(keys, out count, ref stringContext); /// - public GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool error) + public GarnetStatus HyperLogLogMerge(ref StringInput input, out bool error) => storageSession.HyperLogLogMerge(ref input, out error); #endregion #region Server Methods /// - public List GetDbKeys(ArgSlice pattern) + public List GetDbKeys(PinnedSpanByte pattern) => storageSession.DBKeys(pattern); /// @@ -471,78 +345,118 @@ public int GetDbSize() => storageSession.DbSize(); /// - public bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out long storeCursor, out List Keys, long count = 10, ReadOnlySpan type = default) + public readonly bool DbScan(PinnedSpanByte patternB, bool allKeys, long cursor, out long storeCursor, out List Keys, long count = 10, ReadOnlySpan type = default) => storageSession.DbScan(patternB, allKeys, cursor, out storeCursor, out Keys, count, type); /// - public bool IterateMainStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => storageSession.IterateMainStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); - - /// - public ITsavoriteScanIterator IterateMainStore() - => storageSession.IterateMainStore(); + public readonly bool IterateStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + => storageSession.IterateStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); /// - public bool IterateObjectStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => storageSession.IterateObjectStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); - - /// - public ITsavoriteScanIterator IterateObjectStore() - => storageSession.IterateObjectStore(); + public readonly void DeleteSlotKeys(HashSet slots) + => storageSession.DeleteSlotKeys(slots); #endregion #region Common Methods /// - public GarnetStatus ObjectScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) - => storageSession.ObjectScan(key, ref input, ref output, ref objectContext); - - /// - public int GetScratchBufferOffset() - => storageSession.scratchBufferBuilder.ScratchBufferOffset; + public GarnetStatus ObjectScan(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ObjectScan(key.ReadOnlySpan, ref input, ref output, ref objectContext); /// - public bool ResetScratchBuffer(int offset) - => storageSession.scratchBufferBuilder.ResetScratchBuffer(offset); + public void ResetScratchBuffer() + => storageSession.scratchBufferAllocator.Reset(); #endregion #region VectorSet commands /// - public unsafe GarnetStatus VectorSetAdd(ArgSlice key, int reduceDims, VectorValueType valueType, ArgSlice values, ArgSlice element, VectorQuantType quantizer, int buildExplorationFactor, ArgSlice attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg) - => storageSession.VectorSetAdd(SpanByte.FromPinnedPointer(key.ptr, key.length), reduceDims, valueType, values, element, quantizer, buildExplorationFactor, attributes, numLinks, distanceMetric, out result, out errorMsg); + public unsafe GarnetStatus VectorSetAdd(PinnedSpanByte key, int reduceDims, VectorValueType valueType, PinnedSpanByte values, PinnedSpanByte element, VectorQuantType quantizer, int buildExplorationFactor, PinnedSpanByte attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg) + => storageSession.VectorSetAdd(key, reduceDims, valueType, values, element, quantizer, buildExplorationFactor, attributes, numLinks, distanceMetric, out result, out errorMsg); /// - public unsafe GarnetStatus VectorSetRemove(ArgSlice key, ArgSlice element) - => storageSession.VectorSetRemove(SpanByte.FromPinnedPointer(key.ptr, key.length), SpanByte.FromPinnedPointer(element.ptr, element.length)); + public unsafe GarnetStatus VectorSetRemove(PinnedSpanByte key, PinnedSpanByte element) + => storageSession.VectorSetRemove(key, element); /// - public unsafe GarnetStatus VectorSetValueSimilarity(ArgSlice key, VectorValueType valueType, ArgSlice values, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) - => storageSession.VectorSetValueSimilarity(SpanByte.FromPinnedPointer(key.ptr, key.length), valueType, values, count, delta, searchExplorationFactor, filter.ReadOnlySpan, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); + public unsafe GarnetStatus VectorSetValueSimilarity(PinnedSpanByte key, VectorValueType valueType, PinnedSpanByte values, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + => storageSession.VectorSetValueSimilarity(key, valueType, values, count, delta, searchExplorationFactor, filter.ReadOnlySpan, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); /// - public unsafe GarnetStatus VectorSetElementSimilarity(ArgSlice key, ArgSlice element, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) - => storageSession.VectorSetElementSimilarity(SpanByte.FromPinnedPointer(key.ptr, key.length), element.ReadOnlySpan, count, delta, searchExplorationFactor, filter.ReadOnlySpan, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); + public unsafe GarnetStatus VectorSetElementSimilarity(PinnedSpanByte key, PinnedSpanByte element, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + => storageSession.VectorSetElementSimilarity(key, element.ReadOnlySpan, count, delta, searchExplorationFactor, filter.ReadOnlySpan, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); /// - public unsafe GarnetStatus VectorSetEmbedding(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputDistances) - => storageSession.VectorSetEmbedding(SpanByte.FromPinnedPointer(key.ptr, key.length), element.ReadOnlySpan, ref outputDistances); + public unsafe GarnetStatus VectorSetEmbedding(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputDistances) + => storageSession.VectorSetEmbedding(key, element.ReadOnlySpan, ref outputDistances); /// - public unsafe GarnetStatus VectorSetDimensions(ArgSlice key, out int dimensions) - => storageSession.VectorSetDimensions(SpanByte.FromPinnedPointer(key.ptr, key.length), out dimensions); + public unsafe GarnetStatus VectorSetDimensions(PinnedSpanByte key, out int dimensions) + => storageSession.VectorSetDimensions(key, out dimensions); /// - public unsafe GarnetStatus VectorSetInfo(ArgSlice key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size) - => storageSession.VectorSetInfo(SpanByte.FromPinnedPointer(key.ptr, key.length), out quantType, out distanceMetricType, out vectorDimensions, out reducedDimensions, out buildExplorationFactor, out numberOfLinks, out size); + public unsafe GarnetStatus VectorSetInfo(PinnedSpanByte key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size) + => storageSession.VectorSetInfo(key, out quantType, out distanceMetricType, out vectorDimensions, out reducedDimensions, out buildExplorationFactor, out numberOfLinks, out size); /// - public unsafe GarnetStatus VectorSetGetAttribute(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputAttributes) - => storageSession.VectorSetGetAttribute(SpanByte.FromPinnedPointer(key.ptr, key.length), element, ref outputAttributes); + public unsafe GarnetStatus VectorSetGetAttribute(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputAttributes) + => storageSession.VectorSetGetAttribute(key, element, ref outputAttributes); + + #endregion + + #region RangeIndex + /// + public GarnetStatus RangeIndexCreate(PinnedSpanByte key, byte storageBackend, + ulong cacheSize, uint minRecordSize, uint maxRecordSize, uint maxKeyLen, uint leafPageSize, + out RangeIndexResult result, out ReadOnlySpan errorMsg) + => storageSession.RangeIndexCreate(key, storageBackend, cacheSize, minRecordSize, maxRecordSize, maxKeyLen, leafPageSize, out result, out errorMsg); + + /// + public GarnetStatus RangeIndexSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg) + => storageSession.RangeIndexSet(key, field, value, out result, out errorMsg); + /// + public GarnetStatus RangeIndexGet(PinnedSpanByte key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result) + => storageSession.RangeIndexGet(key, field, ref output, out result); + + /// + public GarnetStatus RangeIndexDel(PinnedSpanByte key, PinnedSpanByte field, + out RangeIndexResult result) + => storageSession.RangeIndexDel(key, field, out result); + + /// + public GarnetStatus RangeIndexScan(PinnedSpanByte key, PinnedSpanByte startKey, int count, + BfTreeInterop.ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result) + => storageSession.RangeIndexScan(key, startKey, count, returnField, ref output, out recordCount, out result); + + /// + public GarnetStatus RangeIndexRange(PinnedSpanByte key, PinnedSpanByte startKey, PinnedSpanByte endKey, + BfTreeInterop.ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result) + => storageSession.RangeIndexRange(key, startKey, endKey, returnField, ref output, out recordCount, out result); + + /// + public GarnetStatus RangeIndexExists(PinnedSpanByte key, out bool exists) + => storageSession.RangeIndexExists(key, out exists); + + /// + public GarnetStatus RangeIndexConfig(PinnedSpanByte key, + out byte storageBackend, out ulong cacheSize, out uint minRecordSize, + out uint maxRecordSize, out uint maxKeyLen, out uint leafPageSize, + out RangeIndexResult result) + => storageSession.RangeIndexConfig(key, out storageBackend, out cacheSize, out minRecordSize, + out maxRecordSize, out maxKeyLen, out leafPageSize, out result); + + /// + public GarnetStatus RangeIndexMetrics(PinnedSpanByte key, + out nint treeHandle, out bool isLive, out bool isFlushed, out bool isRecovered, + out RangeIndexResult result) + => storageSession.RangeIndexMetrics(key, out treeHandle, out isLive, out isFlushed, out isRecovered, out result); #endregion } } \ No newline at end of file diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index 9ba483e08d7..f03f94cb19d 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -3,199 +3,194 @@ using System; using System.Collections.Generic; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Garnet API implementation /// - public partial struct GarnetApi : IGarnetApi, IGarnetWatchApi - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - where TVectorContext : ITsavoriteContext + public partial struct GarnetApi : IGarnetApi, IGarnetWatchApi + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext { #region SortedSet Methods /// - public GarnetStatus SortedSetAdd(ArgSlice key, ArgSlice score, ArgSlice member, out int zaddCount) + public GarnetStatus SortedSetAdd(PinnedSpanByte key, PinnedSpanByte score, PinnedSpanByte member, out int zaddCount) => storageSession.SortedSetAdd(key, score, member, out zaddCount, ref objectContext); /// - public GarnetStatus SortedSetAdd(ArgSlice key, (ArgSlice score, ArgSlice member)[] inputs, out int zaddCount) + public GarnetStatus SortedSetAdd(PinnedSpanByte key, (PinnedSpanByte score, PinnedSpanByte member)[] inputs, out int zaddCount) => storageSession.SortedSetAdd(key, inputs, out zaddCount, ref objectContext); /// - public GarnetStatus SortedSetAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetAdd(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRangeStore(ArgSlice dstKey, ArgSlice srcKey, ref ObjectInput input, out int result) + public GarnetStatus SortedSetRangeStore(PinnedSpanByte dstKey, PinnedSpanByte srcKey, ref ObjectInput input, out int result) => storageSession.SortedSetRangeStore(dstKey, srcKey, ref input, out result, ref objectContext); /// - public GarnetStatus SortedSetRemove(ArgSlice key, ArgSlice member, out int zremCount) - => storageSession.SortedSetRemove(key.ToArray(), member, out zremCount, ref objectContext); + public GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte member, out int zremCount) + => storageSession.SortedSetRemove(key, member, out zremCount, ref objectContext); /// - public GarnetStatus SortedSetRemove(ArgSlice key, ArgSlice[] members, out int zaddCount) - => storageSession.SortedSetRemove(key.ToArray(), members, out zaddCount, ref objectContext); + public GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int zaddCount) + => storageSession.SortedSetRemove(key, members, out zaddCount, ref objectContext); /// - public GarnetStatus SortedSetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SortedSetRemove(key, ref input, out output, ref objectContext); + public GarnetStatus SortedSetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SortedSetRemove(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetLength(ArgSlice key, out int len) + public GarnetStatus SortedSetLength(PinnedSpanByte key, out int len) => storageSession.SortedSetLength(key, out len, ref objectContext); /// - public GarnetStatus SortedSetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SortedSetLength(key, ref input, out output, ref objectContext); + public GarnetStatus SortedSetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SortedSetLength(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetRange(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetScore(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetScore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetScore(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetScores(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetScores(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetScores(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetPop(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out ArgSlice poppedKey, out (ArgSlice member, ArgSlice score)[] pairs) + public GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out PinnedSpanByte poppedKey, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs) => storageSession.SortedSetMPop(keys, count, lowScoresFirst, out poppedKey, out pairs); /// - public GarnetStatus SortedSetPop(ArgSlice key, out (ArgSlice member, ArgSlice score)[] pairs, int count = 1, bool lowScoresFirst = true) + public GarnetStatus SortedSetPop(PinnedSpanByte key, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs, int count = 1, bool lowScoresFirst = true) => storageSession.SortedSetPop(key, count, lowScoresFirst, out pairs, ref objectContext); /// - public GarnetStatus SortedSetCount(ArgSlice key, ArgSlice minScore, ArgSlice maxScore, out int numElements) + public GarnetStatus SortedSetCount(PinnedSpanByte key, PinnedSpanByte minScore, PinnedSpanByte maxScore, out int numElements) => storageSession.SortedSetCount(key, minScore, maxScore, out numElements, ref objectContext); /// - public GarnetStatus SortedSetCount(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetCount(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetCount(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetLengthByValue(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SortedSetLengthByValue(key, ref input, out output, ref objectContext); + public GarnetStatus SortedSetLengthByValue(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SortedSetLengthByValue(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRemoveRangeByLex(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SortedSetRemoveRangeByLex(key, ref input, out output, ref objectContext); + public GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SortedSetRemoveRangeByLex(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRemoveRangeByLex(ArgSlice key, string min, string max, out int countRemoved) + public GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, string min, string max, out int countRemoved) => storageSession.SortedSetRemoveRangeByLex(key, min, max, out countRemoved, ref objectContext); /// - public GarnetStatus SortedSetRemoveRangeByScore(ArgSlice key, string min, string max, out int countRemoved) + public GarnetStatus SortedSetRemoveRangeByScore(PinnedSpanByte key, string min, string max, out int countRemoved) => storageSession.SortedSetRemoveRangeByScore(key, min, max, out countRemoved, ref objectContext); /// - public GarnetStatus SortedSetRemoveRangeByRank(ArgSlice key, int start, int stop, out int countRemoved) + public GarnetStatus SortedSetRemoveRangeByRank(PinnedSpanByte key, int start, int stop, out int countRemoved) => storageSession.SortedSetRemoveRangeByRank(key, start, stop, out countRemoved, ref objectContext); /// - public GarnetStatus SortedSetIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetIncrement(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetIncrement(ArgSlice key, double increment, ArgSlice member, out double newScore) + public GarnetStatus SortedSetIncrement(PinnedSpanByte key, double increment, PinnedSpanByte member, out double newScore) => storageSession.SortedSetIncrement(key, increment, member, out newScore, ref objectContext); /// - public GarnetStatus SortedSetRemoveRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRemoveRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetRemoveRange(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRank(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRank(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetRank(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRank(ArgSlice key, ArgSlice member, bool reverse, out long? rank) + public GarnetStatus SortedSetRank(PinnedSpanByte key, PinnedSpanByte member, bool reverse, out long? rank) => storageSession.SortedSetRank(key, member, reverse, out rank, ref objectContext); /// - public GarnetStatus SortedSetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetRandomMember(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetRange(ArgSlice key, ArgSlice min, ArgSlice max, SortedSetOrderOperation sortedSetOrderOperation, out ArgSlice[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) + public GarnetStatus SortedSetRange(PinnedSpanByte key, PinnedSpanByte min, PinnedSpanByte max, SortedSetOrderOperation sortedSetOrderOperation, out PinnedSpanByte[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) => storageSession.SortedSetRange(key, min, max, sortedSetOrderOperation, ref objectContext, out elements, out error, withScores, reverse, limit); /// - public GarnetStatus SortedSetDifference(ArgSlice[] keys, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetDifference(PinnedSpanByte[] keys, out SortedSet<(double, byte[])> pairs) => storageSession.SortedSetDifference(keys, out pairs); /// - public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) => storageSession.SortedSetUnion(keys, weights, aggregateType, out pairs); /// - public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count) + public GarnetStatus SortedSetDifferenceStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, out int count) => storageSession.SortedSetDifferenceStore(destinationKey, keys, out count); - public GarnetStatus SortedSetUnionStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) + public GarnetStatus SortedSetUnionStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) => storageSession.SortedSetUnionStore(destinationKey, keys, weights, aggregateType, out count); /// - public GarnetStatus SortedSetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus SortedSetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) => storageSession.ObjectScan(GarnetObjectType.SortedSet, key, cursor, match, count, out items, ref objectContext); /// - public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) => storageSession.SortedSetIntersect(keys, weights, aggregateType, out pairs); /// - public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) => storageSession.SortedSetIntersectLength(keys, limit, out count); /// - public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) + public GarnetStatus SortedSetIntersectStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) => storageSession.SortedSetIntersectStore(destinationKey, keys, weights, aggregateType, out count); /// - public GarnetStatus SortedSetExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetExpire(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results) + public GarnetStatus SortedSetExpire(PinnedSpanByte key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results) => storageSession.SortedSetExpire(key, members, expireAt, expireOption, out results, ref objectContext); /// - public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetPersist(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results) + public GarnetStatus SortedSetPersist(PinnedSpanByte key, ReadOnlySpan members, out int[] results) => storageSession.SortedSetPersist(key, members, out results, ref objectContext); /// - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SortedSetTimeToLive(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn) + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ReadOnlySpan members, out TimeSpan[] expireIn) => storageSession.SortedSetTimeToLive(key, members, out expireIn, ref objectContext); /// - public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input) + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input) => storageSession.SortedSetCollect(keys, ref input, ref objectContext); /// @@ -203,7 +198,7 @@ public GarnetStatus SortedSetCollect() => storageSession.SortedSetCollect(ref objectContext); /// - public GarnetStatus SortedSetCollect(ReadOnlySpan keys) + public GarnetStatus SortedSetCollect(ReadOnlySpan keys) => storageSession.SortedSetCollect(keys, ref objectContext); #endregion @@ -211,20 +206,20 @@ public GarnetStatus SortedSetCollect(ReadOnlySpan keys) #region Geospatial commands /// - public GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus GeoAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.GeoAdd(key, ref input, ref output, ref objectContext); /// - public GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus GeoCommands(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.GeoCommands(key, ref input, ref output, ref objectContext); /// - public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, + public GarnetStatus GeoSearchReadOnly(PinnedSpanByte key, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output) => storageSession.GeoSearchReadOnly(key, ref opts, ref input, ref output, ref objectContext); /// - public GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearchOptions opts, + public GarnetStatus GeoSearchStore(PinnedSpanByte key, PinnedSpanByte destinationKey, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output) => storageSession.GeoSearchStore(key, destinationKey, ref opts, ref input, ref output, ref objectContext); #endregion @@ -234,105 +229,105 @@ public GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref Ge #region PUSHPOP /// - public GarnetStatus ListRightPush(ArgSlice key, ArgSlice element, out int itemsCount, bool whenExists = false) + public GarnetStatus ListRightPush(PinnedSpanByte key, PinnedSpanByte element, out int itemsCount, bool whenExists = false) => storageSession.ListPush(key, element, whenExists ? ListOperation.RPUSHX : ListOperation.RPUSH, out itemsCount, ref objectContext); /// - public GarnetStatus ListRightPush(ArgSlice key, ArgSlice[] elements, out int itemsCount, bool whenExists = false) + public GarnetStatus ListRightPush(PinnedSpanByte key, PinnedSpanByte[] elements, out int itemsCount, bool whenExists = false) => storageSession.ListPush(key, elements, whenExists ? ListOperation.RPUSHX : ListOperation.RPUSH, out itemsCount, ref objectContext); /// - public GarnetStatus ListRightPush(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.ListPush(key, ref input, out output, ref objectContext); + public GarnetStatus ListRightPush(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ListPush(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListLeftPush(ArgSlice key, ArgSlice[] elements, out int itemsCount, bool onlyWhenExists = false) + public GarnetStatus ListLeftPush(PinnedSpanByte key, PinnedSpanByte[] elements, out int itemsCount, bool onlyWhenExists = false) => storageSession.ListPush(key, elements, onlyWhenExists ? ListOperation.LPUSHX : ListOperation.LPUSH, out itemsCount, ref objectContext); /// - public GarnetStatus ListLeftPush(ArgSlice key, ArgSlice element, out int count, bool onlyWhenExists = false) + public GarnetStatus ListLeftPush(PinnedSpanByte key, PinnedSpanByte element, out int count, bool onlyWhenExists = false) => storageSession.ListPush(key, element, onlyWhenExists ? ListOperation.LPUSHX : ListOperation.LPUSH, out count, ref objectContext); /// - public GarnetStatus ListLeftPush(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.ListPush(key, ref input, out output, ref objectContext); + public GarnetStatus ListLeftPush(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ListPush(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListPosition(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListPosition(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListPosition(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListLeftPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListLeftPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListPop(key, ref input, ref output, ref objectContext); /// - public unsafe GarnetStatus ListLeftPop(ArgSlice key, out ArgSlice element) + public unsafe GarnetStatus ListLeftPop(PinnedSpanByte key, out PinnedSpanByte element) => storageSession.ListPop(key, ListOperation.LPOP, ref objectContext, out element); /// - public GarnetStatus ListLeftPop(ArgSlice key, int count, out ArgSlice[] poppedElements) + public GarnetStatus ListLeftPop(PinnedSpanByte key, int count, out PinnedSpanByte[] poppedElements) => storageSession.ListPop(key, count, ListOperation.LPOP, ref objectContext, out poppedElements); /// - public GarnetStatus ListLeftPop(ArgSlice[] keys, int count, out ArgSlice poppedKey, out ArgSlice[] poppedElements) + public GarnetStatus ListLeftPop(PinnedSpanByte[] keys, int count, out PinnedSpanByte poppedKey, out PinnedSpanByte[] poppedElements) => storageSession.ListPopMultiple(keys, OperationDirection.Left, count, ref objectContext, out poppedKey, out poppedElements); /// - public GarnetStatus ListRightPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListRightPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListPop(key, ref input, ref output, ref objectContext); /// - public unsafe GarnetStatus ListRightPop(ArgSlice key, out ArgSlice element) + public unsafe GarnetStatus ListRightPop(PinnedSpanByte key, out PinnedSpanByte element) => storageSession.ListPop(key, ListOperation.RPOP, ref objectContext, out element); /// - public GarnetStatus ListRightPop(ArgSlice key, int count, out ArgSlice[] poppedElements) + public GarnetStatus ListRightPop(PinnedSpanByte key, int count, out PinnedSpanByte[] poppedElements) => storageSession.ListPop(key, count, ListOperation.RPOP, ref objectContext, out poppedElements); /// - public GarnetStatus ListRightPop(ArgSlice[] keys, int count, out ArgSlice poppedKey, out ArgSlice[] poppedElements) + public GarnetStatus ListRightPop(PinnedSpanByte[] keys, int count, out PinnedSpanByte poppedKey, out PinnedSpanByte[] poppedElements) => storageSession.ListPopMultiple(keys, OperationDirection.Right, count, ref objectContext, out poppedKey, out poppedElements); #endregion /// - public GarnetStatus ListLength(ArgSlice key, out int count) + public GarnetStatus ListLength(PinnedSpanByte key, out int count) => storageSession.ListLength(key, ref objectContext, out count); /// - public GarnetStatus ListLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.ListLength(key, ref input, out output, ref objectContext); + public GarnetStatus ListLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ListLength(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListMove(ArgSlice source, ArgSlice destination, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element) + public GarnetStatus ListMove(PinnedSpanByte source, PinnedSpanByte destination, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element) => storageSession.ListMove(source, destination, sourceDirection, destinationDirection, out element); /// - public bool ListTrim(ArgSlice key, int start, int stop) + public bool ListTrim(PinnedSpanByte key, int start, int stop) => storageSession.ListTrim(key, start, stop, ref objectContext); /// - public GarnetStatus ListTrim(byte[] key, ref ObjectInput input) + public GarnetStatus ListTrim(PinnedSpanByte key, ref ObjectInput input) => storageSession.ListTrim(key, ref input, ref objectContext); /// - public GarnetStatus ListRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListRange(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListInsert(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.ListInsert(key, ref input, out output, ref objectContext); + public GarnetStatus ListInsert(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ListInsert(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListIndex(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListIndex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListIndex(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.ListRemove(key, ref input, out output, ref objectContext); + public GarnetStatus ListRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.ListRemove(key, ref input, ref output, ref objectContext); /// - public GarnetStatus ListSet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.ListSet(key, ref input, ref output, ref objectContext); #endregion @@ -340,102 +335,102 @@ public GarnetStatus ListSet(byte[] key, ref ObjectInput input, ref GarnetObjectS #region Set Methods /// - public GarnetStatus SetAdd(ArgSlice key, ArgSlice member, out int saddCount) + public GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte member, out int saddCount) => storageSession.SetAdd(key, member, out saddCount, ref objectContext); /// - public GarnetStatus SetAdd(ArgSlice key, ArgSlice[] members, out int saddCount) + public GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte[] members, out int saddCount) => storageSession.SetAdd(key, members, out saddCount, ref objectContext); /// - public GarnetStatus SetAdd(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SetAdd(key, ref input, out output, ref objectContext); + public GarnetStatus SetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SetAdd(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetRemove(ArgSlice key, ArgSlice member, out int sremCount) + public GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte member, out int sremCount) => storageSession.SetRemove(key, member, out sremCount, ref objectContext); /// - public GarnetStatus SetRemove(ArgSlice key, ArgSlice[] members, out int sremCount) + public GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int sremCount) => storageSession.SetRemove(key, members, out sremCount, ref objectContext); /// - public GarnetStatus SetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SetRemove(key, ref input, out output, ref objectContext); + public GarnetStatus SetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SetRemove(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetLength(ArgSlice key, out int count) + public GarnetStatus SetLength(PinnedSpanByte key, out int count) => storageSession.SetLength(key, out count, ref objectContext); /// - public GarnetStatus SetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.SetLength(key, ref input, out output, ref objectContext); + public GarnetStatus SetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.SetLength(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetMembers(ArgSlice key, out ArgSlice[] members) + public GarnetStatus SetMembers(PinnedSpanByte key, out PinnedSpanByte[] members) => storageSession.SetMembers(key, out members, ref objectContext); /// - public GarnetStatus SetMembers(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetMembers(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SetMembers(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetIsMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetIsMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SetIsMember(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] members, out int[] result) + public GarnetStatus SetIsMember(PinnedSpanByte key, PinnedSpanByte[] members, out int[] result) => storageSession.SetIsMember(key, members, out result, ref objectContext); /// - public GarnetStatus SetPop(ArgSlice key, out ArgSlice member) + public GarnetStatus SetPop(PinnedSpanByte key, out PinnedSpanByte member) => storageSession.SetPop(key, out member, ref objectContext); /// - public GarnetStatus SetPop(ArgSlice key, int count, out ArgSlice[] members) + public GarnetStatus SetPop(PinnedSpanByte key, int count, out PinnedSpanByte[] members) => storageSession.SetPop(key, count, out members, ref objectContext); /// - public GarnetStatus SetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SetPop(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.SetRandomMember(key, ref input, ref output, ref objectContext); /// - public GarnetStatus SetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus SetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) => storageSession.ObjectScan(GarnetObjectType.Set, key, cursor, match, count, out items, ref objectContext); /// - public GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey, ArgSlice member, out int smoveResult) + public GarnetStatus SetMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, PinnedSpanByte member, out int smoveResult) => storageSession.SetMove(sourceKey, destinationKey, member, out smoveResult); - public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetUnion(PinnedSpanByte[] keys, out HashSet output) => storageSession.SetUnion(keys, out output); /// - public GarnetStatus SetUnionStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetUnionStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) => storageSession.SetUnionStore(key, keys, out count); /// - public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet members) + public GarnetStatus SetDiff(PinnedSpanByte[] keys, out HashSet members) => storageSession.SetDiff(keys, out members); /// - public GarnetStatus SetDiffStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetDiffStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) => storageSession.SetDiffStore(key, keys, out count); /// - public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetIntersect(PinnedSpanByte[] keys, out HashSet output) => storageSession.SetIntersect(keys, out output); /// - public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) => storageSession.SetIntersectLength(keys, limit, out count); /// - public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetIntersectStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) => storageSession.SetIntersectStore(key, keys, out count); #endregion @@ -443,126 +438,121 @@ public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count #region Hash Methods /// - public GarnetStatus HashSet(ArgSlice key, ArgSlice field, ArgSlice value, out int count) + public GarnetStatus HashSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, out int count) => storageSession.HashSet(key, field, value, out count, ref objectContext); /// - public GarnetStatus HashSetWhenNotExists(ArgSlice key, ArgSlice field, ArgSlice value, out int count) + public GarnetStatus HashSetWhenNotExists(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, out int count) => storageSession.HashSet(key, field, value, out count, ref objectContext, nx: true); /// - public GarnetStatus HashSet(ArgSlice key, (ArgSlice field, ArgSlice value)[] elements, out int count) + public GarnetStatus HashSet(PinnedSpanByte key, (PinnedSpanByte field, PinnedSpanByte value)[] elements, out int count) => storageSession.HashSet(key, elements, out count, ref objectContext); /// - public GarnetStatus HashSet(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.HashSet(key, ref input, out output, ref objectContext); + public GarnetStatus HashSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.HashSet(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashDelete(ArgSlice key, ArgSlice field, out int count) + public GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte field, out int count) => storageSession.HashDelete(key, field, out count, ref objectContext); /// - public GarnetStatus HashDelete(ArgSlice key, ArgSlice[] fields, out int count) + public GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte[] fields, out int count) => storageSession.HashDelete(key, fields, out count, ref objectContext); /// - public GarnetStatus HashGet(ArgSlice key, ArgSlice field, out ArgSlice value) + public GarnetStatus HashGet(PinnedSpanByte key, PinnedSpanByte field, out PinnedSpanByte value) => storageSession.HashGet(key, field, out value, ref objectContext); /// - public GarnetStatus HashGetAll(ArgSlice key, out ArgSlice[] values) + public GarnetStatus HashGetAll(PinnedSpanByte key, out PinnedSpanByte[] values) => storageSession.HashGetAll(key, out values, ref objectContext); /// - public GarnetStatus HashGet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashGet(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashGetAll(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGetAll(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashGetAll(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashGetMultiple(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGetMultiple(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashGetMultiple(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashGetMultiple(ArgSlice key, ArgSlice[] fields, out ArgSlice[] values) + public GarnetStatus HashGetMultiple(PinnedSpanByte key, PinnedSpanByte[] fields, out PinnedSpanByte[] values) => storageSession.HashGetMultiple(key, fields, out values, ref objectContext); /// - public GarnetStatus HashLength(ArgSlice key, out int count) + public GarnetStatus HashLength(PinnedSpanByte key, out int count) => storageSession.HashLength(key, out count, ref objectContext); /// - public GarnetStatus HashLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.HashLength(key, ref input, out output, ref objectContext); + public GarnetStatus HashLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.HashLength(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashStrLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.HashStrLength(key, ref input, out output, ref objectContext); + public GarnetStatus HashStrLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.HashStrLength(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashExists(ArgSlice key, ArgSlice field, out bool exists) + public GarnetStatus HashExists(PinnedSpanByte key, PinnedSpanByte field, out bool exists) => storageSession.HashExists(key, field, out exists, ref objectContext); /// - public GarnetStatus HashExists(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.HashExists(key, ref input, out output, ref objectContext); + public GarnetStatus HashExists(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.HashExists(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashRandomField(ArgSlice key, out ArgSlice field) + public GarnetStatus HashRandomField(PinnedSpanByte key, out PinnedSpanByte field) => storageSession.HashRandomField(key, out field, ref objectContext); /// - public GarnetStatus HashRandomField(ArgSlice key, int count, bool withValues, out ArgSlice[] fields) + public GarnetStatus HashRandomField(PinnedSpanByte key, int count, bool withValues, out PinnedSpanByte[] fields) => storageSession.HashRandomField(key, count, withValues, out fields, ref objectContext); /// - public GarnetStatus HashRandomField(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashRandomField(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashRandomField(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashDelete(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) - => storageSession.HashDelete(key, ref input, out output, ref objectContext); + public GarnetStatus HashDelete(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) + => storageSession.HashDelete(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashKeys(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashKeys(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashKeys(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashVals(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashVals(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashVals(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashIncrement(byte[] key, ArgSlice input, out ObjectOutputHeader output) - => storageSession.HashIncrement(key, input, out output, ref objectContext); - - /// - public GarnetStatus HashIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashIncrement(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashExpire(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashPersist(key, ref input, ref output, ref objectContext); /// - public GarnetStatus HashScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus HashScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) => storageSession.ObjectScan(GarnetObjectType.Hash, key, cursor, match, count, out items, ref objectContext); /// - public GarnetStatus HashTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashTimeToLive(PinnedSpanByte key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref ObjectOutput output) => storageSession.HashTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref output, ref objectContext); /// - public GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input) + public GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input) => storageSession.HashCollect(keys, ref input, ref objectContext); #endregion } - } \ No newline at end of file diff --git a/libs/server/API/GarnetApiUnifiedCommands.cs b/libs/server/API/GarnetApiUnifiedCommands.cs new file mode 100644 index 00000000000..1e8e665556e --- /dev/null +++ b/libs/server/API/GarnetApiUnifiedCommands.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Garnet API implementation + /// + public partial struct GarnetApi : IGarnetApi, IGarnetWatchApi + where TStringContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + where TUnifiedContext : ITsavoriteContext + { + #region MEMORY + + /// + public GarnetStatus MEMORYUSAGE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + #endregion + + #region TYPE + + /// + public GarnetStatus TYPE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + #endregion + + #region TTL + + /// + public GarnetStatus TTL(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + #endregion + + #region EXPIRETIME + + /// + public GarnetStatus EXPIRETIME(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + #endregion + + #region EXISTS + + /// + public GarnetStatus EXISTS(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + /// + public GarnetStatus EXISTS(PinnedSpanByte key) + => storageSession.EXISTS(key, ref unifiedContext); + + #endregion + + #region DELETE + + /// + public GarnetStatus DELETE(PinnedSpanByte key) + => storageSession.DELETE(key, ref unifiedContext); + + /// + public GarnetStatus DELIFEXPIM(PinnedSpanByte key) + => storageSession.DELIFEXPIM(key, ref unifiedContext); + + #endregion + + #region EXPIRE + + /// + public unsafe GarnetStatus EXPIRE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.RMW_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + /// + public unsafe GarnetStatus EXPIRE(PinnedSpanByte key, PinnedSpanByte expiryMs, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIRE(key, expiryMs, out timeoutSet, expireOption, ref unifiedContext); + + /// + public GarnetStatus EXPIRE(PinnedSpanByte key, TimeSpan expiry, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIRE(key, expiry, out timeoutSet, expireOption, ref unifiedContext); + + #endregion + + #region EXPIREAT + + /// + public GarnetStatus EXPIREAT(PinnedSpanByte key, long expiryTimestamp, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, expireOption, ref unifiedContext); + + /// + public GarnetStatus PEXPIREAT(PinnedSpanByte key, long expiryTimestamp, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, expireOption, ref unifiedContext, milliseconds: true); + + #endregion + + #region PERSIST + + /// + public unsafe GarnetStatus PERSIST(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) + => storageSession.RMW_UnifiedStore(key, ref input, ref output, ref unifiedContext); + + #endregion + } +} \ No newline at end of file diff --git a/libs/server/API/GarnetStatus.cs b/libs/server/API/GarnetStatus.cs index 20c965d1668..0972da6fb66 100644 --- a/libs/server/API/GarnetStatus.cs +++ b/libs/server/API/GarnetStatus.cs @@ -24,9 +24,5 @@ public enum GarnetStatus : byte /// Wrong type /// WRONGTYPE, - /// - /// Bad state - /// - BADSTATE, } } \ No newline at end of file diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index d60d8546f65..d37a1aa5cf8 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -23,35 +23,35 @@ public GarnetWatchApi(TGarnetApi garnetApi) #region GET /// - public GarnetStatus GET(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output) + public GarnetStatus GET(PinnedSpanByte key, ref StringInput input, ref StringOutput output) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.GET(key, ref input, ref output); } /// - public GarnetStatus GETForMemoryResult(ArgSlice key, out MemoryResult value) + public GarnetStatus GETForMemoryResult(PinnedSpanByte key, out MemoryResult value) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.GETForMemoryResult(key, out value); } /// - public GarnetStatus GET(ArgSlice key, out ArgSlice value) + public GarnetStatus GET(PinnedSpanByte key, out PinnedSpanByte value) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.GET(key, out value); } /// - public GarnetStatus GET(byte[] key, out GarnetObjectStoreOutput value) + public GarnetStatus GET(PinnedSpanByte key, out ObjectOutput value) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.GET(key, out value); } /// - public GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) + public GarnetStatus LCS(PinnedSpanByte key1, PinnedSpanByte key2, ref StringOutput output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) { garnetApi.WATCH(key1, StoreType.Object); garnetApi.WATCH(key2, StoreType.Object); @@ -61,26 +61,19 @@ public GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory outp #region GETRANGE /// - public GarnetStatus GETRANGE(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) + public GarnetStatus GETRANGE(PinnedSpanByte key, ref StringInput input, ref StringOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); - return garnetApi.GETRANGE(ref key, ref input, ref output); + garnetApi.WATCH(key, StoreType.Main); + return garnetApi.GETRANGE(key, ref input, ref output); } #endregion #region TTL /// - public GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - { - garnetApi.WATCH(new ArgSlice(ref key), storeType); - return garnetApi.TTL(ref key, storeType, ref output); - } - - /// - public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + public GarnetStatus TTL(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), storeType); - return garnetApi.PTTL(ref key, storeType, ref output); + garnetApi.WATCH(key, StoreType.All); + return garnetApi.TTL(key, ref input, ref output); } #endregion @@ -88,17 +81,10 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM #region EXPIRETIME /// - public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + public GarnetStatus EXPIRETIME(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), storeType); - return garnetApi.EXPIRETIME(ref key, storeType, ref output); - } - - /// - public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) - { - garnetApi.WATCH(new ArgSlice(ref key), storeType); - return garnetApi.PEXPIRETIME(ref key, storeType, ref output); + garnetApi.WATCH(key, StoreType.All); + return garnetApi.EXPIRETIME(key, ref input, ref output); } #endregion @@ -106,91 +92,91 @@ public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanB #region SortedSet Methods /// - public GarnetStatus SortedSetLength(ArgSlice key, out int zcardCount) + public GarnetStatus SortedSetLength(PinnedSpanByte key, out int zcardCount) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetLength(key, out zcardCount); } /// - public GarnetStatus SortedSetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus SortedSetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.SortedSetLength(key, ref input, out output); + return garnetApi.SortedSetLength(key, ref input, ref output); } /// - public GarnetStatus SortedSetCount(ArgSlice key, ArgSlice minScore, ArgSlice maxScore, out int numElements) + public GarnetStatus SortedSetCount(PinnedSpanByte key, PinnedSpanByte minScore, PinnedSpanByte maxScore, out int numElements) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetCount(key, minScore, maxScore, out numElements); } /// - public GarnetStatus SortedSetCount(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetCount(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetCount(key, ref input, ref output); } /// - public GarnetStatus SortedSetLengthByValue(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus SortedSetLengthByValue(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.SortedSetLengthByValue(key, ref input, out output); + return garnetApi.SortedSetLengthByValue(key, ref input, ref output); } /// - public GarnetStatus SortedSetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetRandomMember(key, ref input, ref output); } /// - public GarnetStatus SortedSetRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetRange(key, ref input, ref output); } /// - public GarnetStatus SortedSetScore(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetScore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetScore(key, ref input, ref output); } /// - public GarnetStatus SortedSetScores(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetScores(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetScores(key, ref input, ref output); } /// - public GarnetStatus SortedSetRank(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetRank(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetRank(key, ref input, ref output); } /// - public GarnetStatus SortedSetRank(ArgSlice key, ArgSlice member, bool reverse, out long? rank) + public GarnetStatus SortedSetRank(PinnedSpanByte key, PinnedSpanByte member, bool reverse, out long? rank) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetRank(key, member, reverse, out rank); } /// - public GarnetStatus SortedSetRange(ArgSlice key, ArgSlice min, ArgSlice max, SortedSetOrderOperation sortedSetOrderOperation, out ArgSlice[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) + public GarnetStatus SortedSetRange(PinnedSpanByte key, PinnedSpanByte min, PinnedSpanByte max, SortedSetOrderOperation sortedSetOrderOperation, out PinnedSpanByte[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetRange(key, min, max, sortedSetOrderOperation, out elements, out error, withScores, reverse, limit); } /// - public GarnetStatus SortedSetDifference(ArgSlice[] keys, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetDifference(PinnedSpanByte[] keys, out SortedSet<(double, byte[])> pairs) { foreach (var key in keys) { @@ -200,7 +186,7 @@ public GarnetStatus SortedSetDifference(ArgSlice[] keys, out SortedSet<(double, } /// - public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) { foreach (var key in keys) { @@ -210,14 +196,14 @@ public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights } /// - public GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus GeoCommands(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.GeoCommands(key, ref input, ref output); } /// - public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, + public GarnetStatus GeoSearchReadOnly(PinnedSpanByte key, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output) { garnetApi.WATCH(key, StoreType.Object); @@ -225,14 +211,14 @@ public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, } /// - public GarnetStatus SortedSetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus SortedSetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetScan(key, cursor, match, count, out items); } /// - public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) { foreach (var key in keys) { @@ -242,7 +228,7 @@ public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] wei } /// - public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) { foreach (var key in keys) { @@ -252,14 +238,14 @@ public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? l } /// - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetTimeToLive(key, ref input, ref output); } /// - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn) + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ReadOnlySpan members, out TimeSpan[] expireIn) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SortedSetTimeToLive(key, members, out expireIn); @@ -270,28 +256,28 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan mem #region List Methods /// - public GarnetStatus ListLength(ArgSlice key, out int count) + public GarnetStatus ListLength(PinnedSpanByte key, out int count) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.ListLength(key, out count); } /// - public GarnetStatus ListLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus ListLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.ListLength(key, ref input, out output); + return garnetApi.ListLength(key, ref input, ref output); } /// - public GarnetStatus ListRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.ListRange(key, ref input, ref output); } /// - public GarnetStatus ListIndex(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ListIndex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.ListIndex(key, ref input, ref output); @@ -302,56 +288,56 @@ public GarnetStatus ListIndex(byte[] key, ref ObjectInput input, ref GarnetObjec #region Set Methods /// - public GarnetStatus SetLength(ArgSlice key, out int scardCount) + public GarnetStatus SetLength(PinnedSpanByte key, out int scardCount) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetLength(key, out scardCount); } /// - public GarnetStatus SetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus SetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.SetLength(key, ref input, out output); + return garnetApi.SetLength(key, ref input, ref output); } /// - public GarnetStatus SetMembers(ArgSlice key, out ArgSlice[] members) + public GarnetStatus SetMembers(PinnedSpanByte key, out PinnedSpanByte[] members) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetMembers(key, out members); } /// - public GarnetStatus SetIsMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetIsMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetIsMember(key, ref input, ref output); } /// - public GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] members, out int[] result) + public GarnetStatus SetIsMember(PinnedSpanByte key, PinnedSpanByte[] members, out int[] result) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetIsMember(key, members, out result); } /// - public GarnetStatus SetMembers(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus SetMembers(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetMembers(key, ref input, ref output); } /// - public GarnetStatus SetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus SetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.SetScan(key, cursor, match, count, out items); } /// - public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetUnion(PinnedSpanByte[] keys, out HashSet output) { foreach (var key in keys) { @@ -361,7 +347,7 @@ public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) } /// - public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetIntersect(PinnedSpanByte[] keys, out HashSet output) { foreach (var key in keys) { @@ -371,7 +357,7 @@ public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) } /// - public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetDiff(PinnedSpanByte[] keys, out HashSet output) { foreach (var key in keys) { @@ -380,7 +366,7 @@ public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet output) return garnetApi.SetDiff(keys, out output); } - public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) { foreach (var key in keys) { @@ -393,124 +379,124 @@ public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, #region Hash Methods /// - public GarnetStatus HashGet(ArgSlice key, ArgSlice field, out ArgSlice value) + public GarnetStatus HashGet(PinnedSpanByte key, PinnedSpanByte field, out PinnedSpanByte value) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGet(key, field, out value); } /// - public GarnetStatus HashGetMultiple(ArgSlice key, ArgSlice[] fields, out ArgSlice[] values) + public GarnetStatus HashGetMultiple(PinnedSpanByte key, PinnedSpanByte[] fields, out PinnedSpanByte[] values) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGetMultiple(key, fields, out values); } /// - public GarnetStatus HashGetAll(ArgSlice key, out ArgSlice[] values) + public GarnetStatus HashGetAll(PinnedSpanByte key, out PinnedSpanByte[] values) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGetAll(key, out values); } /// - public GarnetStatus HashLength(ArgSlice key, out int count) + public GarnetStatus HashLength(PinnedSpanByte key, out int count) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashLength(key, out count); } /// - public GarnetStatus HashExists(ArgSlice key, ArgSlice field, out bool exists) + public GarnetStatus HashExists(PinnedSpanByte key, PinnedSpanByte field, out bool exists) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashExists(key, field, out exists); } /// - public GarnetStatus HashRandomField(ArgSlice key, int count, bool withValues, out ArgSlice[] fields) + public GarnetStatus HashRandomField(PinnedSpanByte key, int count, bool withValues, out PinnedSpanByte[] fields) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashRandomField(key, count, withValues, out fields); } /// - public GarnetStatus HashRandomField(ArgSlice key, out ArgSlice field) + public GarnetStatus HashRandomField(PinnedSpanByte key, out PinnedSpanByte field) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashRandomField(key, out field); } /// - public GarnetStatus HashRandomField(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashRandomField(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashRandomField(key, ref input, ref output); } /// - public GarnetStatus HashGet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGet(key, ref input, ref output); } - public GarnetStatus HashGetAll(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGetAll(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGetAll(key, ref input, ref output); } - public GarnetStatus HashGetMultiple(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashGetMultiple(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashGetMultiple(key, ref input, ref output); } /// - public GarnetStatus HashStrLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus HashStrLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.HashStrLength(key, ref input, out output); + return garnetApi.HashStrLength(key, ref input, ref output); } /// - public GarnetStatus HashExists(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus HashExists(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.HashExists(key, ref input, out output); + return garnetApi.HashExists(key, ref input, ref output); } /// - public GarnetStatus HashKeys(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashKeys(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashKeys(key, ref input, ref output); } /// - public GarnetStatus HashVals(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashVals(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashVals(key, ref input, ref output); } /// - public GarnetStatus HashLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output) + public GarnetStatus HashLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); - return garnetApi.HashLength(key, ref input, out output); + return garnetApi.HashLength(key, ref input, ref output); } /// - public GarnetStatus HashScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) + public GarnetStatus HashScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashScan(key, cursor, match, count, out items); } /// - public GarnetStatus HashTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus HashTimeToLive(PinnedSpanByte key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.Object); return garnetApi.HashTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref output); @@ -521,45 +507,45 @@ public GarnetStatus HashTimeToLive(ArgSlice key, bool isMilliseconds, bool isTim #region Bitmap Methods /// - public GarnetStatus StringGetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) + public GarnetStatus StringGetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); - return garnetApi.StringGetBit(ref key, ref input, ref output); + garnetApi.WATCH(key, StoreType.Main); + return garnetApi.StringGetBit(key, ref input, ref output); } /// - public GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, out bool bValue) + public GarnetStatus StringGetBit(PinnedSpanByte key, PinnedSpanByte offset, out bool bValue) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.StringGetBit(key, offset, out bValue); } /// - public GarnetStatus StringBitCount(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) + public GarnetStatus StringBitCount(PinnedSpanByte key, ref StringInput input, ref StringOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); - return garnetApi.StringBitCount(ref key, ref input, ref output); + garnetApi.WATCH(key, StoreType.Main); + return garnetApi.StringBitCount(key, ref input, ref output); } /// - public GarnetStatus StringBitCount(ArgSlice key, long start, long end, out long result, bool useBitInterval = false) + public GarnetStatus StringBitCount(PinnedSpanByte key, long start, long end, out long result, bool useBitInterval = false) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.StringBitCount(key, start, end, out result, useBitInterval); } /// - public GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output) + public GarnetStatus StringBitPosition(PinnedSpanByte key, ref StringInput input, ref StringOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); - return garnetApi.StringBitPosition(ref key, ref input, ref output); + garnetApi.WATCH(key, StoreType.Main); + return garnetApi.StringBitPosition(key, ref input, ref output); } /// - public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) + public GarnetStatus StringBitFieldReadOnly(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output) { - garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); - return garnetApi.StringBitFieldReadOnly(ref key, ref input, secondaryCommand, ref output); + garnetApi.WATCH(key, StoreType.Main); + return garnetApi.StringBitFieldReadOnly(key, ref input, secondaryCommand, ref output); } #endregion @@ -567,7 +553,7 @@ public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput #region HLL Methods /// - public GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, out bool error) + public GarnetStatus HyperLogLogLength(ref StringInput input, out long count, out bool error) { for (var i = 0; i < input.parseState.Count; i++) { @@ -579,7 +565,7 @@ public GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, } /// - public GarnetStatus HyperLogLogLength(Span keys, out long count) + public GarnetStatus HyperLogLogLength(Span keys, out long count) { foreach (var key in keys) { @@ -593,7 +579,7 @@ public GarnetStatus HyperLogLogLength(Span keys, out long count) #region Server Methods /// - public List GetDbKeys(ArgSlice pattern) + public List GetDbKeys(PinnedSpanByte pattern) { return garnetApi.GetDbKeys(pattern); } @@ -605,87 +591,74 @@ public int GetDbSize() } /// - public bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out long cursorStore, out List keys, long count = 10, ReadOnlySpan type = default) + public bool DbScan(PinnedSpanByte patternB, bool allKeys, long cursor, out long cursorStore, out List keys, long count = 10, ReadOnlySpan type = default) { return garnetApi.DbScan(patternB, allKeys, cursor, out cursorStore, out keys, count, type); } /// - public bool IterateMainStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => garnetApi.IterateMainStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); - - /// - public ITsavoriteScanIterator IterateMainStore() - => garnetApi.IterateMainStore(); - - /// - public bool IterateObjectStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => garnetApi.IterateObjectStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); + public bool IterateStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + => garnetApi.IterateStore(ref scanFunctions, ref cursor, untilAddress, maxAddress: maxAddress, includeTombstones: includeTombstones); /// - public ITsavoriteScanIterator IterateObjectStore() - => garnetApi.IterateObjectStore(); + public void DeleteSlotKeys(HashSet slots) + => garnetApi.DeleteSlotKeys(slots); #endregion #region Common Methods - public GarnetStatus ObjectScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output) + public GarnetStatus ObjectScan(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output) { garnetApi.WATCH(key, StoreType.All); return garnetApi.ObjectScan(key, ref input, ref output); } /// - public int GetScratchBufferOffset() - => garnetApi.GetScratchBufferOffset(); - - /// - public bool ResetScratchBuffer(int offset) - => garnetApi.ResetScratchBuffer(offset); + public void ResetScratchBuffer() + => garnetApi.ResetScratchBuffer(); #endregion #region Vector Sets /// - public GarnetStatus VectorSetValueSimilarity(ArgSlice key, VectorValueType valueType, ArgSlice value, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + public GarnetStatus VectorSetValueSimilarity(PinnedSpanByte key, VectorValueType valueType, PinnedSpanByte value, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetValueSimilarity(key, valueType, value, count, delta, searchExplorationFactor, filter, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); } /// - public GarnetStatus VectorSetElementSimilarity(ArgSlice key, ArgSlice element, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + public GarnetStatus VectorSetElementSimilarity(PinnedSpanByte key, PinnedSpanByte element, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetElementSimilarity(key, element, count, delta, searchExplorationFactor, filter, maxFilteringEffort, includeAttributes, ref outputIds, out outputIdFormat, ref outputDistances, ref outputAttributes, out result, ref filterBitmap); } /// - public GarnetStatus VectorSetEmbedding(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputDistances) + public GarnetStatus VectorSetEmbedding(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputDistances) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetEmbedding(key, element, ref outputDistances); } /// - public GarnetStatus VectorSetDimensions(ArgSlice key, out int dimensions) + public GarnetStatus VectorSetDimensions(PinnedSpanByte key, out int dimensions) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetDimensions(key, out dimensions); } /// - public GarnetStatus VectorSetInfo(ArgSlice key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size) + public GarnetStatus VectorSetInfo(PinnedSpanByte key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetInfo(key, out quantType, out distanceMetricType, out vectorDimensions, out reducedDimensions, out buildExplorationFactor, out numberOfLinks, out size); } /// - public GarnetStatus VectorSetGetAttribute(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputAttributes) + public GarnetStatus VectorSetGetAttribute(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputAttributes) { garnetApi.WATCH(key, StoreType.Main); return garnetApi.VectorSetGetAttribute(key, element, ref outputAttributes); diff --git a/libs/server/API/IGarnetAdvancedApi.cs b/libs/server/API/IGarnetAdvancedApi.cs index 252ccb38ebe..ff9d7c4dbe9 100644 --- a/libs/server/API/IGarnetAdvancedApi.cs +++ b/libs/server/API/IGarnetAdvancedApi.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using Garnet.common; using Tsavorite.core; namespace Garnet.server @@ -13,14 +14,14 @@ public interface IGarnetAdvancedApi /// /// GET with support for pending multiple ongoing operations, scatter gather IO for outputs /// - GarnetStatus GET_WithPending(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, long ctx, out bool pending); + GarnetStatus GET_WithPending(PinnedSpanByte key, ref StringInput input, ref StringOutput output, long ctx, out bool pending); /// /// Complete pending read operations on main store /// /// /// - bool GET_CompletePending((GarnetStatus, SpanByteAndMemory)[] outputArr, bool wait = false); + bool GET_CompletePending((GarnetStatus, StringOutput)[] outputArr, bool wait = false); /// /// Complete pending read operations on main store @@ -28,33 +29,43 @@ public interface IGarnetAdvancedApi /// /// /// - bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait = false); + bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait = false); /// /// RMW operation on main store /// - GarnetStatus RMW_MainStore(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus RMW_MainStore(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Read operation on main store /// - GarnetStatus Read_MainStore(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus Read_MainStore(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// RMW operation on object store /// - GarnetStatus RMW_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus RMW_ObjectStore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Read operation on object store /// - GarnetStatus Read_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus Read_ObjectStore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); + + /// + /// RMW operation on unified store + /// + GarnetStatus RMW_UnifiedStore(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); + + /// + /// Read operation on unified store + /// + GarnetStatus Read_UnifiedStore(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); /// /// Read batch of keys on main store. /// void ReadWithPrefetch(ref TBatch batch, long context = default) - where TBatch : IReadArgBatch + where TBatch : IReadArgBatch #if NET9_0_OR_GREATER , allows ref struct #endif diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index ac4e370a762..24057f187ad 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -18,7 +18,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// GETEX /// - GarnetStatus GETEX(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus GETEX(PinnedSpanByte key, ref StringInput input, ref StringOutput output); #endregion @@ -26,46 +26,53 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// SET /// - GarnetStatus SET(ArgSlice key, ref RawStringInput input, ref SpanByte value); + GarnetStatus SET(PinnedSpanByte key, PinnedSpanByte value); /// - /// SET Conditional + /// SET /// - GarnetStatus SET_Conditional(ArgSlice key, ref RawStringInput input); + GarnetStatus SET(PinnedSpanByte key, ref StringInput input, PinnedSpanByte value); /// - /// DEL Conditional + /// SET Conditional /// - GarnetStatus DEL_Conditional(ref SpanByte key, ref RawStringInput input); + GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input); /// /// SET Conditional /// - GarnetStatus SET_Conditional(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input, ref StringOutput output); + + /// + /// SETWITHETAG / SETIFMATCH / SETIFGREATER — ETag-aware conditional set + /// + GarnetStatus SET_ETagConditional(PinnedSpanByte key, ref StringInput input, ref StringOutput output); + + /// + /// DELIFGREATER — ETag-aware conditional delete + /// + GarnetStatus DEL_ETagConditional(PinnedSpanByte key, ref StringInput input); /// /// SET /// - /// - /// - /// - GarnetStatus SET(ArgSlice key, Memory value); + GarnetStatus SET(PinnedSpanByte key, Memory value); /// /// SET /// - /// - /// - /// - GarnetStatus SET(ArgSlice key, ArgSlice value); + GarnetStatus SET(PinnedSpanByte key, IGarnetObject value); /// /// SET /// - /// - /// - /// - GarnetStatus SET(byte[] key, IGarnetObject value); + GarnetStatus SET(in TSourceLogRecord srcLogRecord) where TSourceLogRecord : ISourceLogRecord; + + /// + /// SET + /// + GarnetStatus SET(PinnedSpanByte key, ref UnifiedInput input, in TSourceLogRecord srcLogRecord) where TSourceLogRecord : ISourceLogRecord; + #endregion #region SETEX @@ -76,7 +83,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Value /// Expiry in milliseconds, formatted as ASCII digits /// - GarnetStatus SETEX(ArgSlice key, ArgSlice value, ArgSlice expiryMs); + GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, PinnedSpanByte expiryMs); /// /// SETEX @@ -84,7 +91,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key /// Value /// Expiry - GarnetStatus SETEX(ArgSlice key, ArgSlice value, TimeSpan expiry); + GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, TimeSpan expiry); #endregion @@ -97,7 +104,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// The output of the operation /// - GarnetStatus SETRANGE(ArgSlice key, ref RawStringInput input, ref ArgSlice output); + GarnetStatus SETRANGE(PinnedSpanByte key, ref StringInput input, ref PinnedSpanByte output); #endregion @@ -108,7 +115,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus MSET_Conditional(ref RawStringInput input); + GarnetStatus MSET_Conditional(ref StringInput input); #endregion #region APPEND @@ -120,7 +127,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// Length of updated value /// Operation status - GarnetStatus APPEND(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus APPEND(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// APPEND command @@ -129,18 +136,17 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Value to be appended /// Length of updated value /// Operation status - GarnetStatus APPEND(ArgSlice key, ArgSlice value, ref ArgSlice output); + GarnetStatus APPEND(PinnedSpanByte key, PinnedSpanByte value, ref PinnedSpanByte output); #endregion #region RENAME /// /// RENAME /// - /// - /// - /// + /// The old key to be renamed. + /// The new key name. /// - GarnetStatus RENAME(ArgSlice oldKey, ArgSlice newKey, bool withEtag = false, StoreType storeType = StoreType.All); + GarnetStatus RENAME(PinnedSpanByte oldKey, PinnedSpanByte newKey); /// /// Renames key to newkey if newkey does not yet exist. It returns an error when key does not exist. @@ -148,19 +154,28 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The old key to be renamed. /// The new key name. /// The result of the operation. - /// The type of store to perform the operation on. /// - GarnetStatus RENAMENX(ArgSlice oldKey, ArgSlice newKey, out int result, bool withEtag = false, StoreType storeType = StoreType.All); + GarnetStatus RENAMENX(PinnedSpanByte oldKey, PinnedSpanByte newKey, out int result); #endregion #region EXISTS + /// /// EXISTS /// /// - /// + /// + /// + /// + GarnetStatus EXISTS(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); + + /// + /// EXISTS + /// + /// Key /// - GarnetStatus EXISTS(ArgSlice key, StoreType storeType = StoreType.All); + GarnetStatus EXISTS(PinnedSpanByte key); + #endregion #region EXPIRE @@ -170,20 +185,18 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key /// Expiry in milliseconds, formatted as ASCII digits /// Whether timeout was set by the call - /// Store type: main, object, or both /// Expire option /// - GarnetStatus EXPIRE(ArgSlice key, ArgSlice expiryMs, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + GarnetStatus EXPIRE(PinnedSpanByte key, PinnedSpanByte expiryMs, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None); /// /// Set a timeout on key using a timeSpan in seconds /// /// Key /// - /// Whether timeout was set by the call - /// Store type: main, object, or both + /// /// - GarnetStatus EXPIRE(ArgSlice key, ref RawStringInput input, out bool timeoutSet, StoreType storeType = StoreType.All); + GarnetStatus EXPIRE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); /// /// Set a timeout on key using a timeSpan in seconds @@ -191,10 +204,9 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key /// Expiry in TimeSpan /// Whether timeout was set by the call - /// Store type: main, object, or both /// Expire option /// - GarnetStatus EXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + GarnetStatus EXPIRE(PinnedSpanByte key, TimeSpan expiry, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None); #endregion #region EXPIREAT @@ -205,10 +217,9 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key /// Absolute Unix timestamp in seconds /// Whether timeout was set by the call - /// Store type: main, object, or both /// Expire option /// - GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + GarnetStatus EXPIREAT(PinnedSpanByte key, long expiryTimestamp, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None); /// /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in milliseconds @@ -216,21 +227,22 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key /// Absolute Unix timestamp in milliseconds /// Whether timeout was set by the call - /// Store type: main, object, or both /// Expire option /// - GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + GarnetStatus PEXPIREAT(PinnedSpanByte key, long expiryTimestamp, out bool timeoutSet, ExpireOption expireOption = ExpireOption.None); #endregion #region PERSIST + /// /// PERSIST /// /// Key - /// Store type: main, object, or both + /// + /// /// - GarnetStatus PERSIST(ArgSlice key, StoreType storeType = StoreType.All); + GarnetStatus PERSIST(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); #endregion #region Increment (INCR, INCRBY, DECR, DECRBY) @@ -241,7 +253,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus Increment(ArgSlice key, ref RawStringInput input, ref ArgSlice output); + GarnetStatus Increment(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Increment (INCR, INCRBY) @@ -250,7 +262,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus Increment(ArgSlice key, out long output, long incrementCount = 1); + GarnetStatus Increment(PinnedSpanByte key, out long output, long incrementCount = 1); /// /// Decrement (DECR, DECRBY) @@ -259,7 +271,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus Decrement(ArgSlice key, out long output, long decrementCount = 1); + GarnetStatus Decrement(PinnedSpanByte key, out long output, long decrementCount = 1); /// /// Increment by float (INCRBYFLOAT) @@ -268,7 +280,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus IncrementByFloat(ArgSlice key, ref ArgSlice output, double val); + GarnetStatus IncrementByFloat(PinnedSpanByte key, ref StringOutput output, double val); /// /// Increment by float (INCRBYFLOAT) @@ -277,33 +289,23 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus IncrementByFloat(ArgSlice key, out double output, double val); + GarnetStatus IncrementByFloat(PinnedSpanByte key, out double output, double val); #endregion #region DELETE - /// - /// DELETE - /// - /// - /// - /// - GarnetStatus DELETE(ArgSlice key, StoreType storeType = StoreType.All); /// - /// DELETE + /// Deletes a key from the unified store /// /// - /// /// - GarnetStatus DELETE(ref SpanByte key, StoreType storeType = StoreType.All); + GarnetStatus DELETE(PinnedSpanByte key); /// - /// DELETE + /// Deletes a key if it is in memory and expired /// - /// - /// - /// - GarnetStatus DELETE(byte[] key, StoreType storeType = StoreType.All); + GarnetStatus DELIFEXPIM(PinnedSpanByte key); + #endregion #region GETDEL @@ -313,15 +315,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Key to get and delete /// Current value of key /// Operation status - GarnetStatus GETDEL(ref SpanByte key, ref SpanByteAndMemory output); - - /// - /// GETDEL - /// - /// Key to get and delete - /// Current value of key - /// Operation status - GarnetStatus GETDEL(ArgSlice key, ref SpanByteAndMemory output); + GarnetStatus GETDEL(PinnedSpanByte key, ref StringOutput output); #endregion #region TYPE @@ -331,9 +325,10 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// string, list, set, zset, and hash. /// /// - /// + /// + /// /// - GarnetStatus GetKeyType(ArgSlice key, out string typeName); + GarnetStatus TYPE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); #endregion @@ -343,10 +338,10 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Gets the number of bytes that a key and its value require to be stored in RAM. /// /// Name of the key or object to get the memory usage - /// The value in bytes the key or object is using - /// Number of sampled nested values + /// + /// /// GarnetStatus - GarnetStatus MemoryUsageForKey(ArgSlice key, out long memoryUsage, int samples = 0); + GarnetStatus MEMORYUSAGE(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); #endregion @@ -360,7 +355,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Member /// Number of adds performed /// - GarnetStatus SortedSetAdd(ArgSlice key, ArgSlice score, ArgSlice member, out int zaddCount); + GarnetStatus SortedSetAdd(PinnedSpanByte key, PinnedSpanByte score, PinnedSpanByte member, out int zaddCount); /// /// Adds all the specified members with the specified scores to the sorted set stored at key. @@ -370,7 +365,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Input key-value pairs to add /// Number of adds performed /// - GarnetStatus SortedSetAdd(ArgSlice key, (ArgSlice score, ArgSlice member)[] inputs, out int zaddCount); + GarnetStatus SortedSetAdd(PinnedSpanByte key, (PinnedSpanByte score, PinnedSpanByte member)[] inputs, out int zaddCount); /// /// Adds all the specified members with the specified scores to the sorted set stored at key. @@ -380,7 +375,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Stores a range of sorted set elements in the specified key space. @@ -390,12 +385,12 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The input object containing the elements to store. /// The result of the store operation. /// A indicating the status of the operation. - GarnetStatus SortedSetRangeStore(ArgSlice dstKey, ArgSlice srcKey, ref ObjectInput input, out int result); + GarnetStatus SortedSetRangeStore(PinnedSpanByte dstKey, PinnedSpanByte srcKey, ref ObjectInput input, out int result); /// /// Removes the specified member from the sorted set stored at key. /// - GarnetStatus SortedSetRemove(ArgSlice key, ArgSlice member, out int zremCount); + GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte member, out int zremCount); /// /// Removes the specified members from the sorted set stored at key. @@ -405,7 +400,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// Input members to remove /// Number of removes performed /// - GarnetStatus SortedSetRemove(ArgSlice key, ArgSlice[] members, out int zremCount); + GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int zremCount); /// /// Removes the specified members from the sorted set stored at key. @@ -415,7 +410,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SortedSetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes all elements in the sorted set between the @@ -425,7 +420,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemoveRangeByLex(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes and returns the first element from the sorted set stored at key, @@ -435,7 +430,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes and returns multiple elements from a sorted set. @@ -446,7 +441,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// The key of the popped element. /// An array of tuples containing the member and score of each popped element. /// A indicating the result of the operation. - GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out ArgSlice poppedKey, out (ArgSlice member, ArgSlice score)[] pairs); + GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out PinnedSpanByte poppedKey, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs); /// /// Removes and returns up to count members with the highest or lowest scores in the sorted set stored at key. @@ -456,7 +451,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// When true, return the members with the lowest scores, otherwise return the highest scores. /// - GarnetStatus SortedSetPop(ArgSlice key, out (ArgSlice member, ArgSlice score)[] pairs, int count = 1, bool lowScoresFirst = true); + GarnetStatus SortedSetPop(PinnedSpanByte key, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs, int count = 1, bool lowScoresFirst = true); /// /// Increments the score of member in the sorted set stored at key by increment. @@ -466,7 +461,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Increments the score of member in the sorted set stored at key by increment. @@ -478,7 +473,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetIncrement(ArgSlice key, Double increment, ArgSlice member, out double newScore); + GarnetStatus SortedSetIncrement(PinnedSpanByte key, double increment, PinnedSpanByte member, out double newScore); /// /// ZREMRANGEBYRANK: Removes all elements in the sorted set stored at key with rank between start and stop. @@ -489,7 +484,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemoveRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetRemoveRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes all elements in the range specified by min and max, having the same score. @@ -499,7 +494,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemoveRangeByLex(ArgSlice key, string min, string max, out int countRemoved); + GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, string min, string max, out int countRemoved); /// /// Removes all elements that have a score in the range specified by min and max. @@ -509,7 +504,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemoveRangeByScore(ArgSlice key, string min, string max, out int countRemoved); + GarnetStatus SortedSetRemoveRangeByScore(PinnedSpanByte key, string min, string max, out int countRemoved); /// /// Removes all elements with the index in the range specified by start and stop. @@ -519,7 +514,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetRemoveRangeByRank(ArgSlice key, int start, int stop, out int countRemoved); + GarnetStatus SortedSetRemoveRangeByRank(PinnedSpanByte key, int start, int stop, out int countRemoved); /// /// Computes the difference between the first and all successive sorted sets and store resulting pairs in the output key. @@ -528,7 +523,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count); + GarnetStatus SortedSetDifferenceStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, out int count); /// /// Adds geospatial items (longitude, latitude, name) to the specified key. @@ -537,7 +532,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus GeoAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Geospatial search and store in destination key. @@ -548,7 +543,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearchOptions opts, + GarnetStatus GeoSearchStore(PinnedSpanByte key, PinnedSpanByte destinationKey, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output); /// @@ -560,7 +555,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The type of aggregation to use for the intersection. /// The number of elements in the resulting sorted set. /// A indicating the status of the operation. - GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count); + GarnetStatus SortedSetIntersectStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count); /// /// Performs a union of multiple sorted sets and stores the result in the destination key. @@ -571,7 +566,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// Optional weights to apply to each sorted set. /// The type of aggregation to perform (e.g., Sum, Min, Max). /// A indicating the status of the operation. - GarnetStatus SortedSetUnionStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count); + GarnetStatus SortedSetUnionStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count); /// /// Sets an expiration time on a sorted set member. @@ -580,7 +575,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus SortedSetExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Sets an expiration time on a sorted set member. @@ -591,7 +586,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The expiration option to apply. /// The results of the operation. /// The status of the operation. - GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results); + GarnetStatus SortedSetExpire(PinnedSpanByte key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results); /// /// Persists the specified sorted set member, removing any expiration time set on it. @@ -600,7 +595,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Persists the specified sorted set members, removing any expiration time set on them. @@ -609,7 +604,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The members to persist. /// The results of the operation. /// The status of the operation. - GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results); + GarnetStatus SortedSetPersist(PinnedSpanByte key, ReadOnlySpan members, out int[] results); /// /// Deletes already expired members from the sorted set. @@ -617,7 +612,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The keys of the sorted set members to check for expiration. /// The input object containing additional parameters. /// The status of the operation. - GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input); + GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input); /// /// Collects expired elements from the sorted set. @@ -630,7 +625,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// The keys of the sorted sets to collect expired elements from. /// The status of the operation. - GarnetStatus SortedSetCollect(ReadOnlySpan keys); + GarnetStatus SortedSetCollect(ReadOnlySpan keys); #endregion @@ -644,7 +639,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetAdd(ArgSlice key, ArgSlice member, out int saddCount); + GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte member, out int saddCount); /// /// Adds the specified members to the set at key. @@ -655,7 +650,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetAdd(ArgSlice key, ArgSlice[] members, out int saddCount); + GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte[] members, out int saddCount); /// /// Adds the specified members to the set at key. @@ -666,7 +661,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetAdd(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes the specified member from the set. @@ -677,7 +672,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetRemove(ArgSlice key, ArgSlice member, out int sremCount); + GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte member, out int sremCount); /// /// Removes the specified members from the set. @@ -688,7 +683,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetRemove(ArgSlice key, ArgSlice[] members, out int sremCount); + GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int sremCount); /// /// Removes the specified members from the set. @@ -699,7 +694,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes and returns one random member from the set at key. @@ -707,7 +702,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetPop(ArgSlice key, out ArgSlice member); + GarnetStatus SetPop(PinnedSpanByte key, out PinnedSpanByte member); /// /// Removes and returns random members from the set at key. @@ -716,7 +711,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetPop(ArgSlice key, int count, out ArgSlice[] members); + GarnetStatus SetPop(PinnedSpanByte key, int count, out PinnedSpanByte[] members); /// /// Removes and returns random members from the set at key. @@ -725,7 +720,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Moves a member from a source set to a destination set. @@ -737,7 +732,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey, ArgSlice member, out int smoveResult); + GarnetStatus SetMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, PinnedSpanByte member, out int smoveResult); /// /// When called with just the key argument, return a random element from the set value stored at key. @@ -750,7 +745,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// This command is equal to SUNION, but instead of returning the resulting set, it is stored in destination. @@ -760,7 +755,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetUnionStore(byte[] key, ArgSlice[] keys, out int count); + GarnetStatus SetUnionStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count); /// /// This command is equal to SINTER, but instead of returning the resulting set, it is stored in destination. @@ -770,7 +765,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count); + GarnetStatus SetIntersectStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count); /// /// This command is equal to SDIFF, but instead of returning the resulting set, it is stored in destination. @@ -780,7 +775,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - public GarnetStatus SetDiffStore(byte[] key, ArgSlice[] keys, out int count); + public GarnetStatus SetDiffStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count); #endregion #region List Methods @@ -795,7 +790,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListPosition(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListPosition(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ListLeftPush ArgSlice version with ObjectOutputHeader output @@ -804,7 +799,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListLeftPush(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus ListLeftPush(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ListLeftPush ArgSlice version, one element @@ -814,7 +809,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// When true the operation is executed only if the key already exists /// - GarnetStatus ListLeftPush(ArgSlice key, ArgSlice element, out int count, bool whenExists = false); + GarnetStatus ListLeftPush(PinnedSpanByte key, PinnedSpanByte element, out int count, bool whenExists = false); /// /// ListLeftPush ArgSlice version for multiple values @@ -824,7 +819,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// When true the operation is executed only if the key already exists /// - GarnetStatus ListLeftPush(ArgSlice key, ArgSlice[] elements, out int count, bool whenExists = false); + GarnetStatus ListLeftPush(PinnedSpanByte key, PinnedSpanByte[] elements, out int count, bool whenExists = false); /// /// ListRightPush ArgSlice version with ObjectOutputHeader output @@ -833,7 +828,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - public GarnetStatus ListRightPush(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + public GarnetStatus ListRightPush(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ListRightPush ArgSlice version, one element @@ -843,7 +838,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// When true the operation is executed only if the key already exists /// - GarnetStatus ListRightPush(ArgSlice key, ArgSlice element, out int count, bool whenExists = false); + GarnetStatus ListRightPush(PinnedSpanByte key, PinnedSpanByte element, out int count, bool whenExists = false); /// /// ListRightPush ArgSlice version for multiple values @@ -853,20 +848,20 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// When true the operation is executed only if the key already exists /// - GarnetStatus ListRightPush(ArgSlice key, ArgSlice[] elements, out int count, bool whenExists = false); + GarnetStatus ListRightPush(PinnedSpanByte key, PinnedSpanByte[] elements, out int count, bool whenExists = false); #endregion #region ListPop Methods /// - /// ListLeftPop ArgSlice version, with GarnetObjectStoreOuput + /// ListLeftPop ArgSlice version, with ObjectOutput /// /// /// /// /// - GarnetStatus ListLeftPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListLeftPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ListLeftPop ArgSlice version, one element @@ -874,7 +869,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListLeftPop(ArgSlice key, out ArgSlice element); + GarnetStatus ListLeftPop(PinnedSpanByte key, out PinnedSpanByte element); /// /// ListLeftPop ArgSlice version for multiple values @@ -883,7 +878,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListLeftPop(ArgSlice key, int count, out ArgSlice[] elements); + GarnetStatus ListLeftPop(PinnedSpanByte key, int count, out PinnedSpanByte[] elements); /// /// ListLeftPop ArgSlice version for multiple keys and values @@ -893,16 +888,16 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// GarnetStatus - GarnetStatus ListLeftPop(ArgSlice[] keys, int count, out ArgSlice key, out ArgSlice[] elements); + GarnetStatus ListLeftPop(PinnedSpanByte[] keys, int count, out PinnedSpanByte key, out PinnedSpanByte[] elements); /// - /// ListRightPop ArgSlice version, with GarnetObjectStoreOutput + /// ListRightPop ArgSlice version, with ObjectOutput /// /// /// /// /// - GarnetStatus ListRightPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListRightPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ListRightPop ArgSlice version, one element @@ -910,7 +905,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListRightPop(ArgSlice key, out ArgSlice element); + GarnetStatus ListRightPop(PinnedSpanByte key, out PinnedSpanByte element); /// /// ListRightPop ArgSlice version for multiple values @@ -919,7 +914,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListRightPop(ArgSlice key, int count, out ArgSlice[] elements); + GarnetStatus ListRightPop(PinnedSpanByte key, int count, out PinnedSpanByte[] elements); /// @@ -930,7 +925,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// GarnetStatus - GarnetStatus ListRightPop(ArgSlice[] keys, int count, out ArgSlice key, out ArgSlice[] elements); + GarnetStatus ListRightPop(PinnedSpanByte[] keys, int count, out PinnedSpanByte key, out PinnedSpanByte[] elements); #endregion @@ -944,7 +939,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// The element being popped and pushed /// GarnetStatus - public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element); + public GarnetStatus ListMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element); /// /// Trim an existing list so it only contains the specified range of elements. @@ -953,7 +948,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - public bool ListTrim(ArgSlice key, int start, int stop); + public bool ListTrim(PinnedSpanByte key, int start, int stop); /// /// Trim an existing list so it only contains the specified range of elements. @@ -961,7 +956,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListTrim(byte[] key, ref ObjectInput input); + GarnetStatus ListTrim(PinnedSpanByte key, ref ObjectInput input); /// /// Inserts a new element in the list stored at key either before or after a value pivot @@ -970,7 +965,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListInsert(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus ListInsert(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Removes the first count occurrences of elements equal to element from the list. @@ -979,7 +974,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus ListRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Sets the list element at index to element. @@ -988,7 +983,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus ListSet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); #endregion @@ -1002,7 +997,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashSet(ArgSlice key, ArgSlice field, ArgSlice value, out int count); + GarnetStatus HashSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, out int count); /// /// Sets the specified fields to their respective values in the hash stored at key. @@ -1011,7 +1006,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashSet(ArgSlice key, (ArgSlice field, ArgSlice value)[] elements, out int count); + GarnetStatus HashSet(PinnedSpanByte key, (PinnedSpanByte field, PinnedSpanByte value)[] elements, out int count); /// /// Sets or updates the values of the specified fields that exist in the hash. @@ -1024,7 +1019,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashSet(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus HashSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Set only if field does not yet exist. If key does not exist, a new key holding a hash is created. @@ -1036,7 +1031,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashSetWhenNotExists(ArgSlice key, ArgSlice field, ArgSlice value, out int count); + GarnetStatus HashSetWhenNotExists(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, out int count); /// /// Removes the specified field from the hash stored at key. @@ -1045,7 +1040,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// Number of fields removed /// - GarnetStatus HashDelete(ArgSlice key, ArgSlice field, out int count); + GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte field, out int count); /// /// Removes the specified fields from the hash stored at key. @@ -1054,7 +1049,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// Number of fields removed /// - GarnetStatus HashDelete(ArgSlice key, ArgSlice[] fields, out int count); + GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte[] fields, out int count); /// /// Removes the specified fields from the hash stored at key. @@ -1063,16 +1058,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashDelete(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); - - /// - /// Increments the number stored at field in the hash key by increment parameter. - /// - /// - /// - /// - /// - GarnetStatus HashIncrement(byte[] key, ArgSlice input, out ObjectOutputHeader output); + GarnetStatus HashDelete(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Increments the number stored at field representing a floating point value @@ -1082,7 +1068,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HashIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Sets an expiration time on a hash field. @@ -1091,7 +1077,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus HashExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Persists the specified hash key, removing any expiration time set on it. @@ -1100,7 +1086,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus HashPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Deletes already expired fields from the hash. @@ -1108,7 +1094,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// The keys of the hash fields to check for expiration. /// The input object containing additional parameters. /// The status of the operation. - GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input); + GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input); #endregion @@ -1122,7 +1108,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus StringSetBit(ArgSlice key, ArgSlice offset, bool bit, out bool previous); + GarnetStatus StringSetBit(PinnedSpanByte key, PinnedSpanByte offset, bool bit, out bool previous); /// /// Sets or clears the bit at offset in the given key. @@ -1133,7 +1119,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus StringSetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus StringSetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Performs a bitwise operations on multiple keys @@ -1142,7 +1128,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOperation bitOp, out long result); + GarnetStatus StringBitOperation(ref StringInput input, BitmapOperation bitOp, out long result); /// /// Perform a bitwise operation between multiple keys @@ -1153,7 +1139,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus StringBitOperation(BitmapOperation bitop, ArgSlice destinationKey, ArgSlice[] keys, out long result); + GarnetStatus StringBitOperation(BitmapOperation bitop, PinnedSpanByte destinationKey, PinnedSpanByte[] keys, out long result); /// /// Performs arbitrary bitfield integer operations on strings. @@ -1163,12 +1149,12 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output); + GarnetStatus StringBitField(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output); /// /// Performs arbitrary bitfield integer operations on strings. /// - GarnetStatus StringBitField(ArgSlice key, List commandArguments, out List result); + GarnetStatus StringBitField(PinnedSpanByte key, List commandArguments, out List result); #endregion #region HyperLogLog Methods @@ -1180,7 +1166,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HyperLogLogAdd(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus HyperLogLogAdd(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Adds all the element arguments to the HyperLogLog data structure stored at the variable name specified as key. @@ -1189,7 +1175,7 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// true if at least 1 HyperLogLog internal register was altered /// - GarnetStatus HyperLogLogAdd(ArgSlice keys, string[] elements, out bool updated); + GarnetStatus HyperLogLogAdd(PinnedSpanByte keys, string[] elements, out bool updated); /// /// Merge multiple HyperLogLog values into a unique value that will approximate the cardinality @@ -1198,7 +1184,135 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// /// - GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool error); + GarnetStatus HyperLogLogMerge(ref StringInput input, out bool error); + + #endregion + + #region RangeIndex + + /// + /// RI.CREATE – create a new RangeIndex backed by a BfTree. + /// + /// Key under which the index is stored. + /// Storage backend type (Disk or Memory). + /// BfTree circular buffer size in bytes. + /// BfTree minimum record size. + /// BfTree maximum record size. + /// BfTree maximum key length. + /// BfTree leaf page size (0 = auto-compute from maxRecordSize). + /// Result code of the operation. + /// Error message if the operation failed. + /// Garnet status. + GarnetStatus RangeIndexCreate(PinnedSpanByte key, byte storageBackend, + ulong cacheSize, uint minRecordSize, uint maxRecordSize, uint maxKeyLen, uint leafPageSize, + out RangeIndexResult result, out ReadOnlySpan errorMsg); + + /// + /// RI.SET – insert or update a field in a RangeIndex. + /// + /// Key of the RangeIndex. + /// Entry key within the BfTree. + /// Entry value. + /// Result code of the operation. + /// Error message if the operation failed. + /// Garnet status. + GarnetStatus RangeIndexSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg); + + /// + /// RI.GET – read a field from a RangeIndex. + /// Writes the value as a RESP bulk string directly into . + /// + /// Key of the RangeIndex. + /// Entry key within the BfTree. + /// Output buffer (typically pointing at the network buffer). + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexGet(PinnedSpanByte key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result); + + /// + /// RI.DEL – delete a field from a RangeIndex. + /// + /// Key of the RangeIndex. + /// Entry key within the BfTree. + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexDel(PinnedSpanByte key, PinnedSpanByte field, + out RangeIndexResult result); + + /// + /// RI.SCAN – scan entries from a RangeIndex starting at a key. + /// Writes the complete RESP array response into . + /// + /// Key of the RangeIndex. + /// Key to start scanning from (inclusive). + /// Maximum number of records to return. + /// Which fields to return (Key, Value, or KeyAndValue). + /// Output buffer for the RESP-formatted response (points at network buffer). + /// Number of records scanned. + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexScan(PinnedSpanByte key, PinnedSpanByte startKey, int count, + BfTreeInterop.ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result); + + /// + /// RI.RANGE – scan entries in [start, end] range from a RangeIndex. + /// Writes the complete RESP array response into . + /// + /// Key of the RangeIndex. + /// Start key (inclusive). + /// End key (inclusive). + /// Which fields to return (Key, Value, or KeyAndValue). + /// Output buffer for the RESP-formatted response (points at network buffer). + /// Number of records scanned. + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexRange(PinnedSpanByte key, PinnedSpanByte startKey, PinnedSpanByte endKey, + BfTreeInterop.ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result); + + /// + /// RI.EXISTS – check whether a key exists and is a RangeIndex. + /// Returns with = true when + /// the key is a RangeIndex, false otherwise (including wrong type or missing key). + /// + /// Key to check. + /// True if the key is a RangeIndex. + /// Garnet status. + GarnetStatus RangeIndexExists(PinnedSpanByte key, out bool exists); + + /// + /// RI.CONFIG – return the configuration of a RangeIndex as individual fields. + /// + /// Key of the RangeIndex. + /// Storage backend type (0=Disk, 1=Memory). + /// BfTree circular buffer size in bytes. + /// BfTree minimum record size. + /// BfTree maximum record size. + /// BfTree maximum key length. + /// BfTree leaf page size. + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexConfig(PinnedSpanByte key, + out byte storageBackend, out ulong cacheSize, out uint minRecordSize, + out uint maxRecordSize, out uint maxKeyLen, out uint leafPageSize, + out RangeIndexResult result); + + /// + /// RI.METRICS – return runtime metrics for a RangeIndex. + /// + /// Key of the RangeIndex. + /// Native pointer of the BfTree instance. + /// True if the tree handle is registered in live indexes. + /// True if the stub has been flushed. + /// True if the stub was recovered from a checkpoint. + /// Result code of the operation. + /// Garnet status. + GarnetStatus RangeIndexMetrics(PinnedSpanByte key, + out nint treeHandle, out bool isLive, out bool isFlushed, out bool isRecovered, + out RangeIndexResult result); #endregion @@ -1206,12 +1320,12 @@ GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref GeoSearch /// /// Adds to (and may create) a vector set with the given parameters. /// - GarnetStatus VectorSetAdd(ArgSlice key, int reduceDims, VectorValueType valueType, ArgSlice value, ArgSlice element, VectorQuantType quantizer, int buildExplorationFactor, ArgSlice attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg); + GarnetStatus VectorSetAdd(PinnedSpanByte key, int reduceDims, VectorValueType valueType, PinnedSpanByte value, PinnedSpanByte element, VectorQuantType quantizer, int buildExplorationFactor, PinnedSpanByte attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg); /// /// Remove a member from a vector set, if it is present and the key exists. /// - GarnetStatus VectorSetRemove(ArgSlice key, ArgSlice element); + GarnetStatus VectorSetRemove(PinnedSpanByte key, PinnedSpanByte element); #endregion } @@ -1224,7 +1338,7 @@ public interface IGarnetReadApi /// /// GET /// - GarnetStatus GET(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus GET(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// GET @@ -1232,7 +1346,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GETForMemoryResult(ArgSlice key, out MemoryResult value); + GarnetStatus GETForMemoryResult(PinnedSpanByte key, out MemoryResult value); /// /// GET @@ -1240,7 +1354,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GET(ArgSlice key, out ArgSlice value); + GarnetStatus GET(PinnedSpanByte key, out PinnedSpanByte value); /// /// GET @@ -1248,7 +1362,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GET(byte[] key, out GarnetObjectStoreOutput value); + GarnetStatus GET(PinnedSpanByte key, out ObjectOutput value); /// /// Finds the longest common subsequence (LCS) between two keys. @@ -1261,7 +1375,7 @@ public interface IGarnetReadApi /// If true, the length of each match is returned. /// The minimum length of a match to be considered. /// The status of the operation. - GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0); + GarnetStatus LCS(PinnedSpanByte key1, PinnedSpanByte key2, ref StringOutput output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0); #endregion #region GETRANGE @@ -1273,7 +1387,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GETRANGE(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus GETRANGE(PinnedSpanByte key, ref StringInput input, ref StringOutput output); #endregion #region TTL @@ -1282,19 +1396,10 @@ public interface IGarnetReadApi /// Returns the remaining time to live in seconds of a key that has a timeout. /// /// The key to return the remaining time to live in the store - /// The store type to operate on. - /// The span to allocate the output of the operation. - /// - GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); - - /// - /// Returns the remaining time to live in milliseconds of a key that has a timeout. - /// - /// The key to return the remaining time to live in the store. - /// The store type to operate on. + /// /// The span to allocate the output of the operation. /// - GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); + GarnetStatus TTL(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); #endregion @@ -1304,19 +1409,10 @@ public interface IGarnetReadApi /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds at which the given key will expire. /// /// The key to get the expiration time for. - /// The type of store to retrieve the key from. - /// The output containing the expiration time. - /// The status of the operation. - GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); - - /// - /// Returns the absolute Unix timestamp (since January 1, 1970) in milliseconds at which the given key will expire. - /// - /// The key to get the expiration time for. - /// The type of store to retrieve the key from. + /// /// The output containing the expiration time. /// The status of the operation. - GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); + GarnetStatus EXPIRETIME(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output); #endregion @@ -1328,7 +1424,7 @@ public interface IGarnetReadApi /// Key /// /// - GarnetStatus SortedSetLength(ArgSlice key, out int zcardCount); + GarnetStatus SortedSetLength(PinnedSpanByte key, out int zcardCount); /// /// Returns the sorted set cardinality (number of elements) of the sorted set @@ -1337,7 +1433,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SortedSetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the specified range of elements in the sorted set stored at key. @@ -1348,7 +1444,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the score of member in the sorted set at key. @@ -1358,7 +1454,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetScore(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetScore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the scores associated with the specified members in the sorted set stored at key. @@ -1368,7 +1464,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetScores(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetScores(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the number of elements in the sorted set at key with a score between min and max. @@ -1378,7 +1474,7 @@ public interface IGarnetReadApi /// Max score /// Number of elements /// - GarnetStatus SortedSetCount(ArgSlice key, ArgSlice minScore, ArgSlice maxScore, out int numElements); + GarnetStatus SortedSetCount(PinnedSpanByte key, PinnedSpanByte minScore, PinnedSpanByte maxScore, out int numElements); /// /// Returns the number of elements in the sorted set at key with a score between min and max. @@ -1387,7 +1483,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetCount(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetCount(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the number of elements in the sorted set with a value between min and max. @@ -1398,7 +1494,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetLengthByValue(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SortedSetLengthByValue(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ZRANK: Returns the rank of member in the sorted set, the scores in the sorted set are ordered from low to high @@ -1408,7 +1504,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetRank(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetRank(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// ZRANK: Returns the rank of member in the sorted set, the scores in the sorted set are ordered from low to high @@ -1419,7 +1515,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetRank(ArgSlice key, ArgSlice member, bool reverse, out long? rank); + GarnetStatus SortedSetRank(PinnedSpanByte key, PinnedSpanByte member, bool reverse, out long? rank); /// /// Returns a random element from the sorted set key. @@ -1428,7 +1524,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the specified range of elements in the sorted set stored at key, using byscore, bylex and rev modifiers. @@ -1445,7 +1541,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetRange(ArgSlice key, ArgSlice min, ArgSlice max, SortedSetOrderOperation sortedSetOrderOperation, out ArgSlice[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default); + GarnetStatus SortedSetRange(PinnedSpanByte key, PinnedSpanByte min, PinnedSpanByte max, SortedSetOrderOperation sortedSetOrderOperation, out PinnedSpanByte[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default); /// /// Computes the difference between the first and all successive sorted sets and returns resulting pairs. @@ -1453,7 +1549,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus SortedSetDifference(ArgSlice[] keys, out SortedSet<(double, byte[])> pairs); + GarnetStatus SortedSetDifference(PinnedSpanByte[] keys, out SortedSet<(double, byte[])> pairs); /// /// Performs a union of multiple sorted sets and stores the result in a dictionary. @@ -1463,7 +1559,7 @@ public interface IGarnetReadApi /// An optional array of doubles representing the weights to apply to each sorted set during the union. /// The type of aggregation to use when combining scores from the sorted sets. Defaults to . /// A indicating the status of the operation. - GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double Element, byte[] Score)> pairs); + GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double Element, byte[] Score)> pairs); /// /// Iterates members of SortedSet key and their associated scores using a cursor, @@ -1475,7 +1571,7 @@ public interface IGarnetReadApi /// Limit number for the response /// The list of items for the response /// - GarnetStatus SortedSetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items); + GarnetStatus SortedSetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items); /// /// Intersects multiple sorted sets and returns the result. @@ -1485,7 +1581,7 @@ public interface IGarnetReadApi /// The type of aggregation to perform. /// The resulting dictionary of intersected elements and their scores. /// A indicating the status of the operation. - GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs); + GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs); /// /// Computes the intersection of multiple sorted sets and counts the elements. @@ -1494,7 +1590,7 @@ public interface IGarnetReadApi /// Optional max count limit /// The count of elements in the intersection /// Operation status - GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count); + GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count); /// /// Returns the time to live for a sorted set members. @@ -1503,7 +1599,7 @@ public interface IGarnetReadApi /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus SortedSetTimeToLive(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the time to live for a sorted set members. @@ -1512,7 +1608,7 @@ public interface IGarnetReadApi /// The members to get the time to live for. /// The output array containing the time to live for each member. /// The status of the operation. - GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn); + GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ReadOnlySpan members, out TimeSpan[] expireIn); #endregion @@ -1527,7 +1623,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus GeoCommands(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// GEORADIUS (read variant): Return the members of a sorted set populated with geospatial data, which are inside the circular area delimited by center and radius. @@ -1541,8 +1637,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, - ref ObjectInput input, ref SpanByteAndMemory output); + GarnetStatus GeoSearchReadOnly(PinnedSpanByte key, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output); #endregion @@ -1554,7 +1649,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus ListLength(ArgSlice key, out int count); + GarnetStatus ListLength(PinnedSpanByte key, out int count); /// /// Gets length of the list, RESP version @@ -1563,7 +1658,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus ListLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus ListLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Gets the specified elements of the list stored at key. @@ -1572,7 +1667,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus ListRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the element at index. @@ -1581,7 +1676,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus ListIndex(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus ListIndex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); #endregion @@ -1593,7 +1688,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// Key /// /// - GarnetStatus SetLength(ArgSlice key, out int count); + GarnetStatus SetLength(PinnedSpanByte key, out int count); /// /// Returns the number of elements of the set. @@ -1602,7 +1697,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus SetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// SMEMBERS key @@ -1610,7 +1705,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetMembers(ArgSlice key, out ArgSlice[] members); + GarnetStatus SetMembers(PinnedSpanByte key, out PinnedSpanByte[] members); /// /// Returns all members of the set at key. @@ -1619,7 +1714,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetMembers(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SetMembers(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns if member is a member of the set stored at key. @@ -1628,15 +1723,16 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetIsMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus SetIsMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns whether each member is a member of the set stored at key. /// /// /// + /// /// - GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] members, out int[] result); + GarnetStatus SetIsMember(PinnedSpanByte key, PinnedSpanByte[] members, out int[] result); /// /// Iterates over the members of the Set with the given key using a cursor, @@ -1648,7 +1744,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items); + GarnetStatus SetScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items); /// /// Returns the members of the set resulting from the union of all the given sets. @@ -1657,7 +1753,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output); + GarnetStatus SetUnion(PinnedSpanByte[] keys, out HashSet output); /// /// Returns the members of the set resulting from the intersection of all the given sets. @@ -1666,7 +1762,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output); + GarnetStatus SetIntersect(PinnedSpanByte[] keys, out HashSet output); /// /// Returns the members of the set resulting from the difference between the first set and all the successive sets. @@ -1674,7 +1770,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus SetDiff(ArgSlice[] keys, out HashSet members); + GarnetStatus SetDiff(PinnedSpanByte[] keys, out HashSet members); /// /// Returns the cardinality of the intersection between multiple sets. @@ -1684,7 +1780,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// Optional limit to stop counting at /// The cardinality of the intersection /// Operation status - GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count); + GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count); #endregion #region Hash Methods @@ -1696,7 +1792,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashGet(ArgSlice key, ArgSlice field, out ArgSlice value); + GarnetStatus HashGet(PinnedSpanByte key, PinnedSpanByte field, out PinnedSpanByte value); /// /// Returns the values associated with the fields in the hash stored at key. @@ -1705,7 +1801,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashGetMultiple(ArgSlice key, ArgSlice[] fields, out ArgSlice[] values); + GarnetStatus HashGetMultiple(PinnedSpanByte key, PinnedSpanByte[] fields, out PinnedSpanByte[] values); /// /// Returns the value associated with field in the hash stored at key. @@ -1714,7 +1810,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// The metadata input for the operation /// /// - GarnetStatus HashGet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashGet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns all fields and values of the hash stored at key. @@ -1723,7 +1819,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// The metadata input for the operation /// /// - GarnetStatus HashGetAll(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashGetAll(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the values associated with the specified fields in the hash stored at key. @@ -1732,7 +1828,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// The metadata input for the operation /// /// - GarnetStatus HashGetMultiple(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashGetMultiple(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns ALL the values in the hash stored at key. @@ -1740,7 +1836,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashGetAll(ArgSlice key, out ArgSlice[] values); + GarnetStatus HashGetAll(PinnedSpanByte key, out PinnedSpanByte[] values); /// /// Returns the number of fields contained in the hash Key @@ -1748,7 +1844,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashLength(ArgSlice key, out int count); + GarnetStatus HashLength(PinnedSpanByte key, out int count); /// ///Returns the string length of the value associated with field in the hash stored at key. If the key or the field do not exist, 0 is returned. @@ -1757,7 +1853,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashStrLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus HashStrLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns the number of fields contained in the hash Key. @@ -1766,7 +1862,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus HashLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns if field is an existing field in the hash stored at key. @@ -1775,7 +1871,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashExists(ArgSlice key, ArgSlice field, out bool exists); + GarnetStatus HashExists(PinnedSpanByte key, PinnedSpanByte field, out bool exists); /// /// Returns if field is an existing field in the hash stored at key. @@ -1784,7 +1880,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashExists(byte[] key, ref ObjectInput input, out ObjectOutputHeader output); + GarnetStatus HashExists(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns count random fields from the hash value. @@ -1794,7 +1890,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashRandomField(ArgSlice key, int count, bool withValues, out ArgSlice[] fields); + GarnetStatus HashRandomField(PinnedSpanByte key, int count, bool withValues, out PinnedSpanByte[] fields); /// /// Returns a random field from the hash value stored at key. @@ -1802,7 +1898,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashRandomField(ArgSlice key, out ArgSlice field); + GarnetStatus HashRandomField(PinnedSpanByte key, out PinnedSpanByte field); /// /// Returns a random field(s) from the hash value stored at key. @@ -1811,7 +1907,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashRandomField(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashRandomField(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns all field names in the hash key. @@ -1820,7 +1916,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashKeys(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashKeys(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Returns all values in the hash key. @@ -1829,7 +1925,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashVals(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashVals(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// /// Iterates fields of Hash key and their associated values using a cursor, @@ -1841,7 +1937,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HashScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items); + GarnetStatus HashScan(PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items); /// /// Returns the time to live for a hash key. @@ -1852,7 +1948,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// The input object containing additional parameters. /// The output object to store the result. /// The status of the operation. - GarnetStatus HashTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput output); + GarnetStatus HashTimeToLive(PinnedSpanByte key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref ObjectOutput output); #endregion @@ -1865,7 +1961,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringGetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus StringGetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Returns the bit value at offset in the key stored. @@ -1874,7 +1970,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, out bool bValue); + GarnetStatus StringGetBit(PinnedSpanByte key, PinnedSpanByte offset, out bool bValue); /// /// Count the number of set bits in a string. @@ -1884,7 +1980,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringBitCount(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus StringBitCount(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Count the number of set bits in a string. @@ -1896,7 +1992,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringBitCount(ArgSlice key, long start, long end, out long result, bool useBitInterval = false); + GarnetStatus StringBitCount(PinnedSpanByte key, long start, long end, out long result, bool useBitInterval = false); /// /// Returns the position of the first bit set to 1 or 0 in a key. @@ -1905,7 +2001,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output); + GarnetStatus StringBitPosition(PinnedSpanByte key, ref StringInput input, ref StringOutput output); /// /// Read-only variant of the StringBitField method. @@ -1915,7 +2011,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output); + GarnetStatus StringBitFieldReadOnly(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output); #endregion @@ -1928,7 +2024,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, out bool error); + GarnetStatus HyperLogLogLength(ref StringInput input, out long count, out bool error); /// /// @@ -1936,7 +2032,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// /// - GarnetStatus HyperLogLogLength(Span keys, out long count); + GarnetStatus HyperLogLogLength(Span keys, out long count); #endregion #region Server Methods @@ -1946,7 +2042,7 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// /// Expression to match the keys name /// - List GetDbKeys(ArgSlice pattern); + List GetDbKeys(PinnedSpanByte pattern); /// /// Gets the number of existing keys in both stores @@ -1965,29 +2061,10 @@ GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, /// The size of the batch of keys /// Type of key to filter out /// - public bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out long storeCursor, out List Keys, long count = 10, ReadOnlySpan type = default); - - /// - /// Iterate the contents of the main store - /// - /// - /// - /// - /// - /// - /// - /// - public bool IterateMainStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions; - - /// - /// Iterate the contents of the main store (pull based) - /// - /// - public ITsavoriteScanIterator IterateMainStore(); + public bool DbScan(PinnedSpanByte patternB, bool allKeys, long cursor, out long storeCursor, out List Keys, long count = 10, ReadOnlySpan type = default); /// - /// Iterate the contents of the object store + /// Iterate the contents of the store /// /// /// @@ -1996,14 +2073,16 @@ public bool IterateMainStore(ref TScanFunctions scanFunctions, r /// /// /// - public bool IterateObjectStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions; + public bool IterateStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions; /// - /// Iterate the contents of the object store (pull based) + /// Delete every live key whose hash slot is in . + /// Uses lookup-based iteration (no tempKv); preserves pull-iterator semantics — + /// every matched live key is deleted, including expired-but-not-yet-tombstoned records. /// - /// - public ITsavoriteScanIterator IterateObjectStore(); + /// Hash slot set to delete. + public void DeleteSlotKeys(HashSet slots); #endregion @@ -2016,20 +2095,12 @@ public bool IterateObjectStore(ref TScanFunctions scanFunctions, /// The key of the sorted set /// /// - GarnetStatus ObjectScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output); - - /// - /// Retrieve the current scratch buffer offset. - /// - /// Current offset - int GetScratchBufferOffset(); + GarnetStatus ObjectScan(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output); /// - /// Resets the scratch buffer to the given offset. + /// Resets the scratch buffer, releasing all allocated slices. /// - /// Offset to reset to - /// True if successful, else false - bool ResetScratchBuffer(int offset); + void ResetScratchBuffer(); #endregion @@ -2041,7 +2112,7 @@ public bool IterateObjectStore(ref TScanFunctions scanFunctions, /// Ids are encoded in as length prefixed blobs of bytes. /// Attributes are encoded in as length prefixed blobs of bytes. /// - GarnetStatus VectorSetValueSimilarity(ArgSlice key, VectorValueType valueType, ArgSlice value, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap); + GarnetStatus VectorSetValueSimilarity(PinnedSpanByte key, VectorValueType valueType, PinnedSpanByte value, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap); /// /// Perform a similarity search given an element already in the vector set and these parameters. @@ -2049,29 +2120,29 @@ public bool IterateObjectStore(ref TScanFunctions scanFunctions, /// Ids are encoded in as length prefixed blobs of bytes. /// Attributes are encoded in as length prefixed blobs of bytes. /// - GarnetStatus VectorSetElementSimilarity(ArgSlice key, ArgSlice element, int count, float delta, int searchExplorationFactor, ArgSlice filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap); + GarnetStatus VectorSetElementSimilarity(PinnedSpanByte key, PinnedSpanByte element, int count, float delta, int searchExplorationFactor, PinnedSpanByte filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap); /// /// Fetch the embedding of a given element in a Vector set. /// - GarnetStatus VectorSetEmbedding(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputDistances); + GarnetStatus VectorSetEmbedding(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputDistances); /// /// Fetch the dimensionality of the given Vector Set. /// /// If the Vector Set was created with reduced dimensions, reports the reduced dimensions. /// - GarnetStatus VectorSetDimensions(ArgSlice key, out int dimensions); + GarnetStatus VectorSetDimensions(PinnedSpanByte key, out int dimensions); /// /// Fetch debugging information about the Vector Set. /// - GarnetStatus VectorSetInfo(ArgSlice key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size); + GarnetStatus VectorSetInfo(PinnedSpanByte key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, out uint reducedDimensions, out uint buildExplorationFactor, out uint numberOfLinks, out long size); /// /// Get the attributes associated with an element in the Vector Set. /// - GarnetStatus VectorSetGetAttribute(ArgSlice key, ArgSlice element, ref SpanByteAndMemory outputAttributes); + GarnetStatus VectorSetGetAttribute(PinnedSpanByte key, PinnedSpanByte element, ref SpanByteAndMemory outputAttributes); #endregion } @@ -2086,13 +2157,6 @@ public interface IGarnetWatchApi /// /// /// - void WATCH(ArgSlice key, StoreType type); - - /// - /// WATCH - /// - /// - /// - void WATCH(byte[] key, StoreType type); + void WATCH(PinnedSpanByte key, StoreType type); } } \ No newline at end of file diff --git a/libs/server/API/SessionApi.cs b/libs/server/API/SessionApi.cs new file mode 100644 index 00000000000..17aca4be888 --- /dev/null +++ b/libs/server/API/SessionApi.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.server +{ + internal sealed unsafe partial class RespServerSession : ServerSessionBase + { + public int DbSize() => basicGarnetApi.GetDbSize(); + } +} \ No newline at end of file diff --git a/libs/server/ArgSlice/ArgSlice.cs b/libs/server/ArgSlice/ArgSlice.cs deleted file mode 100644 index c56e5545ddc..00000000000 --- a/libs/server/ArgSlice/ArgSlice.cs +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Text; -using Tsavorite.core; - -namespace Garnet.server -{ - /// - /// Represents contiguous region of arbitrary _pinned_ memory. - /// - /// - /// SAFETY: This type is used to represent arguments that are assumed to point to pinned memory. - /// - [StructLayout(LayoutKind.Explicit, Size = Size)] - public unsafe struct ArgSlice - { - public const int Size = 12; - - [FieldOffset(0)] - internal byte* ptr; - - [FieldOffset(8)] - internal int length; - - /// - /// Create new ArgSlice from given pointer and length - /// - public ArgSlice(byte* ptr, int length) - { - this.ptr = ptr; - this.length = length; - } - - /// - /// Create new ArgSlice from given SpanByte (without metadata header) - /// - internal ArgSlice(ref SpanByte input) - { - this.ptr = input.ToPointer(); - this.length = input.LengthWithoutMetadata; - } - - /// - /// Get length of ArgSlice - /// - public readonly int Length => length; - - /// - /// Get slice as ReadOnlySpan - /// - public readonly ReadOnlySpan ReadOnlySpan => new(ptr, length); - - /// - /// Get slice as Span - /// - public readonly Span Span => new(ptr, length); - - /// - /// Get slice as SpanByte - /// - public readonly SpanByte SpanByte => new(length, (nint)ptr); - - /// - /// Copies the contents of this slice into a new array. - /// - public readonly byte[] ToArray() => ReadOnlySpan.ToArray(); - - /// - /// Decodes the contents of this slice as ASCII into a new string. - /// - /// A string ASCII decoded string from the slice. - public override readonly string ToString() - => Encoding.ASCII.GetString(ReadOnlySpan); - - /// - /// Create a from the given . - /// - /// - /// SAFETY: The MUST point to pinned memory. - /// - public static ArgSlice FromPinnedSpan(ReadOnlySpan span) - { - return new ArgSlice((byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)), span.Length); - } - - /// - /// Check for equality to the provided argSlice - /// - /// - /// - public readonly bool Equals(ArgSlice argSlice) => argSlice.Span.SequenceEqual(Span); - } -} \ No newline at end of file diff --git a/libs/server/ArgSlice/ArgSliceComparer.cs b/libs/server/ArgSlice/ArgSliceComparer.cs deleted file mode 100644 index 677c2cb6c7d..00000000000 --- a/libs/server/ArgSlice/ArgSliceComparer.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Numerics; - -namespace Garnet.server -{ - /// - /// ArgSlice Comparer - /// - public sealed class ArgSliceComparer : IEqualityComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly ArgSliceComparer Instance = new(); - - /// - public bool Equals(ArgSlice x, ArgSlice y) => x.Equals(y); - - /// - public unsafe int GetHashCode([DisallowNull] ArgSlice obj) - { - fixed (byte* ptr = obj.Span) - { - return (int)HashBytes(ptr, obj.Length); - } - } - - static unsafe long HashBytes(byte* pbString, int len) - { - const long magicno = 40343; - char* pwString = (char*)pbString; - int cbBuf = len / 2; - ulong hashState = (ulong)len; - - for (int i = 0; i < cbBuf; i++, pwString++) - hashState = magicno * hashState + *pwString; - - if ((len & 1) > 0) - { - byte* pC = (byte*)pwString; - hashState = magicno * hashState + *pC; - } - - return (long)BitOperations.RotateRight(magicno * hashState, 4); - } - } -} \ No newline at end of file diff --git a/libs/server/ArgSlice/ArgSliceUtils.cs b/libs/server/ArgSlice/ArgSliceUtils.cs deleted file mode 100644 index b418ef77cad..00000000000 --- a/libs/server/ArgSlice/ArgSliceUtils.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using Garnet.common; - -namespace Garnet.server -{ - /// - /// ArgSlice utils - /// - public static class ArgSliceUtils - { - /// - /// Compute hash slot of given ArgSlice - /// - public static unsafe ushort HashSlot(ref ArgSlice argSlice) - => HashSlotUtils.HashSlot(argSlice.ptr, argSlice.Length); - } -} \ No newline at end of file diff --git a/libs/server/ArgSlice/ArgSliceVector.cs b/libs/server/ArgSlice/ArgSliceVector.cs index 3e1a0778933..e5ceb623783 100644 --- a/libs/server/ArgSlice/ArgSliceVector.cs +++ b/libs/server/ArgSlice/ArgSliceVector.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Buffers.Binary; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -13,7 +14,7 @@ namespace Garnet.server /// Vector of ArgSlices /// /// - public unsafe class ArgSliceVector(int maxItemNum = 1 << 18) : IEnumerable + public unsafe class ArgSliceVector(int maxItemNum = 1 << 18) : IEnumerable<(PinnedSpanByte NamespaceBytes, PinnedSpanByte KeyBytes, bool HasNamespace)> { private bool enumerating; @@ -21,52 +22,48 @@ public unsafe class ArgSliceVector(int maxItemNum = 1 << 18) : IEnumerable items.Count; public bool IsEmpty => items.Count == 0; - readonly List<(int Offset, int Length, bool HasNamespace)> items = []; + readonly List<((int Offset, int Length) Entry, bool HasNamespace)> items = []; /// /// Try to add ArgSlice /// /// /// True if it succeeds to add ArgSlice, false if maxCount has been reached. - public bool TryAddItem(Span item) + public bool TryAddItem(ReadOnlySpan item) { Debug.Assert(!enumerating, "Cannot modify while enumerating"); if (Count + 1 >= maxCount) return false; - var insertLoc = bufferManager.ScratchBufferOffset; - - var sb = bufferManager.CreateArgSlice(item); + var entry = bufferManager.CreateArgSliceAsOffset(item); - items.Add((insertLoc, sb.Length, HasNamespace: false)); + items.Add((entry, false)); return true; } /// /// Try to add ArgSlice /// - /// + /// /// /// True if it succeeds to add ArgSlice, false if maxCount has been reached. - public bool TryAddItem(ulong ns, Span item) + public bool TryAddItem(ReadOnlySpan namespaceBytes, ReadOnlySpan item) { Debug.Assert(!enumerating, "Cannot modify while enumerating"); - Debug.Assert(ns <= byte.MaxValue, "Only byte-size namespaces supported currently"); - if (Count + 1 >= maxCount) return false; var insertLoc = bufferManager.ScratchBufferOffset; - var argSlice = bufferManager.CreateArgSlice(item.Length + 1); - var sb = argSlice.SpanByte; + Span toWrite = stackalloc byte[sizeof(int) + namespaceBytes.Length + sizeof(int) + item.Length]; + BinaryPrimitives.WriteInt32LittleEndian(toWrite, namespaceBytes.Length); + namespaceBytes.CopyTo(toWrite[sizeof(int)..]); + BinaryPrimitives.WriteInt32LittleEndian(toWrite[(sizeof(int) + namespaceBytes.Length)..], item.Length); + item.CopyTo(toWrite[(sizeof(int) + namespaceBytes.Length + sizeof(int))..]); - sb.MarkNamespace(); - sb.SetNamespaceInPayload((byte)ns); - item.CopyTo(sb.AsSpan()); - - items.Add((insertLoc, sb.Length, HasNamespace: true)); + var entry = bufferManager.CreateArgSliceAsOffset(toWrite); + items.Add((entry, true)); return true; } @@ -81,7 +78,8 @@ public void Clear() bufferManager.Reset(); } - public IEnumerator GetEnumerator() + /// + public IEnumerator<(PinnedSpanByte NamespaceBytes, PinnedSpanByte KeyBytes, bool HasNamespace)> GetEnumerator() { Debug.Assert(!enumerating, "Concurrent enumeration is not allwed"); @@ -90,17 +88,27 @@ public IEnumerator GetEnumerator() enumerating = true; try { - foreach (var (offset, length, hasNamespace) in items) + foreach (var ((offset, length), hasNamespace) in items) { - var span = full.ReadOnlySpan.Slice(offset, length); - var ret = SpanByte.FromPinnedSpan(span); - - if (hasNamespace) + if (!hasNamespace) { - ret.MarkNamespace(); + var span = full.ReadOnlySpan.Slice(offset, length); + var ret = PinnedSpanByte.FromPinnedSpan(span); + + yield return (default, ret, false); } + else + { + var span = full.ReadOnlySpan.Slice(offset, length); + + var nsLen = BinaryPrimitives.ReadInt32LittleEndian(span); + var ns = PinnedSpanByte.FromPinnedSpan(span.Slice(sizeof(int), nsLen)); - yield return ret; + var keyLen = BinaryPrimitives.ReadInt32LittleEndian(span[(sizeof(int) + nsLen)..]); + var key = PinnedSpanByte.FromPinnedSpan(span.Slice(sizeof(int) + nsLen + sizeof(int), keyLen)); + + yield return (ns, key, true); + } } } finally @@ -109,6 +117,7 @@ public IEnumerator GetEnumerator() } } + /// IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } \ No newline at end of file diff --git a/libs/server/ArgSlice/ScratchBufferAllocator.cs b/libs/server/ArgSlice/ScratchBufferAllocator.cs index ef4c773d8b5..731d62f3075 100644 --- a/libs/server/ArgSlice/ScratchBufferAllocator.cs +++ b/libs/server/ArgSlice/ScratchBufferAllocator.cs @@ -8,22 +8,23 @@ using System.Runtime.InteropServices; using System.Text; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { /// /// is responsible for allocating sufficient memory and copying data into a buffer - /// and returning an to the caller. + /// and returning an to the caller. /// Whenever the current buffer runs out of space, a new buffer is allocated, without copying the previous buffer data. - /// The previous allocated buffers are kept rooted in a stack by the manager, so that each that wasn't explicitly + /// The previous allocated buffers are kept rooted in a stack by the manager, so that each that wasn't explicitly /// rewound is not going to be GCed. /// /// The manager is meant to be called from a single-threaded context (i.e. one manager per session). /// Each call to CreateArgSlice will copy the data to the current or new buffer that could contain the data in its entirety, - /// so rewinding the (i.e. releasing the memory) should be called in reverse order to assignment. + /// so rewinding the (i.e. releasing the memory) should be called in reverse order to assignment. /// /// Note: Use if you need all data to remain in a continuous chunk of memory (which is not promised by - /// ) and you do not need to reuse previously returned structs + /// ) and you do not need to reuse previously returned structs /// (as consequent allocations may cause them to point to GCed areas in memory). /// internal sealed unsafe class ScratchBufferAllocator @@ -51,12 +52,12 @@ private struct ScratchBuffer /// /// Length of the entire scratch buffer /// - internal int Length => scratchBuffer.Length; + internal readonly int Length => scratchBuffer.Length; /// /// True if buffer was not yet allocated /// - internal bool IsDefault => scratchBuffer == null; + internal readonly bool IsDefault => scratchBuffer == null; /// /// Initializes the scratch buffer to a specified length @@ -99,9 +100,9 @@ internal void Initialize(int length) /// /// Creates an instance of /// - /// Min size that can be allocated for a single buffer (Default: 2) + /// Min size that can be allocated for a single buffer (Default: 64) /// Max size of previously allocated unused buffer to keep upon reset (Default: no limit) - public ScratchBufferAllocator(int minSizeBuffer = 2, int maxInitialCapacity = int.MaxValue) + public ScratchBufferAllocator(int minSizeBuffer = 64, int maxInitialCapacity = int.MaxValue) { this.minSizeBuffer = minSizeBuffer; this.maxInitialCapacity = maxInitialCapacity; @@ -109,7 +110,7 @@ public ScratchBufferAllocator(int minSizeBuffer = 2, int maxInitialCapacity = in /// /// Reset all scratch buffers managed by the . - /// Loses all s created on the scratch buffers. + /// Loses all s created on the scratch buffers. /// public void Reset() { @@ -160,11 +161,11 @@ public void Reset() /// /// Rewind (pop) the last entry of the current scratch buffer (rewinding the current scratch buffer offset), - /// if it contains the given + /// if it contains the given /// - /// The to rewind + /// The to rewind /// True if successful - public bool RewindScratchBuffer(ref ArgSlice slice) + public bool RewindScratchBuffer(ref PinnedSpanByte slice) { if (currScratchBuffer.IsDefault) return false; @@ -205,15 +206,15 @@ public bool RewindScratchBuffer(ref ArgSlice slice) } /// - /// Create an from the given ReadOnlySpan + /// Create an from the given ReadOnlySpan /// /// Input bytes - /// Created - public ArgSlice CreateArgSlice(ReadOnlySpan bytes) + /// Created + public PinnedSpanByte CreateArgSlice(ReadOnlySpan bytes) { ExpandScratchBufferIfNeeded(bytes.Length); - var retVal = new ArgSlice(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, bytes.Length); + var retVal = PinnedSpanByte.FromPinnedPointer(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, bytes.Length); bytes.CopyTo(retVal.Span); currScratchBuffer.scratchBufferOffset += bytes.Length; @@ -222,37 +223,51 @@ public ArgSlice CreateArgSlice(ReadOnlySpan bytes) } /// - /// Create an in UTF8 format from the given string + /// Create an in UTF8 format from the given string /// /// Input string - /// Created - public ArgSlice CreateArgSlice(string str) + /// Created + public PinnedSpanByte CreateArgSlice(string str) { var length = Encoding.UTF8.GetByteCount(str); ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, length); - Encoding.UTF8.GetBytes(str, retVal.Span); + var retVal = PinnedSpanByte.FromPinnedPointer(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, length); + _ = Encoding.UTF8.GetBytes(str, retVal.Span); currScratchBuffer.scratchBufferOffset += length; return retVal; } /// - /// Create an of specified length, leaves contents as is + /// Create an of specified length, leaves contents as is /// /// Length of slice - /// Created - public ArgSlice CreateArgSlice(int length) + /// Created + public PinnedSpanByte CreateArgSlice(int length) { ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, length); + var retVal = PinnedSpanByte.FromPinnedPointer(currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, length); currScratchBuffer.scratchBufferOffset += length; Debug.Assert(currScratchBuffer.scratchBufferOffset <= currScratchBuffer.Length); return retVal; } + /// + /// View remaining scratch space (of specified minimum length) as a . + /// Does NOT move the offset forward. + /// + /// Minimum length of remaining space + /// A covering the remaining space + public PinnedSpanByte ViewRemainingArgSlice(int minLength = 0) + { + ExpandScratchBufferIfNeeded(minLength); + return PinnedSpanByte.FromPinnedPointer( + currScratchBuffer.scratchBufferHead + currScratchBuffer.scratchBufferOffset, + currScratchBuffer.Length - currScratchBuffer.scratchBufferOffset); + } + void ExpandScratchBufferIfNeeded(int requiredLength) { if (currScratchBuffer.IsDefault || requiredLength > currScratchBuffer.Length - currScratchBuffer.scratchBufferOffset) @@ -264,8 +279,8 @@ void ExpandScratchBuffer(int requiredLength) var currLength = currScratchBuffer.IsDefault ? 0 : currScratchBuffer.Length; ScratchBuffer newScratchBuffer = default; - InitializeScratchBuffer(ref newScratchBuffer, requiredLength: Math.Max(minSizeBuffer, requiredLength), - currentLength: currLength); + InitializeScratchBuffer(ref newScratchBuffer, requiredLength, + currentLength: currLength, minSizeBuffer); totalLength += newScratchBuffer.Length; @@ -287,7 +302,7 @@ void ExpandScratchBuffer(int requiredLength) currScratchBuffer = newScratchBuffer; } - private static void InitializeScratchBuffer(ref ScratchBuffer buffer, int requiredLength, int currentLength = 0) + private static void InitializeScratchBuffer(ref ScratchBuffer buffer, int requiredLength, int currentLength, int minSizeBuffer) { // Length of new buffer is: // If there is no current buffer - the closest power of 2 to the data length @@ -296,6 +311,9 @@ private static void InitializeScratchBuffer(ref ScratchBuffer buffer, int requir ? (int)BitOperations.RoundUpToPowerOf2((uint)requiredLength + 1) : (int)BitOperations.RoundUpToPowerOf2((uint)Math.Max(currentLength, requiredLength) + 1); + // Ensure minimum buffer size + newLength = Math.Max(newLength, minSizeBuffer); + buffer.Initialize(newLength); } } diff --git a/libs/server/ArgSlice/ScratchBufferBuilder.cs b/libs/server/ArgSlice/ScratchBufferBuilder.cs index 5decbd1e221..c6622c0e1ea 100644 --- a/libs/server/ArgSlice/ScratchBufferBuilder.cs +++ b/libs/server/ArgSlice/ScratchBufferBuilder.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Text; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -15,15 +16,15 @@ namespace Garnet.server /// is responsible for building a single buffer containing data /// supplied by sequential calls to CreateArgSlice. /// Whenever the current buffer runs out of space, a new buffer is allocated and the previous buffer's data is the copied over. - /// The previous allocated buffers are then potentially GCed so any s returned prior to any calls to + /// The previous allocated buffers are then potentially GCed so any s returned prior to any calls to /// CreateArgSlice may be pointing to non-allocated space. /// /// The builder is meant to be called from a single-threaded context (i.e. one builder per session). /// Each call to CreateArgSlice will copy the data to the current or new buffer that could contain the data in its entirety, - /// so rewinding the (i.e. releasing the memory) should be called in reverse order to assignment. + /// so rewinding the (i.e. releasing the memory) should be called in reverse order to assignment. /// /// Note: Use if you do not need all data to remain in a continuous chunk of memory - /// and you do not want previously returned structs to potentially point to non-allocated memory. + /// and you do not want previously returned structs to potentially point to non-allocated memory. /// public sealed unsafe class ScratchBufferBuilder { @@ -42,6 +43,14 @@ public sealed unsafe class ScratchBufferBuilder /// int scratchBufferOffset; +#if DEBUG + /// + /// Number of outstanding PinnedSpanByte slices that have been created but not rewound. + /// Used to detect unsafe multi-alloc patterns where buffer expansion could invalidate earlier pointers. + /// + int outstandingSlices; +#endif + /// Current offset in scratch buffer internal int ScratchBufferOffset => scratchBufferOffset; @@ -52,7 +61,13 @@ public ScratchBufferBuilder() /// /// Reset scratch buffer - loses all ArgSlice instances created on the scratch buffer /// - public void Reset() => scratchBufferOffset = 0; + public void Reset() + { + scratchBufferOffset = 0; +#if DEBUG + outstandingSlices = 0; +#endif + } /// /// Return the full buffer managed by this . @@ -64,11 +79,14 @@ public Span FullBuffer() /// Rewind (pop) the last entry of scratch buffer (rewinding the current scratch buffer offset), /// if it contains the given ArgSlice /// - public bool RewindScratchBuffer(ref ArgSlice slice) + public bool RewindScratchBuffer(PinnedSpanByte slice) { if (slice.ptr + slice.Length == scratchBufferHead + scratchBufferOffset) { scratchBufferOffset -= slice.Length; +#if DEBUG + outstandingSlices--; +#endif slice = default; // invalidate the given ArgSlice return true; } @@ -76,29 +94,40 @@ public bool RewindScratchBuffer(ref ArgSlice slice) } /// - /// Resets scratch buffer offset to the specified offset. + /// Create an arg slice in scratch buffer, from given ReadOnlySpan, returning the + /// offset and length instead of a PinnedSpanByte. Safe for multiple calls without + /// rewind — use to resolve offsets later. /// - /// Offset to reset to - /// True if successful, else false - public bool ResetScratchBuffer(int offset) + public (int Offset, int Length) CreateArgSliceAsOffset(ReadOnlySpan bytes) { - if (offset < 0 || offset > scratchBufferOffset) - return false; + ExpandScratchBufferIfNeeded(bytes.Length); - scratchBufferOffset = offset; - return true; + var offset = scratchBufferOffset; + var dest = new Span(scratchBufferHead + scratchBufferOffset, bytes.Length); + bytes.CopyTo(dest); + scratchBufferOffset += bytes.Length; + return (offset, bytes.Length); } /// /// Create ArgSlice in scratch buffer, from given ReadOnlySpan /// - public ArgSlice CreateArgSlice(ReadOnlySpan bytes) + public PinnedSpanByte CreateArgSlice(ReadOnlySpan bytes) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder already has an outstanding slice. " + + "Rewind or reset before creating a new one, or use CreateArgSliceAsOffset, " + + "or use ScratchBufferAllocator for slices that must coexist."); +#endif ExpandScratchBufferIfNeeded(bytes.Length); - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, bytes.Length); + var retVal = PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, bytes.Length); bytes.CopyTo(retVal.Span); scratchBufferOffset += bytes.Length; +#if DEBUG + outstandingSlices++; +#endif return retVal; } @@ -114,14 +143,22 @@ public void MoveOffset(int length) /// /// Create ArgSlice in UTF8 format in scratch buffer, from given string /// - public ArgSlice CreateArgSlice(string str) + public PinnedSpanByte CreateArgSlice(string str) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder already has an outstanding slice. " + + "Rewind or reset before creating a new one, or use ScratchBufferAllocator."); +#endif int length = Encoding.UTF8.GetByteCount(str); ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); + var retVal = PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, length); Encoding.UTF8.GetBytes(str, retVal.Span); scratchBufferOffset += length; +#if DEBUG + outstandingSlices++; +#endif return retVal; } @@ -146,57 +183,20 @@ public ReadOnlySpan UTF8EncodeString(string str) return space[..written]; } - /// - /// Create an ArgSlice that includes a header of specified size, followed by RESP Bulk-String formatted versions of the specified ArgSlice values (arg1 and arg2) - /// - public ArgSlice FormatScratchAsResp(int headerSize, ArgSlice arg1, ArgSlice arg2) - { - int length = headerSize + GetRespFormattedStringLength(arg1) + GetRespFormattedStringLength(arg2); - ExpandScratchBufferIfNeeded(length); - - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); - retVal.Span[..headerSize].Clear(); // Clear the header - - byte* ptr = scratchBufferHead + scratchBufferOffset + headerSize; - var success = RespWriteUtils.TryWriteBulkString(arg1.Span, ref ptr, scratchBufferHead + scratchBuffer.Length); - Debug.Assert(success); - success = RespWriteUtils.TryWriteBulkString(arg2.Span, ref ptr, scratchBufferHead + scratchBuffer.Length); - Debug.Assert(success); - - scratchBufferOffset += length; - Debug.Assert(scratchBufferOffset <= scratchBuffer.Length); - return retVal; - } - - /// - /// Create an ArgSlice that includes a header of specified size, followed by RESP Bulk-String formatted versions of the specified ArgSlice value arg1 - /// - public ArgSlice FormatScratchAsResp(int headerSize, ArgSlice arg1) - { - int length = headerSize + GetRespFormattedStringLength(arg1); - ExpandScratchBufferIfNeeded(length); - - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); - retVal.Span[..headerSize].Clear(); // Clear the header - - byte* ptr = scratchBufferHead + scratchBufferOffset + headerSize; - var success = RespWriteUtils.TryWriteBulkString(arg1.Span, ref ptr, scratchBufferHead + scratchBuffer.Length); - Debug.Assert(success); - - scratchBufferOffset += length; - Debug.Assert(scratchBufferOffset <= scratchBuffer.Length); - return retVal; - } - /// /// Create an ArgSlice that includes a header of specified size, followed by the specified ArgSlice (arg) /// - public ArgSlice FormatScratch(int headerSize, ArgSlice arg) + public PinnedSpanByte FormatScratch(int headerSize, PinnedSpanByte arg) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder already has an outstanding slice. " + + "Rewind or reset before creating a new one, or use ScratchBufferAllocator."); +#endif int length = headerSize + arg.Length; ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); + var retVal = PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, length); retVal.Span[..headerSize].Clear(); // Clear the header byte* ptr = scratchBufferHead + scratchBufferOffset + headerSize; @@ -204,47 +204,69 @@ public ArgSlice FormatScratch(int headerSize, ArgSlice arg) scratchBufferOffset += length; Debug.Assert(scratchBufferOffset <= scratchBuffer.Length); +#if DEBUG + outstandingSlices++; +#endif return retVal; } /// /// Create an ArgSlice of specified length, leaves contents as is /// - public ArgSlice CreateArgSlice(int length) + public PinnedSpanByte CreateArgSlice(int length) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder already has an outstanding slice. " + + "Rewind or reset before creating a new one, or use ScratchBufferAllocator."); +#endif ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); + var retVal = PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, length); scratchBufferOffset += length; Debug.Assert(scratchBufferOffset <= scratchBuffer.Length); +#if DEBUG + outstandingSlices++; +#endif return retVal; } /// - /// View remaining scratch space (of specified minimum length) as an ArgSlice - /// Does NOT move the offset forward + /// View remaining scratch space (of specified minimum length) as a PinnedSpanByte. + /// Does NOT move the offset forward. The returned value is an immediate-use view + /// that may be invalidated by any subsequent allocation or expansion — do not store + /// or return it. Use to claim space after writing. /// - /// - public ArgSlice ViewRemainingArgSlice(int minLength = 0) + public PinnedSpanByte ViewRemainingArgSlice(int minLength = 0) { ExpandScratchBufferIfNeeded(minLength); - return new ArgSlice(scratchBufferHead + scratchBufferOffset, scratchBuffer.Length - scratchBufferOffset); + return PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, scratchBuffer.Length - scratchBufferOffset); } - public ArgSlice ViewFullArgSlice() + /// + /// View the full scratch buffer contents (up to current offset) as a PinnedSpanByte. + /// The returned value is an immediate-use view that may be invalidated by any + /// subsequent allocation or expansion — do not store or return it. + /// + public PinnedSpanByte ViewFullArgSlice() { - return new ArgSlice(scratchBufferHead, scratchBufferOffset); + return PinnedSpanByte.FromPinnedPointer(scratchBufferHead, scratchBufferOffset); } /// /// Create an ArgSlice that includes a header of specified size, followed by the specified Memory /// - public ArgSlice FormatScratch(int headerSize, ReadOnlySpan arg) + public PinnedSpanByte FormatScratch(int headerSize, ReadOnlySpan arg) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder already has an outstanding slice. " + + "Rewind or reset before creating a new one, or use ScratchBufferAllocator."); +#endif int length = headerSize + arg.Length; ExpandScratchBufferIfNeeded(length); - var retVal = new ArgSlice(scratchBufferHead + scratchBufferOffset, length); + var retVal = PinnedSpanByte.FromPinnedPointer(scratchBufferHead + scratchBufferOffset, length); retVal.Span[..headerSize].Clear(); // Clear the header byte* ptr = scratchBufferHead + scratchBufferOffset + headerSize; @@ -252,6 +274,9 @@ public ArgSlice FormatScratch(int headerSize, ReadOnlySpan arg) scratchBufferOffset += length; Debug.Assert(scratchBufferOffset <= scratchBuffer.Length); +#if DEBUG + outstandingSlices++; +#endif return retVal; } @@ -314,16 +339,6 @@ public void WriteArgument(ReadOnlySpan arg) scratchBufferOffset = (int)(ptr - scratchBufferHead); } - /// - /// Get length of a RESP Bulk-String formatted version of the specified ArgSlice - /// RESP format: $[size]\r\n[value]\r\n - /// Total size: 1 + [number of digits in the size value] + 2 + [size of value] + 2 - /// - /// - /// - static int GetRespFormattedStringLength(ArgSlice slice) - => 1 + NumUtils.CountDigits(slice.Length) + 2 + slice.Length + 2; - void ExpandScratchBufferIfNeeded(int newLength) { if (scratchBuffer == null || newLength > scratchBuffer.Length - scratchBufferOffset) @@ -332,6 +347,13 @@ void ExpandScratchBufferIfNeeded(int newLength) void ExpandScratchBuffer(int newLength, int? copyLengthOverride = null) { +#if DEBUG + Debug.Assert(outstandingSlices == 0, + "ScratchBufferBuilder is expanding with outstanding slices. " + + "Previously returned PinnedSpanByte values will be invalidated. " + + "Use ScratchBufferAllocator for slices that must remain valid across allocations, " + + "or use a single CreateArgSlice and partition the buffer manually."); +#endif if (newLength < 64) newLength = 64; else newLength = (int)BitOperations.RoundUpToPowerOf2((uint)newLength + 1); @@ -347,20 +369,6 @@ void ExpandScratchBuffer(int newLength, int? copyLengthOverride = null) scratchBufferHead = _scratchBufferHead; } - /// - /// Returns a new - /// with the bytes of the buffer; - /// these are the most recently added bytes. - /// - /// Length for the new slice - /// This is called by functions that add multiple items to the buffer, - /// after all items have been added and all reallocations have been done. - /// - public ArgSlice GetSliceFromTail(int length) - { - return new ArgSlice(scratchBufferHead + scratchBufferOffset - length, length); - } - /// /// Force backing buffer to grow. /// diff --git a/libs/server/ByteArrayWrapper.cs b/libs/server/ByteArrayWrapper.cs index 749f4fe53d1..c48034b6439 100644 --- a/libs/server/ByteArrayWrapper.cs +++ b/libs/server/ByteArrayWrapper.cs @@ -3,6 +3,7 @@ using System; using System.Numerics; +using Tsavorite.core; namespace Garnet.server { @@ -12,18 +13,16 @@ namespace Garnet.server public readonly struct ByteArrayWrapper { readonly byte[] arrBytes; - readonly ArgSlice arrSlice; + readonly PinnedSpanByte arrSlice; internal ByteArrayWrapper(byte[] arrBytes, bool isPinned = false) { this.arrBytes = arrBytes; if (isPinned) - { - this.arrSlice = ArgSlice.FromPinnedSpan(arrBytes); - } + this.arrSlice = PinnedSpanByte.FromPinnedSpan(arrBytes); } - internal ByteArrayWrapper(ArgSlice arrSlice) + internal ByteArrayWrapper(PinnedSpanByte arrSlice) { this.arrSlice = arrSlice; } @@ -36,22 +35,16 @@ public static ByteArrayWrapper CopyFrom(ReadOnlySpan bytes, bool usePinned } public unsafe ReadOnlySpan ReadOnlySpan - => arrSlice.ptr == null ? new ReadOnlySpan(arrBytes) : arrSlice.ReadOnlySpan; + => arrSlice.IsValid ? arrSlice.ReadOnlySpan : new ReadOnlySpan(arrBytes); /// public override unsafe int GetHashCode() { - if (arrSlice.ptr == null) - { - fixed (byte* k = arrBytes) - { - return (int)HashBytes(k, arrBytes.Length); - } - } - else - { - return (int)HashBytes(arrSlice.ptr, arrSlice.length); - } + if (arrSlice.IsValid) + return (int)HashBytes(arrSlice.ToPointer(), arrSlice.Length); + + fixed (byte* k = arrBytes) + return (int)HashBytes(k, arrBytes.Length); } static unsafe long HashBytes(byte* pbString, int len) diff --git a/libs/server/Cluster/CheckpointMetadata.cs b/libs/server/Cluster/CheckpointMetadata.cs index 912e10462c4..a15cccdd692 100644 --- a/libs/server/Cluster/CheckpointMetadata.cs +++ b/libs/server/Cluster/CheckpointMetadata.cs @@ -10,28 +10,25 @@ public sealed class CheckpointMetadata public long storeVersion; public Guid storeHlogToken; public Guid storeIndexToken; - public long storeCheckpointCoveredAofAddress; + public AofAddress storeCheckpointCoveredAofAddress; public string storePrimaryReplId; - public long objectStoreVersion; - public Guid objectStoreHlogToken; - public Guid objectStoreIndexToken; - public long objectCheckpointCoveredAofAddress; - public string objectStorePrimaryReplId; - public CheckpointMetadata() { storeVersion = -1; storeHlogToken = default; storeIndexToken = default; - storeCheckpointCoveredAofAddress = long.MaxValue; + storeCheckpointCoveredAofAddress = default; storePrimaryReplId = null; + } - objectStoreVersion = -1; - objectStoreHlogToken = default; - objectStoreIndexToken = default; - objectCheckpointCoveredAofAddress = long.MaxValue; - objectStorePrimaryReplId = null; + public CheckpointMetadata(int sublogCount) + { + storeVersion = -1; + storeHlogToken = default; + storeIndexToken = default; + storeCheckpointCoveredAofAddress = AofAddress.Create(sublogCount, 0); + storePrimaryReplId = null; } /// @@ -45,12 +42,7 @@ public override string ToString() $"storeHlogToken={storeHlogToken}," + $"storeIndexToken={storeIndexToken}," + $"storeCheckpointCoveredAofAddress={storeCheckpointCoveredAofAddress}," + - $"storePrimaryReplId={storePrimaryReplId ?? "(empty)"}," + - $"objectStoreVersion={objectStoreVersion}," + - $"objectStoreHlogToken={objectStoreHlogToken}," + - $"objectStoreIndexToken={objectStoreIndexToken}," + - $"objectCheckpointCoveredAofAddress={objectCheckpointCoveredAofAddress}," + - $"objectStorePrimaryReplId={objectStorePrimaryReplId ?? "(empty)"}"; + $"storePrimaryReplId={storePrimaryReplId ?? "(empty)"}"; } } } \ No newline at end of file diff --git a/libs/server/Cluster/ClusterSlotVerificationInput.cs b/libs/server/Cluster/ClusterSlotVerificationInput.cs index 9ccf6d4b315..d39c26dd35e 100644 --- a/libs/server/Cluster/ClusterSlotVerificationInput.cs +++ b/libs/server/Cluster/ClusterSlotVerificationInput.cs @@ -16,24 +16,14 @@ public struct ClusterSlotVerificationInput public byte sessionAsking; /// - /// Offset of first key in the ArgSlice buffer + /// Simplified key specifications for extracting key positions from the command's parse state /// - public int firstKey; + public SimpleRespKeySpec[] keySpecs; /// - /// Offset of the last key in the ArgSlice buffer + /// Whether the command is a sub-command (affects key index offset calculation) /// - public int lastKey; - - /// - /// The step, or increment, between the first key and the position of the next key - /// - public int step; - - /// - /// Offset of key num if any - /// - public int keyNumOffset; + public bool isSubCommand; /// /// If the command being executed requires a slot be STABLE for executing. diff --git a/libs/server/Cluster/IClusterFactory.cs b/libs/server/Cluster/IClusterFactory.cs index d0496596f1d..6e39c11e222 100644 --- a/libs/server/Cluster/IClusterFactory.cs +++ b/libs/server/Cluster/IClusterFactory.cs @@ -14,7 +14,7 @@ public interface IClusterFactory /// /// Create checkpoint manager /// - DeviceLogCommitCheckpointManager CreateCheckpointManager(INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default); + DeviceLogCommitCheckpointManager CreateCheckpointManager(int aofSublogCount, INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool isMainStore, ILogger logger = default); /// /// Create cluster provider diff --git a/libs/server/Cluster/IClusterProvider.cs b/libs/server/Cluster/IClusterProvider.cs index 3b2253fad18..975adb210cb 100644 --- a/libs/server/Cluster/IClusterProvider.cs +++ b/libs/server/Cluster/IClusterProvider.cs @@ -9,37 +9,18 @@ using Garnet.server.ACL; using Garnet.server.Auth; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.server { - using BasicContext = BasicContext, - SpanByteAllocator>>; - - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - - using VectorContext = BasicContext, SpanByteAllocator>>; - /// /// Cluster provider /// public interface IClusterProvider : IDisposable { - // TODO: I really hate having to pass Vector and Basic contexts here... cleanup - /// /// Create cluster session /// - IClusterSession CreateClusterSession(TransactionManager txnManager, IGarnetAuthenticator authenticator, UserHandle userHandle, GarnetSessionMetrics garnetSessionMetrics, BasicGarnetApi basicGarnetApi, BasicContext basicContext, VectorContext vectorContext, INetworkSender networkSender, ILogger logger = null); + IClusterSession CreateClusterSession(TransactionManager txnManager, IGarnetAuthenticator authenticator, UserHandle userHandle, GarnetSessionMetrics garnetSessionMetrics, BasicGarnetApi basicGarnetApi, StringBasicContext basicContext, VectorBasicContext vectorContext, INetworkSender networkSender, ILogger logger = null); /// @@ -72,7 +53,7 @@ public interface IClusterProvider : IDisposable /// Get info on primary from replica perspective. /// /// - (long replication_offset, List replicaInfo) GetPrimaryInfo(); + (AofAddress replication_offset, List replicaInfo) GetPrimaryInfo(); /// /// Get info on replicas from primary perspective. @@ -86,14 +67,6 @@ public interface IClusterProvider : IDisposable /// void PurgeBufferPool(ManagerType managerType); - /// - /// Extract key specs - /// - /// - /// - /// - void ExtractKeySpecs(RespCommandsInfo commandInfo, RespCommand cmd, ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi); - /// /// Issue a cluster publish message to remote nodes /// @@ -123,7 +96,7 @@ public interface IClusterProvider : IDisposable /// On checkpoint initiated /// /// - void OnCheckpointInitiated(out long CheckpointCoveredAofAddress); + void OnCheckpointInitiated(ref AofAddress CheckpointCoveredAofAddress); /// /// Recover the cluster @@ -138,13 +111,13 @@ public interface IClusterProvider : IDisposable /// /// Safe truncate AOF /// - void SafeTruncateAOF(bool full, long CheckpointCoveredAofAddress, Guid storeCheckpointToken, Guid objectStoreCheckpointToken); + void AddNewCheckpointEntry(bool full, AofAddress CheckpointCoveredAofAddress, Guid storeCheckpointToken, Guid objectStoreCheckpointToken); /// /// Safe truncate AOF until address /// /// - void SafeTruncateAOF(long truncateUntil); + void SafeTruncateAOF(in AofAddress truncateUntil); /// /// Start cluster operations diff --git a/libs/server/Cluster/IClusterSession.cs b/libs/server/Cluster/IClusterSession.cs index f42e5b96490..7f1929bb71b 100644 --- a/libs/server/Cluster/IClusterSession.cs +++ b/libs/server/Cluster/IClusterSession.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using Garnet.common; using Garnet.server.ACL; +using Tsavorite.core; namespace Garnet.server { @@ -23,7 +23,7 @@ public interface IClusterSession bool ReadWriteSession { get; } /// - /// If the current session has seen an APPENDLOG command. + /// If the current session is part of an active replication stream (set on first APPENDLOG, including the init handshake). /// bool IsReplicating { get; } @@ -76,8 +76,9 @@ public interface IClusterSession /// /// /// + /// /// - bool NetworkIterativeSlotVerify(ArgSlice keySlice, bool readOnly, byte SessionAsking, bool waitForStableSlot); + bool NetworkIterativeSlotVerify(PinnedSpanByte keySlice, bool readOnly, byte SessionAsking, bool waitForStableSlot); /// /// Write cached slot verification message to output @@ -85,11 +86,6 @@ public interface IClusterSession /// public void WriteCachedSlotVerificationMessage(ref MemoryResult output); - /// - /// Key array slot verify (write result to network) - /// - unsafe bool NetworkKeyArraySlotVerify(Span keys, bool readOnly, byte SessionAsking, bool waitForStableSlot, ref byte* dcurr, ref byte* dend, int count = -1); - /// /// Array slot verify (write result to network) /// @@ -97,8 +93,9 @@ public interface IClusterSession /// /// /// + /// /// - unsafe bool NetworkMultiKeySlotVerify(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend); + unsafe bool NetworkMultiKeySlotVerify(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend, bool isTxn = false); /// /// Array slot verify with no response @@ -107,12 +104,35 @@ public interface IClusterSession /// /// /// + /// /// - unsafe bool NetworkMultiKeySlotVerifyNoResponse(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend); + unsafe bool NetworkMultiKeySlotVerifyNoResponse(ref SessionParseState parseState, ref ClusterSlotVerificationInput csvi, ref byte* dcurr, ref byte* dend, bool isTxn = false); /// /// Sets the currently authenticated in this session (used for permission checks) /// void SetUserHandle(UserHandle userHandle); + + /// + /// NOTE: Unsafe! DO NOT USE, other than benchmarking + /// + /// + void UnsafeSetConfig(string replicaOf); + + /// + /// Dispose + /// + void Dispose(); + + /// + /// NOTE: Used for micro-benchmark + /// + /// + /// + /// + /// + /// + /// + unsafe void ProcessPrimaryStream(int physicalSublogIdx, byte* record, int recordLength, long previousAddress, long currentAddress, long nextAddress); } } \ No newline at end of file diff --git a/libs/server/Cluster/RoleInfo.cs b/libs/server/Cluster/RoleInfo.cs index ef605520779..b70d7e71113 100644 --- a/libs/server/Cluster/RoleInfo.cs +++ b/libs/server/Cluster/RoleInfo.cs @@ -8,12 +8,17 @@ public struct RoleInfo /// /// Replication offset using string store. /// - public long replication_offset; + public AofAddress replication_offset; + + /// + /// Max send timemstamp + /// + public AofAddress sequenceNumber; /// /// Replication offset lag. /// - public long replication_lag; + public AofAddress replication_lag; /// /// Replication state. @@ -38,7 +43,7 @@ public struct RoleInfo /// string public override readonly string ToString() { - return $"ip={address},port={port},state={replication_state},offset={replication_offset},lag={replication_lag}"; + return $"ip={address},port={port},state={replication_state},offset={replication_offset},lag={replication_lag},sequenceNumber={sequenceNumber}"; } } } \ No newline at end of file diff --git a/libs/server/Custom/CustomCommandUtils.cs b/libs/server/Custom/CustomCommandUtils.cs index 6e9e8e18a84..d30dc33c91b 100644 --- a/libs/server/Custom/CustomCommandUtils.cs +++ b/libs/server/Custom/CustomCommandUtils.cs @@ -23,7 +23,7 @@ public static ReadOnlySpan GetFirstArg(ref ObjectInput input) /// /// Main store input /// - public static ReadOnlySpan GetFirstArg(ref RawStringInput input) + public static ReadOnlySpan GetFirstArg(ref StringInput input) { var idx = 0; return GetNextArg(ref input, ref idx); @@ -50,7 +50,7 @@ public static ReadOnlySpan GetNextArg(ref ObjectInput input, scoped ref in /// Main store input /// Current argument index in input /// Argument as a span - public static ReadOnlySpan GetNextArg(ref RawStringInput input, scoped ref int idx) + public static ReadOnlySpan GetNextArg(ref StringInput input, scoped ref int idx) { var arg = idx < input.parseState.Count ? input.parseState.GetArgSliceByRef(idx).ReadOnlySpan diff --git a/libs/server/Custom/CustomObjectBase.cs b/libs/server/Custom/CustomObjectBase.cs index a659ad4418f..06071eba0c2 100644 --- a/libs/server/Custom/CustomObjectBase.cs +++ b/libs/server/Custom/CustomObjectBase.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; +using Tsavorite.core; namespace Garnet.server { @@ -18,15 +19,21 @@ public abstract class CustomObjectBase : GarnetObjectBase /// Base constructor /// /// Object type - /// - protected CustomObjectBase(byte type, long expiration, long size = 0) - : base(expiration, size) + /// + protected CustomObjectBase(byte type, long heapMemorySize = 0) + : base(heapMemorySize) { this.type = type; } - protected CustomObjectBase(byte type, BinaryReader reader, long size = 0) - : base(reader, size) + /// + /// Base constructor + /// + /// Object type + /// + /// + protected CustomObjectBase(byte type, BinaryReader reader, long heapMemorySize = 0) + : base(reader, heapMemorySize) { this.type = type; } @@ -35,7 +42,7 @@ protected CustomObjectBase(byte type, BinaryReader reader, long size = 0) /// Base copy constructor /// /// Other object - protected CustomObjectBase(CustomObjectBase obj) : this(obj.type, obj.Expiration, obj.Size) { } + protected CustomObjectBase(CustomObjectBase obj) : this(obj.type) { } /// public override byte Type => type; @@ -54,7 +61,7 @@ protected CustomObjectBase(CustomObjectBase obj) : this(obj.type, obj.Expiration /// Clone object (shallow copy) /// /// - public sealed override GarnetObjectBase Clone() => CloneObject(); + public sealed override IHeapObject Clone() => CloneObject(); /// public sealed override void DoSerialize(BinaryWriter writer) @@ -67,11 +74,9 @@ public sealed override void DoSerialize(BinaryWriter writer) public abstract override void Dispose(); /// - public sealed override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, - byte respProtocolVersion, out long sizeChange) + public sealed override bool Operate(ref ObjectInput input, ref ObjectOutput output, + byte respProtocolVersion) { - sizeChange = 0; - switch (input.header.cmd) { // Scan Command @@ -82,7 +87,7 @@ public sealed override bool Operate(ref ObjectInput input, ref GarnetObjectStore if ((byte)input.header.type != this.type) { // Indicates an incorrect type of key - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; output.SpanByteAndMemory.Length = 0; return true; } diff --git a/libs/server/Custom/CustomObjectFunctions.cs b/libs/server/Custom/CustomObjectFunctions.cs index d41758ded41..5ef0f12fcde 100644 --- a/libs/server/Custom/CustomObjectFunctions.cs +++ b/libs/server/Custom/CustomObjectFunctions.cs @@ -40,7 +40,7 @@ public abstract class CustomObjectFunctions /// Input /// Output /// True if an initial update is needed, otherwise false - public virtual bool NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput input, ref RespMemoryWriter writer) => throw new NotImplementedException(); + public virtual bool NeedInitialUpdate(scoped ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) => throw new NotImplementedException(); /// /// Create initial value, given key and input. Optionally generate output for command. @@ -51,7 +51,7 @@ public abstract class CustomObjectFunctions /// Output /// Advanced arguments /// True if done, false if we need to cancel the update - public virtual bool InitialUpdater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => Updater(key, ref input, value, ref writer, ref rmwInfo); + public virtual bool InitialUpdater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => Updater(key, ref input, value, ref writer, ref rmwInfo); /// /// Update given value in place, given key and input. Optionally generate output for command. @@ -62,7 +62,7 @@ public abstract class CustomObjectFunctions /// Output /// Advanced arguments /// True if done, false if we have no space to update in place - public virtual bool Updater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); + public virtual bool Updater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); /// /// Read value, given key and input and generate output for command. @@ -73,7 +73,15 @@ public abstract class CustomObjectFunctions /// Output /// Advanced arguments /// True if done, false if not found - public virtual bool Reader(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new NotImplementedException(); + public virtual bool Reader(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new NotImplementedException(); + + /// + /// Called when a read command does not find a value. + /// + /// Default implementation writes a null value (either _\r\n or $-1\r\n for RESP 3 or 2 respectively). + /// + public virtual void NotFound(ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) + => writer.WriteNull(); /// /// Aborts the execution of the current object store command and outputs diff --git a/libs/server/Custom/CustomProcedureBase.cs b/libs/server/Custom/CustomProcedureBase.cs index 5e18e42971c..04fdf519cf4 100644 --- a/libs/server/Custom/CustomProcedureBase.cs +++ b/libs/server/Custom/CustomProcedureBase.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Diagnostics; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -21,6 +22,11 @@ public abstract class CustomProcedureBase internal RespServerSession respServerSession; + /// + /// Keep track which keys have been updated to update ReplicaTimestampTracker after CustomProc completes + /// + internal CustomProcedureKeyHashCollection customProcKeyHashCollection; + /// /// Create output as simple string, from given string /// @@ -63,7 +69,7 @@ protected static unsafe void WriteSimpleString(ref MemoryResult output, Re /// /// Create output as an array of bulk strings, from given array of ArgSlice values /// - protected static unsafe void WriteBulkStringArray(ref MemoryResult output, params ArgSlice[] values) + protected static unsafe void WriteBulkStringArray(ref MemoryResult output, params PinnedSpanByte[] values) { var totalLen = 1 + NumUtils.CountDigits(values.Length) + 2; for (var i = 0; i < values.Length; i++) @@ -91,7 +97,7 @@ protected static unsafe void WriteBulkStringArray(ref MemoryResult output, /// /// Create output as an array of bulk strings, from given array of ArgSlice values /// - protected static unsafe void WriteBulkStringArray(ref MemoryResult output, List values) + protected static unsafe void WriteBulkStringArray(ref MemoryResult output, List values) { var totalLen = 1 + NumUtils.CountDigits(values.Count) + 2; for (var i = 0; i < values.Count; i++) @@ -201,7 +207,7 @@ protected static unsafe void WriteError(ref MemoryResult output, ReadOnlyS /// Current parse state /// Current argument index in parse state /// Argument as a span - protected static unsafe ArgSlice GetNextArg(ref SessionParseState parseState, ref int idx) + protected static unsafe PinnedSpanByte GetNextArg(ref SessionParseState parseState, ref int idx) { var arg = idx < parseState.Count ? parseState.GetArgSliceByRef(idx) @@ -216,7 +222,7 @@ protected static unsafe ArgSlice GetNextArg(ref SessionParseState parseState, re /// Procedure input /// Current argument index in parse state /// Argument as a span - protected static unsafe ArgSlice GetNextArg(ref CustomProcedureInput procInput, ref int idx) + protected static unsafe PinnedSpanByte GetNextArg(ref CustomProcedureInput procInput, ref int idx) { return GetNextArg(ref procInput.parseState, ref idx); } @@ -243,7 +249,7 @@ protected bool ParseCustomObjectCommand(string cmd, out CustomObjectCommand obje /// Args to the command /// Output from the command /// True if successful - protected bool ExecuteCustomRawStringCommand(TGarnetApi garnetApi, CustomRawStringCommand rawStringCommand, ArgSlice key, ArgSlice[] input, out ArgSlice output) + protected bool ExecuteCustomRawStringCommand(TGarnetApi garnetApi, CustomRawStringCommand rawStringCommand, PinnedSpanByte key, PinnedSpanByte[] input, out PinnedSpanByte output) where TGarnetApi : IGarnetApi { return respServerSession.InvokeCustomRawStringCommand(ref garnetApi, rawStringCommand, key, input, out output); @@ -257,7 +263,7 @@ protected bool ExecuteCustomRawStringCommand(TGarnetApi garnetApi, C /// Args to the command /// Output from the command /// True if successful - protected bool ExecuteCustomObjectCommand(TGarnetApi garnetApi, CustomObjectCommand objectCommand, ArgSlice key, ArgSlice[] input, out ArgSlice output) + protected bool ExecuteCustomObjectCommand(TGarnetApi garnetApi, CustomObjectCommand objectCommand, PinnedSpanByte key, PinnedSpanByte[] input, out PinnedSpanByte output) where TGarnetApi : IGarnetApi { return respServerSession.InvokeCustomObjectCommand(ref garnetApi, objectCommand, key, input, out output); diff --git a/libs/server/Custom/CustomRawStringFunctions.cs b/libs/server/Custom/CustomRawStringFunctions.cs index a1ccd06298d..0bfd37efc53 100644 --- a/libs/server/Custom/CustomRawStringFunctions.cs +++ b/libs/server/Custom/CustomRawStringFunctions.cs @@ -18,7 +18,7 @@ public abstract class CustomRawStringFunctions /// Input as ReadOnlySpan of byte /// Current offset into input /// Argument as a span - protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, scoped ref int offset) => + protected static unsafe ReadOnlySpan GetNextArg(ref StringInput input, scoped ref int offset) => CustomCommandUtils.GetNextArg(ref input, ref offset); /// @@ -26,7 +26,7 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// /// /// - protected static ReadOnlySpan GetFirstArg(ref RawStringInput input) => CustomCommandUtils.GetFirstArg(ref input); + protected static ReadOnlySpan GetFirstArg(ref StringInput input) => CustomCommandUtils.GetFirstArg(ref input); /// /// Whether we need an initial update, given input, if item does not already exist in store @@ -34,7 +34,7 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Key /// Input /// Output - public virtual bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref RespMemoryWriter writer) => true; + public virtual bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) => true; /// /// Whether we need to need to perform an update, given old value and input @@ -43,21 +43,21 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Input /// Old value /// Output - public virtual bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) => true; + public virtual bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) => true; /// /// Length of initial value, given input /// /// Input /// - public abstract int GetInitialLength(ref RawStringInput input); + public abstract int GetInitialLength(ref StringInput input); /// /// Length of updated value, given old value and input /// /// Old value /// Input - public abstract int GetLength(ReadOnlySpan value, ref RawStringInput input); + public abstract int GetLength(ReadOnlySpan value, ref StringInput input); /// /// Create initial value, given key and input. Optionally generate output for command. @@ -68,7 +68,7 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Output /// Advanced arguments /// True if done, false if we need to cancel the update - public abstract bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); + public abstract bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); /// /// Update given value in place, given key and input. Optionally generate output for command. @@ -80,7 +80,7 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Output /// Advanced arguments /// True if done, false if we have no space to update in place - public abstract bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); + public abstract bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); /// /// Update to new value in new location, given key, input, and old value. Optionally generate output for command. @@ -92,7 +92,7 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Output /// Advanced arguments /// True if done, false if we have no space to update in place - public abstract bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); + public abstract bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo); /// /// Read value, given key and input and generate output for command. @@ -103,6 +103,15 @@ protected static unsafe ReadOnlySpan GetNextArg(ref RawStringInput input, /// Output /// Advanced arguments /// True if done, false if not found - public abstract bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo); + public abstract bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo); + + + /// + /// Called when a read command does not find a value. + /// + /// Default implementation writes a null value (either _\r\n or $-1\r\n for RESP 3 or 2 respectively). + /// + public virtual void NotFound(ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) + => writer.WriteNull(); } } \ No newline at end of file diff --git a/libs/server/Custom/CustomRespCommands.cs b/libs/server/Custom/CustomRespCommands.cs index 3b97d3c5e5a..b61f003d882 100644 --- a/libs/server/Custom/CustomRespCommands.cs +++ b/libs/server/Custom/CustomRespCommands.cs @@ -53,11 +53,12 @@ private bool TryTransactionProc(byte id, CustomTransactionProcedure proc, int st return true; } - public bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, ref MemoryResult output, bool isReplaying) + public bool RunCustomTxnProcAtReplica(byte id, ref CustomProcedureInput procInput, ref MemoryResult output, bool isRecovering = false, CustomProcedureKeyHashCollection customProcTimestampBitmap = null) { var proc = customCommandManagerSession .GetCustomTransactionProcedure(id, this, txnManager, scratchBufferAllocator, out _); - return txnManager.RunTransactionProc(id, ref procInput, proc, ref output, isReplaying); + proc.customProcKeyHashCollection = customProcTimestampBitmap; + return txnManager.RunTransactionProc(id, ref procInput, proc, ref output, isRecovering); } private void TryCustomProcedure(CustomProcedure proc, int startIdx = 0) @@ -88,44 +89,63 @@ private void TryCustomProcedure(CustomProcedure proc, int startIdx = 0) /// /// Custom command /// - private bool TryCustomRawStringCommand(RespCommand cmd, long expirationTicks, CommandType type, ref TGarnetApi storageApi) + private bool TryCustomRawStringCommand(RespCommand cmd, CustomRawStringCommand customRawStringCommand, ref TGarnetApi storageApi) where TGarnetApi : IGarnetAdvancedApi { - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); + var expirationTicks = customRawStringCommand.expirationTicks; var inputArg = expirationTicks > 0 ? DateTimeOffset.UtcNow.Ticks + expirationTicks : expirationTicks; - var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); + var input = new StringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); - var output = new SpanByteAndMemory(null); - GarnetStatus status; - if (type == CommandType.ReadModifyWrite) + var output = new StringOutput(); + if (customRawStringCommand.type == CommandType.ReadModifyWrite) { - status = storageApi.RMW_MainStore(ref sbKey, ref input, ref output); - Debug.Assert(!output.IsSpanByte); + _ = storageApi.RMW_MainStore(key, ref input, ref output); + Debug.Assert(!output.SpanByteAndMemory.IsSpanByte); - if (output.Memory != null) - SendAndReset(output.Memory, output.Length); + if (output.SpanByteAndMemory.Memory != null) + SendAndReset(output.SpanByteAndMemory.Memory, output.SpanByteAndMemory.Length); else while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); } else { - status = storageApi.Read_MainStore(ref sbKey, ref input, ref output); - Debug.Assert(!output.IsSpanByte); + var status = storageApi.Read_MainStore(key, ref input, ref output); + Debug.Assert(!output.SpanByteAndMemory.IsSpanByte); if (status == GarnetStatus.OK) { - if (output.Memory != null) - SendAndReset(output.Memory, output.Length); + if (output.SpanByteAndMemory.Memory != null) + SendAndReset(output.SpanByteAndMemory.Memory, output.SpanByteAndMemory.Length); else while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); } - else + else if (status == GarnetStatus.NOTFOUND) { - Debug.Assert(output.Memory == null); - WriteNull(); + Debug.Assert(output.SpanByteAndMemory.Memory == null); + + var notFoundOutput = new SpanByteAndMemory(PinnedSpanByte.FromPinnedPointer(dcurr, (int)(dend - dcurr))); + var writer = new RespMemoryWriter(respProtocolVersion, ref notFoundOutput); + customRawStringCommand.functions.NotFound(key, ref input, ref writer); + + if (!notFoundOutput.IsSpanByte) + { + // Couldn't write not found response in place, so copy it over + SendAndReset(notFoundOutput.Memory, writer.GetPosition()); + } + else + { + // Wrote not found response in place, just advance pointer + dcurr += writer.GetPosition(); + } + } + else if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); } } @@ -135,23 +155,23 @@ private bool TryCustomRawStringCommand(RespCommand cmd, long expirat /// /// Custom object command /// - private bool TryCustomObjectCommand(GarnetObjectType objType, byte subid, CommandType type, ref TGarnetApi storageApi) + private bool TryCustomObjectCommand(GarnetObjectType objType, CustomObjectCommand customObjectCommand, ref TGarnetApi storageApi) where TGarnetApi : IGarnetAdvancedApi { - var keyBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input - var header = new RespInputHeader(objType) { SubId = subid }; + var header = new RespInputHeader(objType) { SubId = customObjectCommand.subid }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); GarnetStatus status; - if (type == CommandType.ReadModifyWrite) + if (customObjectCommand.type == CommandType.ReadModifyWrite) { - status = storageApi.RMW_ObjectStore(ref keyBytes, ref input, ref output); + status = storageApi.RMW_ObjectStore(key, ref input, ref output); Debug.Assert(!output.SpanByteAndMemory.IsSpanByte); switch (status) @@ -171,7 +191,7 @@ private bool TryCustomObjectCommand(GarnetObjectType objType, byte s } else { - status = storageApi.Read_ObjectStore(ref keyBytes, ref input, ref output); + status = storageApi.Read_ObjectStore(key, ref input, ref output); Debug.Assert(!output.SpanByteAndMemory.IsSpanByte); switch (status) @@ -184,8 +204,21 @@ private bool TryCustomObjectCommand(GarnetObjectType objType, byte s SendAndReset(); break; case GarnetStatus.NOTFOUND: - Debug.Assert(output.SpanByteAndMemory.Memory == null); - WriteNull(); + var notFoundOutput = new SpanByteAndMemory(PinnedSpanByte.FromPinnedPointer(dcurr, (int)(dend - dcurr))); + + var writer = new RespMemoryWriter(respProtocolVersion, ref notFoundOutput); + customObjectCommand.functions.NotFound(key, ref input, ref writer); + + if (!notFoundOutput.IsSpanByte) + { + // Couldn't write not found response in place, so copy it over + SendAndReset(notFoundOutput.Memory, writer.GetPosition()); + } + else + { + // Wrote not found response in place, just advance pointer + dcurr += writer.GetPosition(); + } break; case GarnetStatus.WRONGTYPE: while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) @@ -219,58 +252,67 @@ public bool ParseCustomObjectCommand(string cmd, out CustomObjectCommand customO /// Args to the command /// Output from the command /// True if successful - public bool InvokeCustomRawStringCommand(ref TGarnetApi storageApi, CustomRawStringCommand customCommand, ArgSlice key, ArgSlice[] args, out ArgSlice output) + public bool InvokeCustomRawStringCommand(ref TGarnetApi storageApi, CustomRawStringCommand customCommand, PinnedSpanByte key, PinnedSpanByte[] args, out PinnedSpanByte output) where TGarnetApi : IGarnetAdvancedApi { ArgumentNullException.ThrowIfNull(customCommand); - var sbKey = key.SpanByte; var inputArg = customCommand.expirationTicks > 0 ? DateTimeOffset.UtcNow.Ticks + customCommand.expirationTicks : customCommand.expirationTicks; customCommandParseState.InitializeWithArguments(args); var cmd = customCommandManagerSession.GetCustomRespCommand(customCommand.id); - var rawStringInput = new RawStringInput(cmd, ref customCommandParseState, arg1: inputArg); + var stringInput = new StringInput(cmd, ref customCommandParseState, arg1: inputArg); - var _output = new SpanByteAndMemory(null); - GarnetStatus status; + var _output = new StringOutput(); if (customCommand.type == CommandType.ReadModifyWrite) { - status = storageApi.RMW_MainStore(ref sbKey, ref rawStringInput, ref _output); - Debug.Assert(!_output.IsSpanByte); + _ = storageApi.RMW_MainStore(key, ref stringInput, ref _output); + Debug.Assert(!_output.SpanByteAndMemory.IsSpanByte); - if (_output.Memory != null) + if (_output.SpanByteAndMemory.Memory != null) { - output = scratchBufferBuilder.FormatScratch(0, _output.AsReadOnlySpan()); - _output.Memory.Dispose(); + output = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan); + _output.SpanByteAndMemory.Memory.Dispose(); } else { - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_OK); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_OK); } } else { - status = storageApi.Read_MainStore(ref sbKey, ref rawStringInput, ref _output); - Debug.Assert(!_output.IsSpanByte); + var status = storageApi.Read_MainStore(key, ref stringInput, ref _output); + Debug.Assert(!_output.SpanByteAndMemory.IsSpanByte); if (status == GarnetStatus.OK) { - if (_output.Memory != null) + if (_output.SpanByteAndMemory.Memory != null) { - output = scratchBufferBuilder.FormatScratch(0, _output.AsReadOnlySpan()); - _output.Memory.Dispose(); + output = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan); + _output.SpanByteAndMemory.Memory.Dispose(); } else { - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_OK); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_OK); } } + else if (status == GarnetStatus.NOTFOUND) + { + Debug.Assert(_output.SpanByteAndMemory.Memory == null); + + var writer = new RespMemoryWriter(respProtocolVersion, ref _output.SpanByteAndMemory); + customCommand.functions.NotFound(key, ref stringInput, ref writer); + + output = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan[..writer.GetPosition()]); + + _output.SpanByteAndMemory.Memory.Dispose(); + } else { - Debug.Assert(_output.Memory == null); - if (respProtocolVersion >= 3) - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP3_NULL_REPLY); - else - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_ERRNOTFOUND); + Debug.Assert(status == GarnetStatus.WRONGTYPE, "Unexpected status"); + + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_ERR_WRONG_TYPE.Length + 1); // +1 because RESP_ERR_WRONG_TYPE doesn't contain the - prefix, but does starts with WRONGTYPE + output.Span[0] = (byte)'-'; + CmdStrings.RESP_ERR_WRONG_TYPE.CopyTo(output.Span[1..]); } } @@ -285,63 +327,71 @@ public bool InvokeCustomRawStringCommand(ref TGarnetApi storageApi, /// Args to the command /// Output from the command /// True if successful - public bool InvokeCustomObjectCommand(ref TGarnetApi storageApi, CustomObjectCommand customObjCommand, ArgSlice key, ArgSlice[] args, out ArgSlice output) + public bool InvokeCustomObjectCommand(ref TGarnetApi storageApi, CustomObjectCommand customObjCommand, PinnedSpanByte key, PinnedSpanByte[] args, out PinnedSpanByte output) where TGarnetApi : IGarnetAdvancedApi { ArgumentNullException.ThrowIfNull(customObjCommand); output = default; - var keyBytes = key.ToArray(); - // Prepare input var type = customCommandManagerSession.GetCustomGarnetObjectType(customObjCommand.id); var header = new RespInputHeader(type) { SubId = customObjCommand.subid }; customCommandParseState.InitializeWithArguments(args); var input = new ObjectInput(header, ref customCommandParseState); - var _output = new GarnetObjectStoreOutput(); + var _output = new ObjectOutput(); GarnetStatus status; if (customObjCommand.type == CommandType.ReadModifyWrite) { - status = storageApi.RMW_ObjectStore(ref keyBytes, ref input, ref _output); + status = storageApi.RMW_ObjectStore(key, ref input, ref _output); Debug.Assert(!_output.SpanByteAndMemory.IsSpanByte); switch (status) { case GarnetStatus.WRONGTYPE: - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_ERR_WRONG_TYPE); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_ERR_WRONG_TYPE); break; default: if (_output.SpanByteAndMemory.Memory != null) - output = scratchBufferBuilder.FormatScratch(0, _output.SpanByteAndMemory.AsReadOnlySpan()); + { + output = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan); + _output.SpanByteAndMemory.Memory.Dispose(); + } else - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_OK); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_OK); break; } } else { - status = storageApi.Read_ObjectStore(ref keyBytes, ref input, ref _output); + status = storageApi.Read_ObjectStore(key, ref input, ref _output); Debug.Assert(!_output.SpanByteAndMemory.IsSpanByte); switch (status) { case GarnetStatus.OK: if (_output.SpanByteAndMemory.Memory != null) - output = scratchBufferBuilder.FormatScratch(0, _output.SpanByteAndMemory.AsReadOnlySpan()); + { + output = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan); + _output.SpanByteAndMemory.Memory.Dispose(); + } else - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_OK); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_OK); break; case GarnetStatus.NOTFOUND: Debug.Assert(_output.SpanByteAndMemory.Memory == null); - if (respProtocolVersion >= 3) - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP3_NULL_REPLY); - else - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_ERRNOTFOUND); + + var writer = new RespMemoryWriter(respProtocolVersion, ref _output.SpanByteAndMemory); + + customObjCommand.functions.NotFound(key.ReadOnlySpan, ref input, ref writer); + + output = scratchBufferAllocator.CreateArgSlice(writer.GetPosition()); + + _output.SpanByteAndMemory.Memory.Dispose(); break; case GarnetStatus.WRONGTYPE: - output = scratchBufferBuilder.CreateArgSlice(CmdStrings.RESP_ERR_WRONG_TYPE); + output = scratchBufferAllocator.CreateArgSlice(CmdStrings.RESP_ERR_WRONG_TYPE); break; } } diff --git a/libs/server/Custom/CustomTransactionProcedure.cs b/libs/server/Custom/CustomTransactionProcedure.cs index 216364cae08..77556e1747e 100644 --- a/libs/server/Custom/CustomTransactionProcedure.cs +++ b/libs/server/Custom/CustomTransactionProcedure.cs @@ -14,6 +14,9 @@ public abstract class CustomTransactionProcedure : CustomProcedureBase { internal ScratchBufferAllocator scratchBufferAllocator; internal TransactionManager txnManager; + internal int virtualSublogParticipantCount; + internal ulong physicalSublogAccessVector; + internal BitVector[] replayTaskAccessVector = null; /// /// If enabled, transaction fails fast on key locking failure instead of waiting on lock @@ -36,30 +39,32 @@ public abstract class CustomTransactionProcedure : CustomProcedureBase /// /// /// - /// - protected void AddKey(ArgSlice key, LockType type, bool isObject) + /// + protected void AddKey(PinnedSpanByte key, LockType type, StoreType storeType) { - txnManager.SaveKeyEntryToLock(key, isObject, type); + txnManager.AddTransactionStoreType(storeType); + txnManager.SaveKeyEntryToLock(key, type); txnManager.VerifyKeyOwnership(key, type); + txnManager.ComputeCustomProcShardedLogAccess(key, this); } /// /// Rewind (pop) the last entry of scratch buffer (rewinding the current scratch buffer offset), /// if it contains the given ArgSlice /// - protected bool RewindScratchBuffer(ref ArgSlice slice) + protected bool RewindScratchBuffer(PinnedSpanByte slice) => scratchBufferAllocator.RewindScratchBuffer(ref slice); /// /// Create ArgSlice in scratch buffer, from given ReadOnlySpan /// - protected ArgSlice CreateArgSlice(ReadOnlySpan bytes) + protected PinnedSpanByte CreateArgSlice(ReadOnlySpan bytes) => scratchBufferAllocator.CreateArgSlice(bytes); /// /// Create ArgSlice in UTF8 format in scratch buffer, from given string /// - protected ArgSlice CreateArgSlice(string str) + protected PinnedSpanByte CreateArgSlice(string str) => scratchBufferAllocator.CreateArgSlice(str); /// diff --git a/libs/server/Databases/DatabaseManagerBase.cs b/libs/server/Databases/DatabaseManagerBase.cs index c0b00218072..f5b37fad989 100644 --- a/libs/server/Databases/DatabaseManagerBase.cs +++ b/libs/server/Databases/DatabaseManagerBase.cs @@ -4,18 +4,13 @@ using System; using System.Threading; using System.Threading.Tasks; +using Garnet.common; using Garnet.server.Metrics; using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Base class for logical database management /// @@ -40,14 +35,10 @@ internal abstract class DatabaseManagerBase : IDatabaseManager public abstract void ResumeCheckpoints(int dbId); /// - public abstract void RecoverCheckpoint(bool replicaRecover = false, bool recoverMainStoreFromToken = false, - bool recoverObjectStoreFromToken = false, CheckpointMetadata metadata = null); + public abstract void RecoverCheckpoint(bool replicaRecover = false, bool recoverFromToken = false, CheckpointMetadata metadata = null); /// - public abstract Task TakeCheckpointAsync(bool background, ILogger logger = null, CancellationToken token = default); - - /// - public abstract Task TakeCheckpointAsync(bool background, int dbId, ILogger logger = null, CancellationToken token = default); + public abstract Task TakeCheckpointAsync(bool background, int dbId = -1, CancellationToken token = default, ILogger logger = null); /// public abstract Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, int dbId = 0); @@ -69,10 +60,10 @@ public abstract Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, public abstract void RecoverAOF(); /// - public abstract long ReplayAOF(long untilAddress = -1); + public abstract AofAddress ReplayAOF(AofAddress untilAddress); /// - public abstract void DoCompaction(CancellationToken token = default, ILogger logger = null); + public abstract ValueTask DoCompactionAsync(CancellationToken token = default, ILogger logger = null); /// public abstract ValueTask GrowIndexesIfNeededAsync(CancellationToken token = default); @@ -84,7 +75,7 @@ public abstract Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, public abstract void ExpiredKeyDeletionScan(); /// - public abstract void StartObjectSizeTrackers(CancellationToken token = default); + public abstract void StartSizeTrackers(CancellationToken token = default); /// public abstract void Reset(int dbId = 0); @@ -123,19 +114,16 @@ public abstract Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, public abstract void RecoverVectorSets(); /// - public TsavoriteKV MainStore => DefaultDatabase.MainStore; - - /// - public TsavoriteKV ObjectStore => DefaultDatabase.ObjectStore; + public TsavoriteKV Store => DefaultDatabase.Store; /// - public TsavoriteLog AppendOnlyFile => DefaultDatabase.AppendOnlyFile; + public GarnetAppendOnlyFile AppendOnlyFile => DefaultDatabase.AppendOnlyFile; /// public DateTimeOffset LastSaveTime => DefaultDatabase.LastSaveTime; /// - public CacheSizeTracker ObjectStoreSizeTracker => DefaultDatabase.ObjectStoreSizeTracker; + public CacheSizeTracker SizeTracker => DefaultDatabase.SizeTracker; /// public WatchVersionMap VersionMap => DefaultDatabase.VersionMap; @@ -177,37 +165,14 @@ protected DatabaseManagerBase(StoreWrapper.DatabaseCreatorDelegate createDatabas /// /// Database to recover /// Store version - /// Object store version - protected void RecoverDatabaseCheckpoint(GarnetDatabase db, out long storeVersion, out long objectStoreVersion) + protected void RecoverDatabaseCheckpoint(GarnetDatabase db, out long storeVersion) { storeVersion = 0; - objectStoreVersion = 0; - if (db.ObjectStore != null) - { - // Get store recover version - var currStoreVersion = db.MainStore.GetRecoverVersion(); - // Get object store recover version - var currObjectStoreVersion = db.ObjectStore.GetRecoverVersion(); - - // Choose the minimum common recover version for both stores - if (currStoreVersion < currObjectStoreVersion) - currObjectStoreVersion = currStoreVersion; - else if (objectStoreVersion > 0) // handle the case where object store was disabled at checkpointing time - currStoreVersion = currObjectStoreVersion; - - // Recover to the minimum common recover version - storeVersion = db.MainStore.Recover(recoverTo: currStoreVersion); - objectStoreVersion = db.ObjectStore.Recover(recoverTo: currObjectStoreVersion); - Logger?.LogInformation("Recovered store to version {storeVersion} and object store to version {objectStoreVersion}", storeVersion, objectStoreVersion); - } - else - { - storeVersion = db.MainStore.Recover(); - Logger?.LogInformation("Recovered store to version {storeVersion}", storeVersion); - } + storeVersion = db.Store.Recover(); + Logger?.LogInformation("Recovered store to version {storeVersion}", storeVersion); - if (storeVersion > 0 || objectStoreVersion > 0) + if (storeVersion > 0) { db.LastSaveTime = DateTimeOffset.UtcNow; } @@ -220,36 +185,27 @@ protected void RecoverDatabaseCheckpoint(GarnetDatabase db, out long storeVersio /// Logger /// Cancellation token /// Tuple of store tail address and object store tail address - protected async Task<(long?, long?)> TakeCheckpointAsync(GarnetDatabase db, ILogger logger = null, CancellationToken token = default) + protected async Task TakeCheckpointAsync(GarnetDatabase db, ILogger logger = null, CancellationToken token = default) { try { - DoCompaction(db, isFromCheckpoint: true, logger); - var lastSaveStoreTailAddress = db.MainStore.Log.TailAddress; - var lastSaveObjectStoreTailAddress = (db.ObjectStore?.Log.TailAddress).GetValueOrDefault(); + await DoCompactionAsync(db, isFromCheckpoint: true, logger).ConfigureAwait(false); + var lastSaveStoreTailAddress = db.Store.Log.TailAddress; var full = db.LastSaveStoreTailAddress == 0 || - lastSaveStoreTailAddress - db.LastSaveStoreTailAddress >= StoreWrapper.serverOptions.FullCheckpointLogInterval || - (db.ObjectStore != null && (db.LastSaveObjectStoreTailAddress == 0 || - lastSaveObjectStoreTailAddress - db.LastSaveObjectStoreTailAddress >= StoreWrapper.serverOptions.FullCheckpointLogInterval)); - - var tryIncremental = StoreWrapper.serverOptions.EnableIncrementalSnapshots; - if (db.MainStore.IncrementalSnapshotTailAddress >= StoreWrapper.serverOptions.IncrementalSnapshotLogSizeLimit) - tryIncremental = false; - if (db.ObjectStore?.IncrementalSnapshotTailAddress >= StoreWrapper.serverOptions.IncrementalSnapshotLogSizeLimit) - tryIncremental = false; + lastSaveStoreTailAddress - db.LastSaveStoreTailAddress >= StoreWrapper.serverOptions.FullCheckpointLogInterval; var checkpointType = StoreWrapper.serverOptions.UseFoldOverCheckpoints ? CheckpointType.FoldOver : CheckpointType.Snapshot; - await InitiateCheckpointAsync(db, full, checkpointType, tryIncremental, logger); + await InitiateCheckpointAsync(db, full, checkpointType, logger).ConfigureAwait(false); - return full ? new(lastSaveStoreTailAddress, lastSaveObjectStoreTailAddress) : (null, null); + return full ? lastSaveStoreTailAddress : null; } catch (Exception ex) { logger?.LogError(ex, "Checkpointing threw exception, DB ID: {id}", db.Id); } - return (null, null); + return null; } /// @@ -275,9 +231,9 @@ protected void RecoverDatabaseAOF(GarnetDatabase db) { if (db.AppendOnlyFile == null) return; - db.AppendOnlyFile.Recover(); + db.AppendOnlyFile.Log.Recover(); Logger?.LogInformation("Recovered AOF: begin address = {beginAddress}, tail address = {tailAddress}, DB ID: {id}", - db.AppendOnlyFile.BeginAddress, db.AppendOnlyFile.TailAddress, db.Id); + db.AppendOnlyFile.Log.BeginAddress, db.AppendOnlyFile.Log.TailAddress, db.Id); } /// @@ -287,13 +243,13 @@ protected void RecoverDatabaseAOF(GarnetDatabase db) /// Database to replay /// Tail address /// Tail address - protected long ReplayDatabaseAOF(AofProcessor aofProcessor, GarnetDatabase db, long untilAddress = -1) + protected AofAddress ReplayDatabaseAOF(AofProcessor aofProcessor, GarnetDatabase db, AofAddress untilAddress) { - long replicationOffset = 0; try { - replicationOffset = aofProcessor.Recover(db, untilAddress); + var replicationOffset = aofProcessor.Recover(db, untilAddress); db.LastSaveTime = DateTimeOffset.UtcNow; + return replicationOffset; } catch (Exception ex) { @@ -302,7 +258,7 @@ protected long ReplayDatabaseAOF(AofProcessor aofProcessor, GarnetDatabase db, l throw; } - return replicationOffset; + return default; } /// @@ -313,11 +269,9 @@ protected void ResetDatabase(GarnetDatabase db) { try { - if (db.MainStore.Log.TailAddress > 64) - db.MainStore.Reset(); - if (db.ObjectStore?.Log.TailAddress > 64) - db.ObjectStore?.Reset(); - db.AppendOnlyFile?.Reset(); + if (db.Store.Log.TailAddress > 64) + db.Store.Reset(); + db.AppendOnlyFile?.Log.Reset(); var lastSave = DateTimeOffset.FromUnixTimeSeconds(0); db.LastSaveTime = lastSave; @@ -338,14 +292,7 @@ protected static void EnqueueDatabaseCommit(GarnetDatabase db, AofEntryType entr { if (db.AppendOnlyFile == null) return; - AofHeader header = new() - { - opType = entryType, - storeVersion = version, - sessionID = -1 - }; - - db.AppendOnlyFile.Enqueue(header, out _); + db.AppendOnlyFile.Log.EnqueueDatabaseCommit(entryType, version); } /// @@ -356,11 +303,13 @@ protected static void EnqueueDatabaseCommit(GarnetDatabase db, AofEntryType entr /// Truncate AOF log protected static void FlushDatabase(GarnetDatabase db, bool unsafeTruncateLog, bool truncateAof = true) { - db.MainStore.Log.ShiftBeginAddress(db.MainStore.Log.TailAddress, truncateLog: unsafeTruncateLog); - db.ObjectStore?.Log.ShiftBeginAddress(db.ObjectStore.Log.TailAddress, truncateLog: unsafeTruncateLog); + using (db.VectorManager?.BeginFlush()) + { + db.Store.Log.ShiftBeginAddress(db.Store.Log.TailAddress, truncateLog: unsafeTruncateLog); - if (truncateAof) - db.AppendOnlyFile?.TruncateUntil(db.AppendOnlyFile.TailAddress); + if (truncateAof) + db.AppendOnlyFile?.Log.TruncateUntil(db.AppendOnlyFile.Log.TailAddress); + } } /// @@ -372,30 +321,13 @@ protected async ValueTask GrowIndexesIfNeededAsync(GarnetDatabase db) { var indexesMaxedOut = true; - if (!DefaultDatabase.MainStoreIndexMaxedOut) - { - var dbMainStore = DefaultDatabase.MainStore; - if (await GrowIndexIfNeededAsync(StoreType.Main, - StoreWrapper.serverOptions.AdjustedIndexMaxCacheLines, dbMainStore.OverflowBucketAllocations, - () => dbMainStore.IndexSize, () => dbMainStore.GrowIndexAsync()).ConfigureAwait(false)) - { - db.MainStoreIndexMaxedOut = true; - } - else - { - indexesMaxedOut = false; - } - } - - if (!db.ObjectStoreIndexMaxedOut) + if (!DefaultDatabase.StoreIndexMaxedOut) { - var dbObjectStore = db.ObjectStore; - if (await GrowIndexIfNeededAsync(StoreType.Object, - StoreWrapper.serverOptions.AdjustedObjectStoreIndexMaxCacheLines, - dbObjectStore.OverflowBucketAllocations, - () => dbObjectStore.IndexSize, () => dbObjectStore.GrowIndexAsync()).ConfigureAwait(false)) + var store = DefaultDatabase.Store; + if (await GrowIndexIfNeededAsync(StoreWrapper.serverOptions.AdjustedIndexMaxCacheLines, store.OverflowBucketAllocations, + () => store.IndexSize, () => store.GrowIndexAsync()).ConfigureAwait(false)) { - db.ObjectStoreIndexMaxedOut = true; + db.StoreIndexMaxedOut = true; } else { @@ -413,15 +345,16 @@ protected async ValueTask GrowIndexesIfNeededAsync(GarnetDatabase db) /// Logger protected void ExecuteObjectCollection(GarnetDatabase db, ILogger logger = null) { - if (db.ObjectStoreCollectionDbStorageSession == null) + if (db.StoreCollectionDbStorageSession == null) { - var scratchBufferManager = new ScratchBufferBuilder(); - db.ObjectStoreCollectionDbStorageSession = - new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, db.VectorManager, Logger); + var scratchBufferBuilder = new ScratchBufferBuilder(); + var scratchBufferAllocator = new ScratchBufferAllocator(); + db.StoreCollectionDbStorageSession = + new StorageSession(StoreWrapper, scratchBufferBuilder, scratchBufferAllocator, null, null, db.Id, readSessionState: null, db.VectorManager, Logger); } - ExecuteHashCollect(db.ObjectStoreCollectionDbStorageSession); - ExecuteSortedSetCollect(db.ObjectStoreCollectionDbStorageSession); + ExecuteHashCollect(db.StoreCollectionDbStorageSession); + ExecuteSortedSetCollect(db.StoreCollectionDbStorageSession); } /// @@ -430,12 +363,7 @@ protected void ExecuteObjectCollection(GarnetDatabase db, ILogger logger = null) /// Database protected void ExpiredKeyDeletionScan(GarnetDatabase db) { - _ = MainStoreExpiredKeyDeletionScan(db); - - if (StoreWrapper.serverOptions.DisableObjects) - return; - - _ = ObjectStoreExpiredKeyDeletionScan(db); + _ = StoreExpiredKeyDeletionScan(db); } /// @@ -444,22 +372,21 @@ protected void ExpiredKeyDeletionScan(GarnetDatabase db) /// Database to run compaction on /// Logger /// True if called from checkpointing, false if called from background task - protected void DoCompaction(GarnetDatabase db, bool isFromCheckpoint = false, ILogger logger = null) + protected async ValueTask DoCompactionAsync(GarnetDatabase db, bool isFromCheckpoint = false, ILogger logger = null) { try { // If periodic compaction is enabled and this is called from checkpointing, skip compaction if (isFromCheckpoint && StoreWrapper.serverOptions.CompactionFrequencySecs > 0) return; - DoCompaction(db, StoreWrapper.serverOptions.CompactionMaxSegments, - StoreWrapper.serverOptions.ObjectStoreCompactionMaxSegments, 1, + await DoCompactionAsync(db, StoreWrapper.serverOptions.CompactionMaxSegments, 1, StoreWrapper.serverOptions.CompactionType, StoreWrapper.serverOptions.CompactionForceDelete); } catch (Exception ex) { logger?.LogError(ex, "Exception raised during compaction. AOF tail address = {tailAddress}; AOF committed until address = {commitAddress}; DB ID = {id}", - db.AppendOnlyFile.TailAddress, db.AppendOnlyFile.CommittedUntilAddress, db.Id); + db.AppendOnlyFile.Log.TailAddress, db.AppendOnlyFile.Log.CommittedUntilAddress, db.Id); throw; } } @@ -469,148 +396,97 @@ protected void DoCompaction(GarnetDatabase db, bool isFromCheckpoint = false, IL /// Decision is based on whether overflow bucket allocation is more than a threshold which indicates a contention /// in the index leading many allocations to the same bucket. /// - /// /// /// /// /// /// True if index has reached its max size - protected async ValueTask GrowIndexIfNeededAsync(StoreType storeType, long indexMaxSize, long overflowCount, Func indexSizeRetriever, Func growAction) + protected async ValueTask GrowIndexIfNeededAsync(long indexMaxSize, long overflowCount, Func indexSizeRetriever, Func growAction) { Logger?.LogDebug( - $"IndexAutoGrowTask[{{storeType}}]: checking index size {{indexSizeRetriever}} against max {{indexMaxSize}} with overflow {{overflowCount}}", - storeType, indexSizeRetriever(), indexMaxSize, overflowCount); + $"IndexAutoGrowTask: checking index size {{indexSizeRetriever}} against max {{indexMaxSize}} with overflow {{overflowCount}}", + indexSizeRetriever(), indexMaxSize, overflowCount); if (indexSizeRetriever() < indexMaxSize && overflowCount > (indexSizeRetriever() * StoreWrapper.serverOptions.IndexResizeThreshold / 100)) { Logger?.LogInformation( - $"IndexAutoGrowTask[{{storeType}}]: overflowCount {{overflowCount}} ratio more than threshold {{indexResizeThreshold}}%. Doubling index size...", - storeType, overflowCount, StoreWrapper.serverOptions.IndexResizeThreshold); + $"IndexAutoGrowTask: overflowCount {{overflowCount}} ratio more than threshold {{indexResizeThreshold}}%. Doubling index size...", + overflowCount, StoreWrapper.serverOptions.IndexResizeThreshold); await growAction().ConfigureAwait(false); } if (indexSizeRetriever() < indexMaxSize) return false; Logger?.LogDebug( - $"IndexAutoGrowTask[{{storeType}}]: checking index size {{indexSizeRetriever}} against max {{indexMaxSize}} with overflow {{overflowCount}}", - storeType, indexSizeRetriever(), indexMaxSize, overflowCount); + $"IndexAutoGrowTask: checking index size {{indexSizeRetriever}} against max {{indexMaxSize}} with overflow {{overflowCount}}", + indexSizeRetriever(), indexMaxSize, overflowCount); return true; } - private void DoCompaction(GarnetDatabase db, int mainStoreMaxSegments, int objectStoreMaxSegments, int numSegmentsToCompact, LogCompactionType compactionType, bool compactionForceDelete) + private async ValueTask DoCompactionAsync(GarnetDatabase db, int mainStoreMaxSegments, int numSegmentsToCompact, LogCompactionType compactionType, bool compactionForceDelete) { if (compactionType == LogCompactionType.None) return; - var mainStoreLog = db.MainStore.Log; + var storeLog = db.Store.Log; - var mainStoreMaxLogSize = (1L << StoreWrapper.serverOptions.SegmentSizeBits()) * mainStoreMaxSegments; + var mainStoreMaxLogSize = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * mainStoreMaxSegments; - if (mainStoreLog.ReadOnlyAddress - mainStoreLog.BeginAddress > mainStoreMaxLogSize) + if (storeLog.ReadOnlyAddress - storeLog.BeginAddress > mainStoreMaxLogSize) { - var readOnlyAddress = mainStoreLog.ReadOnlyAddress; - var compactLength = (1L << StoreWrapper.serverOptions.SegmentSizeBits()) * (mainStoreMaxSegments - numSegmentsToCompact); + var readOnlyAddress = storeLog.ReadOnlyAddress; + var compactLength = (1L << StoreWrapper.serverOptions.SegmentSizeBits(isObj: false)) * (mainStoreMaxSegments - numSegmentsToCompact); var untilAddress = readOnlyAddress - compactLength; Logger?.LogInformation( "Begin main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, mainStoreLog.BeginAddress, readOnlyAddress, mainStoreLog.TailAddress); + untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress); switch (compactionType) { case LogCompactionType.Shift: - mainStoreLog.ShiftBeginAddress(untilAddress, true, compactionForceDelete); + storeLog.ShiftBeginAddress(untilAddress, true, compactionForceDelete); break; case LogCompactionType.Scan: - mainStoreLog.Compact>(new SpanByteFunctions(), untilAddress, CompactionType.Scan); + storeLog.Compact(untilAddress, CompactionType.Scan); if (compactionForceDelete) { - CompactionCommitAof(db); - mainStoreLog.Truncate(); + await CompactionCommitAofAsync(db).ConfigureAwait(false); + storeLog.Truncate(); } break; case LogCompactionType.Lookup: - mainStoreLog.Compact>(new SpanByteFunctions(), untilAddress, CompactionType.Lookup); + storeLog.Compact(untilAddress, CompactionType.Lookup); if (compactionForceDelete) { - CompactionCommitAof(db); - mainStoreLog.Truncate(); + await CompactionCommitAofAsync(db).ConfigureAwait(false); + storeLog.Truncate(); } break; } Logger?.LogInformation( - "End main store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, mainStoreLog.BeginAddress, readOnlyAddress, mainStoreLog.TailAddress); - } - - if (db.ObjectStore == null) return; - - var objectStoreLog = db.ObjectStore.Log; - - var objectStoreMaxLogSize = (1L << StoreWrapper.serverOptions.ObjectStoreSegmentSizeBits()) * objectStoreMaxSegments; - - if (objectStoreLog.ReadOnlyAddress - objectStoreLog.BeginAddress > objectStoreMaxLogSize) - { - var readOnlyAddress = objectStoreLog.ReadOnlyAddress; - var compactLength = (1L << StoreWrapper.serverOptions.ObjectStoreSegmentSizeBits()) * (objectStoreMaxSegments - numSegmentsToCompact); - var untilAddress = readOnlyAddress - compactLength; - Logger?.LogInformation( - "Begin object store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, objectStoreLog.BeginAddress, readOnlyAddress, objectStoreLog.TailAddress); - - switch (compactionType) - { - case LogCompactionType.Shift: - objectStoreLog.ShiftBeginAddress(untilAddress, compactionForceDelete); - break; - - case LogCompactionType.Scan: - objectStoreLog.Compact>( - new SimpleSessionFunctions(), untilAddress, CompactionType.Scan); - if (compactionForceDelete) - { - CompactionCommitAof(db); - objectStoreLog.Truncate(); - } - break; - - case LogCompactionType.Lookup: - objectStoreLog.Compact>( - new SimpleSessionFunctions(), untilAddress, CompactionType.Lookup); - if (compactionForceDelete) - { - CompactionCommitAof(db); - objectStoreLog.Truncate(); - } - break; - } - - Logger?.LogInformation( - "End object store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", - untilAddress, mainStoreLog.BeginAddress, readOnlyAddress, mainStoreLog.TailAddress); + "End store compact until {untilAddress}, Begin = {beginAddress}, ReadOnly = {readOnlyAddress}, Tail = {tailAddress}", + untilAddress, storeLog.BeginAddress, readOnlyAddress, storeLog.TailAddress); } } - private void CompactionCommitAof(GarnetDatabase db) + private ValueTask CompactionCommitAofAsync(GarnetDatabase db) { // If we are the primary, we commit the AOF. - // If we are the replica, we commit the AOF only if fast commit is disabled - // because we do not want to clobber AOF addresses. // TODO: replica should instead wait until the next AOF commit is done via primary if (StoreWrapper.serverOptions.EnableAOF) { - if (StoreWrapper.serverOptions.EnableCluster && StoreWrapper.clusterProvider.IsReplica()) + // Replica does not commit here because it would clobber AOF addresses. + if (!(StoreWrapper.serverOptions.EnableCluster && StoreWrapper.clusterProvider.IsReplica())) { - if (!StoreWrapper.serverOptions.EnableFastCommit) - db.AppendOnlyFile?.Commit(); - } - else - { - db.AppendOnlyFile?.Commit(); + if (db.AppendOnlyFile != null) + return db.AppendOnlyFile.Log.CommitAsync(); } } + + return default; } /// @@ -619,26 +495,25 @@ private void CompactionCommitAof(GarnetDatabase db) /// Database to checkpoint /// True if full checkpoint should be initiated /// Type of checkpoint - /// Try to store as incremental delta over last snapshot /// Logger /// Task private async Task InitiateCheckpointAsync(GarnetDatabase db, bool full, CheckpointType checkpointType, - bool tryIncremental, ILogger logger = null) + ILogger logger = null) { - logger?.LogInformation("Initiating checkpoint; full = {full}, type = {checkpointType}, tryIncremental = {tryIncremental}, dbId = {dbId}", full, checkpointType, tryIncremental, db.Id); + logger?.LogInformation("Initiating checkpoint; full = {full}, type = {checkpointType}, dbId = {dbId}", full, checkpointType, db.Id); - long checkpointCoveredAofAddress = 0; + var checkpointCoveredAofAddress = AofAddress.Create(StoreWrapper.serverOptions.AofPhysicalSublogCount, 0); if (db.AppendOnlyFile != null) { if (StoreWrapper.serverOptions.EnableCluster) - StoreWrapper.clusterProvider.OnCheckpointInitiated(out checkpointCoveredAofAddress); + StoreWrapper.clusterProvider.OnCheckpointInitiated(ref checkpointCoveredAofAddress); else { - checkpointCoveredAofAddress = db.AppendOnlyFile.TailAddress; - StoreWrapper.StoreCheckpointManager.CurrentSafeAofAddress = checkpointCoveredAofAddress; + checkpointCoveredAofAddress = db.AppendOnlyFile.Log.TailAddress; + StoreWrapper.StoreCheckpointManager.SetCurrentSafeAofAddress(ref checkpointCoveredAofAddress); } - if (checkpointCoveredAofAddress > 0) + if (checkpointCoveredAofAddress.AnyGreater(0)) logger?.LogInformation("Will truncate AOF to {tailAddress} after checkpoint (files deleted after next commit), dbId = {dbId}", checkpointCoveredAofAddress, db.Id); } @@ -647,55 +522,41 @@ private async Task InitiateCheckpointAsync(GarnetDatabase db, bool full, Checkpo IStateMachine sm; if (full) { - sm = db.ObjectStore == null ? - Checkpoint.Full(db.MainStore, checkpointType, out checkpointResult.token) : - Checkpoint.Full(db.MainStore, db.ObjectStore, checkpointType, out checkpointResult.token); + sm = Checkpoint.Full(db.Store, checkpointType, out checkpointResult.token); } else { - tryIncremental = tryIncremental && db.MainStore.CanTakeIncrementalCheckpoint(checkpointType, out checkpointResult.token); - if (db.ObjectStore != null) - tryIncremental = tryIncremental && db.ObjectStore.CanTakeIncrementalCheckpoint(checkpointType, out var guid2) && checkpointResult.token == guid2; - - if (tryIncremental) - { - sm = db.ObjectStore == null ? - Checkpoint.IncrementalHybridLogOnly(db.MainStore, checkpointResult.token) : - Checkpoint.IncrementalHybridLogOnly(db.MainStore, db.ObjectStore, checkpointResult.token); - } - else - { - sm = db.ObjectStore == null ? - Checkpoint.HybridLogOnly(db.MainStore, checkpointType, out checkpointResult.token) : - Checkpoint.HybridLogOnly(db.MainStore, db.ObjectStore, checkpointType, out checkpointResult.token); - } + sm = Checkpoint.HybridLogOnly(db.Store, checkpointType, out checkpointResult.token); } - checkpointResult.success = await db.StateMachineDriver.RunAsync(sm); + checkpointResult.success = await db.StateMachineDriver.RunAsync(sm).ConfigureAwait(false); // If cluster is enabled the replication manager is responsible for truncating AOF if (StoreWrapper.serverOptions.EnableCluster && StoreWrapper.serverOptions.EnableAOF) { - StoreWrapper.clusterProvider.SafeTruncateAOF(full, checkpointCoveredAofAddress, + StoreWrapper.clusterProvider.AddNewCheckpointEntry(full, checkpointCoveredAofAddress, checkpointResult.token, checkpointResult.token); } else { - db.AppendOnlyFile?.TruncateUntil(checkpointCoveredAofAddress); - db.AppendOnlyFile?.Commit(); + db.AppendOnlyFile?.Log.TruncateUntil(checkpointCoveredAofAddress); + db.AppendOnlyFile?.Log.Commit(); } - if (db.ObjectStore != null) + // During the checkpoint, we may have serialized Garnet objects in (v) versions of objects. + // We can now safely remove these serialized versions as they are no longer needed. + // TODO: this should be done via push-based iterator under epoch protection + // so that we can adjust heap size at the time of clearing serializedBytes and update + // HeapMemorySize. The eviction scan can then avoid double-decrement. + using var iter1 = db.Store.Log.Scan(db.Store.Log.ReadOnlyAddress, db.Store.Log.TailAddress, DiskScanBufferingMode.SinglePageBuffering, includeClosedRecords: true); + while (iter1.GetNext()) { - // During the checkpoint, we may have serialized Garnet objects in (v) versions of objects. - // We can now safely remove these serialized versions as they are no longer needed. - using var iter1 = db.ObjectStore.Log.Scan(db.ObjectStore.Log.ReadOnlyAddress, - db.ObjectStore.Log.TailAddress, ScanBufferingMode.SinglePageBuffering, includeClosedRecords: true); - while (iter1.GetNext(out _, out _, out var value)) - { - if (value != null) - ((GarnetObjectBase)value).serialized = null; - } + if (!iter1.Info.ValueIsObject) + continue; + + var valueObject = iter1.ValueObject; + if (valueObject != null) + ((GarnetObjectBase)iter1.ValueObject).ClearSerializedObjectData(); } logger?.LogInformation("Completed checkpoint for DB ID: {id}", db.Id); @@ -706,48 +567,35 @@ private static void ExecuteHashCollect(StorageSession storageSession) var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HCOLLECT }; var input = new ObjectInput(header); - ReadOnlySpan key = [ArgSlice.FromPinnedSpan("*"u8)]; - storageSession.HashCollect(key, ref input, ref storageSession.objectStoreBasicContext); + ReadOnlySpan key = [PinnedSpanByte.FromPinnedSpan("*"u8)]; + storageSession.HashCollect(key, ref input, ref storageSession.objectBasicContext); storageSession.scratchBufferBuilder.Reset(); } private static void ExecuteSortedSetCollect(StorageSession storageSession) { - storageSession.SortedSetCollect(ref storageSession.objectStoreBasicContext); + storageSession.SortedSetCollect(ref storageSession.objectBasicContext); storageSession.scratchBufferBuilder.Reset(); } /// public abstract (long numExpiredKeysFound, long totalRecordsScanned) ExpiredKeyDeletionScan(int dbId); - protected (long numExpiredKeysFound, long totalRecordsScanned) MainStoreExpiredKeyDeletionScan(GarnetDatabase db) + protected (long numExpiredKeysFound, long totalRecordsScanned) StoreExpiredKeyDeletionScan(GarnetDatabase db) { - if (db.MainStoreExpiredKeyDeletionDbStorageSession == null) + if (db.StoreExpiredKeyDeletionDbStorageSession == null) { - var scratchBufferManager = new ScratchBufferBuilder(); - db.MainStoreExpiredKeyDeletionDbStorageSession = new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, db.VectorManager, Logger); + var scratchBufferBuilder = new ScratchBufferBuilder(); + var scratchBufferAllocator = new ScratchBufferAllocator(); + db.StoreExpiredKeyDeletionDbStorageSession = new StorageSession(StoreWrapper, scratchBufferBuilder, scratchBufferAllocator, null, null, db.Id, readSessionState: null, db.VectorManager, Logger); } var scanFrom = StoreWrapper.store.Log.ReadOnlyAddress; var scanUntil = StoreWrapper.store.Log.TailAddress; - (var deletedCount, var totalCount) = db.MainStoreExpiredKeyDeletionDbStorageSession.MainStoreExpiredKeyDeletionScan(scanFrom, scanUntil); - Logger?.LogDebug("Main Store - Deleted {deletedCount} keys out {totalCount} records in range {scanFrom} to {scanUntil} for DB {id}", deletedCount, totalCount, scanFrom, scanUntil, db.Id); - - return (deletedCount, totalCount); - } - - protected (long numExpiredKeysFound, long totalRecordsScanned) ObjectStoreExpiredKeyDeletionScan(GarnetDatabase db) - { - if (db.ObjectStoreExpiredKeyDeletionDbStorageSession == null) - { - var scratchBufferManager = new ScratchBufferBuilder(); - db.ObjectStoreExpiredKeyDeletionDbStorageSession = new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, db.VectorManager, Logger); - } - - var scanFrom = StoreWrapper.objectStore.Log.ReadOnlyAddress; - var scanUntil = StoreWrapper.objectStore.Log.TailAddress; - (var deletedCount, var totalCount) = db.ObjectStoreExpiredKeyDeletionDbStorageSession.ObjectStoreExpiredKeyDeletionScan(scanFrom, scanUntil); - Logger?.LogDebug("Object Store - Deleted {deletedCount} keys out {totalCount} records in range {scanFrom} to {scanUntil} for DB {id}", deletedCount, totalCount, scanFrom, scanUntil, db.Id); + var (deletedCount, totalCount) = db.StoreExpiredKeyDeletionDbStorageSession.ExpiredKeyDeletionScan(scanFrom, scanUntil); + Logger?.LogDebug( + "Store - Deleted {deletedCount} keys out {totalCount} records in range {scanFrom} to {scanUntil} for DB {id}", + deletedCount, totalCount, scanFrom, scanUntil, db.Id); return (deletedCount, totalCount); } @@ -757,34 +605,31 @@ private static void ExecuteSortedSetCollect(StorageSession storageSession) protected (HybridLogScanMetrics mainStore, HybridLogScanMetrics objectStore) CollectHybridLogStatsForDb(GarnetDatabase db) { - FunctionsState functionsState = CreateFunctionsState(); - MainSessionFunctions mainStoreSessionFuncs = new MainSessionFunctions(functionsState); - var mainStoreStats = CollectHybridLogStats(db, db.MainStore, mainStoreSessionFuncs); + var functionsState = CreateFunctionsState(); + var mainStoreSessionFunctions = new MainSessionFunctions(functionsState); + var mainStoreStats = CollectHybridLogStats(db, db.Store, mainStoreSessionFunctions); - HybridLogScanMetrics objectStoreStats = null; - if (ObjectStore != null) - { - ObjectSessionFunctions objectSessionFunctions = new ObjectSessionFunctions(functionsState); - objectStoreStats = CollectHybridLogStats(db, db.ObjectStore, objectSessionFunctions); - } + var objectSessionFunctions = new ObjectSessionFunctions(functionsState); + var objectStoreStats = CollectHybridLogStats(db, db.Store, objectSessionFunctions); return (mainStoreStats, objectStoreStats); } - private HybridLogScanMetrics CollectHybridLogStats( + private HybridLogScanMetrics CollectHybridLogStats( GarnetDatabase db, - TsavoriteKV store, - ISessionFunctions sessionFunctions) - where TFuncs : IStoreFunctions - where TAllocator : IAllocator + TsavoriteKV store, + ISessionFunctions sessionFunctions) + where TFuncs : IStoreFunctions + where TAllocator : IAllocator { if (db.HybridLogStatScanStorageSession == null) { - var scratchBufferManager = new ScratchBufferBuilder(); - db.HybridLogStatScanStorageSession = new StorageSession(StoreWrapper, scratchBufferManager, null, null, db.Id, db.VectorManager, Logger); + var scratchBufferBuilder = new ScratchBufferBuilder(); + var scratchBufferAllocator = new ScratchBufferAllocator(); + db.HybridLogStatScanStorageSession = new StorageSession(StoreWrapper, scratchBufferBuilder, scratchBufferAllocator, null, null, db.Id, readSessionState: null, db.VectorManager, Logger); } - using var session = store.NewSession>(sessionFunctions); + using var session = store.NewSession>(sessionFunctions); var basicContext = session.BasicContext; // region: Immutable || Mutable // state: RCUdSealed || RCUdUnsealed || Tombstoned || ElidedFromHashIndex || Live @@ -793,27 +638,25 @@ private HybridLogScanMetrics CollectHybridLogStats= db.MainStore.Log.ReadOnlyAddress ? "Mutable" : "Immutable"; + string region = iter.CurrentAddress >= db.Store.Log.ReadOnlyAddress ? "Mutable" : "Immutable"; string state = "Live"; - if (recordInfo.IsSealed) + if (iter.Info.IsSealed) { // while the server is live, this is true for RCUd records, when we recover from checkpoints, we unseal the records, so some RCUd records may not be sealed state = "RCUdSealed"; } - else if (recordInfo.Invalid) + else if (iter.Info.Invalid) { // Setting invalid is done when a record has been elided from the hash index state = "ElidedFromHashIndex"; } - else if (recordInfo.Tombstone) + else if (iter.Info.Tombstone) { state = "Tombstoned"; } - else if (!basicContext.ContainsKeyInMemory(ref key, out long tempKeyAddress, fromAddr).Found || iter.CurrentAddress != tempKeyAddress) + else if (!basicContext.ContainsKeyInMemory((FixedSpanByteKey)iter.Key, out long tempKeyAddress, fromAddr).Found || iter.CurrentAddress != tempKeyAddress) { // check if this was a record that RCUd by checking if the key when queried via hash index points to the same address state = "RCUdUnsealed"; diff --git a/libs/server/Databases/DatabaseManagerFactory.cs b/libs/server/Databases/DatabaseManagerFactory.cs index a1a9bae89a8..bf70f94b6f6 100644 --- a/libs/server/Databases/DatabaseManagerFactory.cs +++ b/libs/server/Databases/DatabaseManagerFactory.cs @@ -37,8 +37,8 @@ private static bool ShouldCreateMultipleDatabaseManager(GarnetServerOptions serv using (createDatabaseDelegate(0)) { // Check if there are multiple databases to recover from checkpoint - var checkpointParentDir = serverOptions.MainStoreCheckpointBaseDirectory; - var checkpointDirBaseName = serverOptions.GetCheckpointDirectoryName(0); + var checkpointParentDir = serverOptions.StoreCheckpointBaseDirectory; + var checkpointDirBaseName = GarnetServerOptions.GetCheckpointDirectoryName(0); if (MultiDatabaseManager.TryGetSavedDatabaseIds(checkpointParentDir, checkpointDirBaseName, out var dbIds) && dbIds.Any(id => id != 0)) @@ -48,7 +48,7 @@ private static bool ShouldCreateMultipleDatabaseManager(GarnetServerOptions serv if (serverOptions.EnableAOF) { var aofParentDir = serverOptions.AppendOnlyFileBaseDirectory; - var aofDirBaseName = serverOptions.GetAppendOnlyFileDirectoryName(0); + var aofDirBaseName = GarnetServerOptions.GetAppendOnlyFileDirectoryName(0); if (MultiDatabaseManager.TryGetSavedDatabaseIds(aofParentDir, aofDirBaseName, out dbIds) && dbIds.Any(id => id != 0)) diff --git a/libs/server/Databases/IDatabaseManager.cs b/libs/server/Databases/IDatabaseManager.cs index 644685a6092..e59176dba23 100644 --- a/libs/server/Databases/IDatabaseManager.cs +++ b/libs/server/Databases/IDatabaseManager.cs @@ -10,12 +10,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Interface for logical database management /// @@ -29,17 +23,12 @@ public interface IDatabaseManager : IDisposable /// /// Store (of DB 0) /// - public TsavoriteKV MainStore { get; } - - /// - /// Object store (of DB 0) - /// - public TsavoriteKV ObjectStore { get; } + public TsavoriteKV Store { get; } /// /// AOF (of DB 0) /// - public TsavoriteLog AppendOnlyFile { get; } + public GarnetAppendOnlyFile AppendOnlyFile { get; } /// /// Last save time (of DB 0) @@ -49,7 +38,7 @@ public interface IDatabaseManager : IDisposable /// /// Object store size tracker (of DB 0) /// - public CacheSizeTracker ObjectStoreSizeTracker { get; } + public CacheSizeTracker SizeTracker { get; } /// /// Version map (of DB 0) @@ -92,29 +81,18 @@ public interface IDatabaseManager : IDisposable /// Recover checkpoint /// /// - /// - /// - /// - public void RecoverCheckpoint(bool replicaRecover = false, bool recoverMainStoreFromToken = false, bool recoverObjectStoreFromToken = false, CheckpointMetadata metadata = null); + /// + public void RecoverCheckpoint(bool replicaRecover = false, bool recoverFromToken = false, CheckpointMetadata metadata = null); /// - /// Take checkpoint of all active databases if checkpointing is not in progress + /// Take checkpoint of all active databases (or a specified database) if checkpointing is not in progress /// /// True if method can return before checkpoint is taken - /// Logger + /// ID of database to checkpoint, or -1 (default) to checkpoint all active databases /// Cancellation token - /// False if another checkpointing process is already in progress - public Task TakeCheckpointAsync(bool background, ILogger logger = null, CancellationToken token = default); - - /// - /// Take checkpoint of specified database ID if checkpointing is not in progress - /// - /// True if method can return before checkpoint is taken - /// ID of database to checkpoint /// Logger - /// Cancellation token /// False if another checkpointing process is already in progress - public Task TakeCheckpointAsync(bool background, int dbId, ILogger logger = null, CancellationToken token = default); + public Task TakeCheckpointAsync(bool background, int dbId = -1, CancellationToken token = default, ILogger logger = null); /// /// Take a checkpoint if no checkpoint was taken after the provided time offset @@ -167,12 +145,12 @@ public Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, Cancellati /// /// When replaying AOF we do not want to write AOF records again. /// - public long ReplayAOF(long untilAddress = -1); + public AofAddress ReplayAOF(AofAddress untilAddress); /// /// Do compaction /// - public void DoCompaction(CancellationToken token = default, ILogger logger = null); + public ValueTask DoCompactionAsync(CancellationToken token = default, ILogger logger = null); /// /// Grows indexes of both main store and object store for all active databases if current size is too small @@ -191,9 +169,9 @@ public Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, Cancellati public void ExpiredKeyDeletionScan(); /// - /// Start object size trackers for all active databases + /// Start size trackers for all active databases /// - public void StartObjectSizeTrackers(CancellationToken token = default); + public void StartSizeTrackers(CancellationToken token = default); /// /// Reset diff --git a/libs/server/Databases/MultiDatabaseManager.cs b/libs/server/Databases/MultiDatabaseManager.cs index 20ad35a523c..94a9620c77e 100644 --- a/libs/server/Databases/MultiDatabaseManager.cs +++ b/libs/server/Databases/MultiDatabaseManager.cs @@ -41,13 +41,6 @@ internal class MultiDatabaseManager : DatabaseManagerBase // Lock for synchronizing checkpointing of all active DBs (if more than one) SingleWriterMultiReaderLock multiDbCheckpointingLock; - // Reusable task array for tracking checkpointing of multiple DBs - // Used by recurring checkpointing task if multiple DBs exist - Task[] checkpointTasks; - - // Reusable array for storing database IDs for checkpointing - int[] dbIdsToCheckpoint; - // True if StartObjectSizeTrackers was previously called bool sizeTrackersStarted; @@ -88,14 +81,14 @@ public MultiDatabaseManager(SingleDatabaseManager src) : } /// - public override void RecoverCheckpoint(bool replicaRecover = false, bool recoverMainStoreFromToken = false, bool recoverObjectStoreFromToken = false, CheckpointMetadata metadata = null) + public override void RecoverCheckpoint(bool replicaRecover = false, bool recoverFromToken = false, CheckpointMetadata metadata = null) { if (replicaRecover) throw new GarnetException( $"Unexpected call to {nameof(MultiDatabaseManager)}.{nameof(RecoverCheckpoint)} with {nameof(replicaRecover)} == true."); - var checkpointParentDir = StoreWrapper.serverOptions.MainStoreCheckpointBaseDirectory; - var checkpointDirBaseName = StoreWrapper.serverOptions.GetCheckpointDirectoryName(0); + var checkpointParentDir = StoreWrapper.serverOptions.StoreCheckpointBaseDirectory; + var checkpointDirBaseName = GarnetServerOptions.GetCheckpointDirectoryName(0); int[] dbIdsToRecover; try @@ -123,7 +116,7 @@ public override void RecoverCheckpoint(bool replicaRecover = false, bool recover try { - RecoverDatabaseCheckpoint(db, out storeVersion, out objectStoreVersion); + RecoverDatabaseCheckpoint(db, out storeVersion); } catch (TsavoriteNoHybridLogException ex) { @@ -141,128 +134,121 @@ public override void RecoverCheckpoint(bool replicaRecover = false, bool recover throw; } - // After recovery, we check if store versions match - if (db.ObjectStore != null && storeVersion != objectStoreVersion) - { - Logger?.LogInformation("Main store and object store checkpoint versions do not match; storeVersion = {storeVersion}; objectStoreVersion = {objectStoreVersion}", storeVersion, objectStoreVersion); - if (StoreWrapper.serverOptions.FailOnRecoveryError) - throw new GarnetException("Main store and object store checkpoint versions do not match"); - } - // Once everything is setup, initialize the VectorManager db.VectorManager.Initialize(); } } /// - public override async Task TakeCheckpointAsync(bool background, ILogger logger = null, CancellationToken token = default) + public override Task TakeCheckpointAsync(bool background, int dbId = -1, CancellationToken token = default, ILogger logger = null) { - var lockAcquired = TryGetDatabasesContentReadLock(token); - if (!lockAcquired) return false; + // Acquire databasesContentLock (read) so a concurrent swap-db can't move GarnetDatabase + // wrappers out from under us mid-checkpoint (which would mis-attribute LASTSAVE to the + // swapped DB and let a second BGSAVE race against the in-flight checkpoint). + if (!TryGetDatabasesContentReadLock(token)) return Task.FromResult(false); - var checkpointTask = TakeCheckpointHelperAsync(); + var multiDbLockHeld = false; + int[] pausedDbIds = null; + var pausedCount = 0; - if (background) - return true; - - return await checkpointTask.ConfigureAwait(false); - - async Task TakeCheckpointHelperAsync() + try { - // Force async - await Task.Yield(); - - var checkpointLockTaken = false; - - try + if (dbId == -1) { + // All-active-DBs path: take multiDbCheckpointingLock if multi-db, then synchronously + // pause per-DB checkpoints for every active DB so any BGSAVE issued after this method + // returns reliably observes the in-progress checkpoint. The buffer is local so a + // concurrent HandleDatabaseAdded that resizes shared state cannot strand the IDs. var activeDbIdsMapSize = activeDbIds.ActualSize; if (activeDbIdsMapSize > 1) { if (!multiDbCheckpointingLock.TryWriteLock()) - return false; + { + databasesContentLock.ReadUnlock(); + return Task.FromResult(false); + } - checkpointLockTaken = true; + multiDbLockHeld = true; } + pausedDbIds = new int[activeDbIdsMapSize]; var activeDbIdsMapSnapshot = activeDbIds.Map; - Array.Copy(activeDbIdsMapSnapshot, dbIdsToCheckpoint, activeDbIdsMapSize); - - return await TakeDatabasesCheckpointAsync(activeDbIdsMapSize, logger: logger, token: token); + for (var i = 0; i < activeDbIdsMapSize; i++) + { + var id = activeDbIdsMapSnapshot[i]; + if (TryPauseCheckpoints(id)) + pausedDbIds[pausedCount++] = id; + } } - finally + else { - if (checkpointLockTaken) - multiDbCheckpointingLock.WriteUnlock(); + // Single-DB path: just pause this one DB. multiDbCheckpointingLock is not taken + // because multiple per-DB BGSAVEs on different DBs are legal. + Debug.Assert(dbId < databases.ActualSize && databases.Map[dbId] != null); + + if (!TryPauseCheckpoints(dbId)) + { + databasesContentLock.ReadUnlock(); + return Task.FromResult(false); + } - databasesContentLock.ReadUnlock(); + pausedDbIds = [dbId]; + pausedCount = 1; } } - } + catch + { + if (pausedDbIds != null) + { + for (var i = 0; i < pausedCount; i++) + ResumeCheckpoints(pausedDbIds[i]); + } - /// - public override async Task TakeCheckpointAsync(bool background, int dbId, ILogger logger = null, CancellationToken token = default) - { - var databasesMapSize = databases.ActualSize; - var databasesMapSnapshot = databases.Map; - Debug.Assert(dbId < databasesMapSize && databasesMapSnapshot[dbId] != null); + if (multiDbLockHeld) + multiDbCheckpointingLock.WriteUnlock(); - // Check if checkpoint already in progress - if (!TryPauseCheckpoints(dbId)) - return false; + databasesContentLock.ReadUnlock(); + throw; + } - var checkpointTask = TakeCheckpointHelperAsync(databasesMapSnapshot, dbId, logger, token); + var checkpointTask = RunPausedCheckpointsAndReleaseLocksAsync(pausedDbIds, pausedCount, multiDbLockHeld, token, logger); if (background) - return true; - - await checkpointTask.ConfigureAwait(false); - return true; - - async Task TakeCheckpointHelperAsync(GarnetDatabase[] databasesMapSnapshot, int dbId, ILogger logger, CancellationToken token) - { - try - { - var (storeTailAddress, objectStoreTailAddress) = await TakeCheckpointAsync(databasesMapSnapshot[dbId], logger: logger, token: token).ConfigureAwait(false); + return Task.FromResult(true); - UpdateLastSaveData(dbId, storeTailAddress, objectStoreTailAddress); - } - finally - { - ResumeCheckpoints(dbId); - } - } + return checkpointTask; } /// public override async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, int dbId = 0) { - var databasesMapSize = databases.ActualSize; - var databasesMapSnapshot = databases.Map; - Debug.Assert(dbId < databasesMapSize && databasesMapSnapshot[dbId] != null); + Debug.Assert(dbId < databases.ActualSize && databases.Map[dbId] != null); - var db = databasesMapSnapshot[dbId]; - - // Check if checkpoint already in progress - var checkpointsPaused = TryPauseCheckpoints(dbId); + // Acquire databasesContentLock (read) so a concurrent swap-db can't mis-attribute LASTSAVE + // (UpdateLastSaveData uses databases.Map[dbId] at write time). + if (!TryGetDatabasesContentReadLock()) return; + var checkpointsPaused = false; try { + checkpointsPaused = TryPauseCheckpoints(dbId); + var db = databases.Map[dbId]; + // If another checkpoint is in progress or a checkpoint was taken beyond the provided entryTime - return if (!checkpointsPaused || db.LastSaveTime > entryTime) return; // Necessary to take a checkpoint because the latest checkpoint is before entryTime - var result = await TakeCheckpointAsync(db, logger: Logger); - - var storeTailAddress = result.Item1; - var objectStoreTailAddress = result.Item2; - UpdateLastSaveData(dbId, storeTailAddress, objectStoreTailAddress); + var storeTailAddress = await TakeCheckpointAsync(db, logger: Logger).ConfigureAwait(false); + UpdateLastSaveData(dbId, storeTailAddress); } finally { - ResumeCheckpoints(dbId); + if (checkpointsPaused) + ResumeCheckpoints(dbId); + + databasesContentLock.ReadUnlock(); } } @@ -270,50 +256,58 @@ public override async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, public override async Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, CancellationToken token = default, ILogger logger = null) { - var lockAcquired = TryGetDatabasesContentReadLock(token); - if (!lockAcquired) return; - - var activeDbIdsMapSize = activeDbIds.ActualSize; + if (!TryGetDatabasesContentReadLock(token)) return; - var checkpointLockTaken = false; + var multiDbLockHeld = false; try { + var activeDbIdsMapSize = activeDbIds.ActualSize; + if (activeDbIdsMapSize > 1) { if (!multiDbCheckpointingLock.TryWriteLock()) return; - checkpointLockTaken = true; + multiDbLockHeld = true; } + // Find first oversized DB and synchronously pause it. var databasesMapSnapshot = databases.Map; var activeDbIdsMapSnapshot = activeDbIds.Map; - - var dbIdsIdx = 0; + var pausedDbId = -1; for (var i = 0; i < activeDbIdsMapSize; i++) { var dbId = activeDbIdsMapSnapshot[i]; var db = databasesMapSnapshot[dbId]; Debug.Assert(db != null); - var dbAofSize = db.AppendOnlyFile.TailAddress - db.AppendOnlyFile.BeginAddress; + var dbAofSize = db.AppendOnlyFile.Log.TailAddress.AggregateDiff(db.AppendOnlyFile.Log.BeginAddress); if (dbAofSize > aofSizeLimit) { logger?.LogInformation("Enforcing AOF size limit currentAofSize: {dbAofSize} > AofSizeLimit: {aofSizeLimit} (Database ID: {dbId})", dbAofSize, aofSizeLimit, dbId); - dbIdsToCheckpoint[dbIdsIdx++] = dbId; + if (TryPauseCheckpoints(dbId)) + pausedDbId = dbId; break; } } - if (dbIdsIdx == 0) return; + if (pausedDbId < 0) return; - await TakeDatabasesCheckpointAsync(dbIdsIdx, logger: logger, token: token); + try + { + var storeTailAddress = await TakeCheckpointAsync(databasesMapSnapshot[pausedDbId], logger: logger, token: token).ConfigureAwait(false); + UpdateLastSaveData(pausedDbId, storeTailAddress); + } + finally + { + ResumeCheckpoints(pausedDbId); + } } finally { - if (checkpointLockTaken) + if (multiDbLockHeld) multiDbCheckpointingLock.WriteUnlock(); databasesContentLock.ReadUnlock(); @@ -333,7 +327,7 @@ public override async Task CommitToAofAsync(CancellationToken token = default, I var activeDbIdsMapSize = activeDbIds.ActualSize; var activeDbIdsMapSnapshot = activeDbIds.Map; - var aofTasks = new Task<(long, long)>[activeDbIdsMapSize]; + var aofTasks = new Task<(AofAddress, AofAddress)>[activeDbIdsMapSize]; for (var i = 0; i < activeDbIdsMapSize; i++) { @@ -341,13 +335,13 @@ public override async Task CommitToAofAsync(CancellationToken token = default, I var db = databasesMapSnapshot[dbId]; Debug.Assert(db != null); - aofTasks[i] = AwaitCommitAsync(db, db.AppendOnlyFile.CommitAsync(token: token)); + aofTasks[i] = AwaitCommitAsync(db, db.AppendOnlyFile.Log.CommitAsync(token: token)); } var exThrown = false; try { - await Task.WhenAll(aofTasks); + await Task.WhenAll(aofTasks).ConfigureAwait(false); } catch (Exception) { @@ -371,11 +365,11 @@ public override async Task CommitToAofAsync(CancellationToken token = default, I databasesContentLock.ReadUnlock(); } - static async Task<(long, long)> AwaitCommitAsync(GarnetDatabase db, ValueTask task) + static async Task<(AofAddress, AofAddress)> AwaitCommitAsync(GarnetDatabase db, ValueTask task) { await task.ConfigureAwait(false); - return (db.AppendOnlyFile.TailAddress, db.AppendOnlyFile.CommittedUntilAddress); + return (db.AppendOnlyFile.Log.TailAddress, db.AppendOnlyFile.Log.CommittedUntilAddress); } } @@ -386,7 +380,7 @@ public override async Task CommitToAofAsync(int dbId, CancellationToken token = var databasesMapSnapshot = databases.Map; Debug.Assert(dbId < databasesMapSize && databasesMapSnapshot[dbId] != null); - await databasesMapSnapshot[dbId].AppendOnlyFile.CommitAsync(token: token); + await databasesMapSnapshot[dbId].AppendOnlyFile.Log.CommitAsync(token: token).ConfigureAwait(false); } /// @@ -410,10 +404,10 @@ public override async Task WaitForCommitToAofAsync(CancellationToken token = def var db = databasesMapSnapshot[dbId]; Debug.Assert(db != null); - aofTasks[i] = db.AppendOnlyFile.WaitForCommitAsync(token: token).AsTask(); + aofTasks[i] = db.AppendOnlyFile.Log.WaitForCommitAsync(token: token).AsTask(); } - await Task.WhenAll(aofTasks); + await Task.WhenAll(aofTasks).ConfigureAwait(false); } finally { @@ -425,7 +419,7 @@ public override async Task WaitForCommitToAofAsync(CancellationToken token = def public override void RecoverAOF() { var aofParentDir = StoreWrapper.serverOptions.AppendOnlyFileBaseDirectory; - var aofDirBaseName = StoreWrapper.serverOptions.GetAppendOnlyFileDirectoryName(0); + var aofDirBaseName = GarnetServerOptions.GetAppendOnlyFileDirectoryName(0); int[] dbIdsToRecover; try @@ -453,16 +447,16 @@ public override void RecoverAOF() } /// - public override long ReplayAOF(long untilAddress = -1) + public override AofAddress ReplayAOF(AofAddress untilAddress) { if (!StoreWrapper.serverOptions.EnableAOF) - return -1; + return default; // When replaying AOF we do not want to write record again to AOF. // So initialize local AofProcessor with recordToAof: false. - var aofProcessor = new AofProcessor(StoreWrapper, recordToAof: false, logger: Logger); + var aofProcessor = new AofProcessor(StoreWrapper, clusterProvider: StoreWrapper.clusterProvider, recordToAof: false, logger: Logger); - long replicationOffset = 0; + var replicationOffset = AofAddress.Create(StoreWrapper.serverOptions.AofPhysicalSublogCount, 0); try { var databasesMapSnapshot = databases.Map; @@ -473,7 +467,7 @@ public override long ReplayAOF(long untilAddress = -1) for (var i = 0; i < activeDbIdsMapSize; i++) { var dbId = activeDbIdsMapSnapshot[i]; - var offset = ReplayDatabaseAOF(aofProcessor, databasesMapSnapshot[dbId], dbId == 0 ? untilAddress : -1); + var offset = ReplayDatabaseAOF(aofProcessor, databasesMapSnapshot[dbId], dbId == 0 ? untilAddress : AppendOnlyFile.InvalidAofAddress); if (dbId == 0) replicationOffset = offset; } } @@ -486,7 +480,7 @@ public override long ReplayAOF(long untilAddress = -1) } /// - public override void DoCompaction(CancellationToken token = default, ILogger logger = null) + public override async ValueTask DoCompactionAsync(CancellationToken token = default, ILogger logger = null) { var lockAcquired = TryGetDatabasesContentReadLock(token); if (!lockAcquired) return; @@ -507,7 +501,7 @@ public override void DoCompaction(CancellationToken token = default, ILogger log try { - DoCompaction(db); + await DoCompactionAsync(db).ConfigureAwait(false); } catch (Exception) { @@ -586,7 +580,7 @@ public override void ExpiredKeyDeletionScan() } /// - public override void StartObjectSizeTrackers(CancellationToken token = default) + public override void StartSizeTrackers(CancellationToken token = default) { sizeTrackersStarted = true; @@ -606,7 +600,7 @@ public override void StartObjectSizeTrackers(CancellationToken token = default) var db = databasesMapSnapshot[dbId]; Debug.Assert(db != null); - db.ObjectStoreSizeTracker?.Start(token); + db.SizeTracker?.Start(token); } } finally @@ -635,8 +629,7 @@ public override void ResetRevivificationStats() for (var i = 0; i < activeDbIdsMapSize; i++) { var dbId = activeDbIdsMapSnapshot[i]; - databaseMapSnapshot[dbId].MainStore.ResetRevivificationStats(); - databaseMapSnapshot[dbId].ObjectStore?.ResetRevivificationStats(); + databaseMapSnapshot[dbId].Store.ResetRevivificationStats(); } } @@ -722,8 +715,7 @@ public override FunctionsState CreateFunctionsState(int dbId = 0, byte respProto if (!success) throw new GarnetException($"Database with ID {dbId} was not found."); - return new(db.AppendOnlyFile, db.VersionMap, StoreWrapper.customCommandManager, null, db.ObjectStoreSizeTracker, - StoreWrapper.GarnetObjectSerializer, db.VectorManager, respProtocolVersion); + return new(db.AppendOnlyFile, db.VersionMap, StoreWrapper, memoryPool: null, db.SizeTracker, db.VectorManager, Logger, respProtocolVersion); } /// @@ -943,21 +935,10 @@ private void HandleDatabaseAdded(int dbId) // If size tracker exists and is stopped, start it (only if DB 0 size tracker is started as well) var db = databases.Map[dbId]; if (sizeTrackersStarted) - db.ObjectStoreSizeTracker?.Start(StoreWrapper.ctsCommit.Token); + db.SizeTracker?.Start(StoreWrapper.ctsCommit.Token); activeDbIds.TryGetNextId(out var nextIdx); activeDbIds.TrySetValue(nextIdx, db.Id); - - activeDbIds.mapLock.ReadLock(); - try - { - checkpointTasks = new Task[activeDbIds.ActualSize]; - dbIdsToCheckpoint = new int[activeDbIds.ActualSize]; - } - finally - { - activeDbIds.mapLock.ReadUnlock(); - } } /// @@ -971,12 +952,14 @@ private void CopyDatabases(IDatabaseManager src, bool enableAof) { case SingleDatabaseManager sdbm: var defaultDbCopy = new GarnetDatabase(0, sdbm.DefaultDatabase, enableAof); + sizeTrackersStarted = sdbm.SizeTracker?.IsStarted ?? false; TryAddDatabase(0, defaultDbCopy); return; case MultiDatabaseManager mdbm: var activeDbIdsMapSize = mdbm.activeDbIds.ActualSize; var activeDbIdsMapSnapshot = mdbm.activeDbIds.Map; var databasesMapSnapshot = mdbm.databases.Map; + sizeTrackersStarted = mdbm.sizeTrackersStarted; for (var i = 0; i < activeDbIdsMapSize; i++) { @@ -992,76 +975,71 @@ private void CopyDatabases(IDatabaseManager src, bool enableAof) } /// - /// Asynchronously checkpoint multiple databases and wait for all to complete + /// Run pre-paused per-DB checkpoints in parallel, then resume the per-DB checkpoint locks + /// and release the outer locks held by the caller. + /// Caller must hold as a reader and must have synchronously + /// pause-locked the first entries of . + /// Per-DB checkpoint locks are held until ALL per-DB checkpoints complete (not just each + /// individual one) so a per-DB BGSAVE issued mid-flight during a general BGSAVE reliably + /// observes the in-progress checkpoint and fails with "checkpoint already in progress". /// - /// Number of databases to checkpoint (first dbIdsCount indexes from dbIdsToCheckpoint) - /// Logger - /// Cancellation token - /// False if checkpointing already in progress - private async Task TakeDatabasesCheckpointAsync(int dbIdsCount, ILogger logger = null, - CancellationToken token = default) + private async Task RunPausedCheckpointsAndReleaseLocksAsync(int[] pausedDbIds, int pausedCount, + bool multiDbLockHeld, CancellationToken token, ILogger logger) { - Debug.Assert(checkpointTasks != null); - Debug.Assert(dbIdsCount <= dbIdsToCheckpoint.Length); - - for (var i = 0; i < checkpointTasks.Length; i++) + // Pre-fill with Task.CompletedTask so the catch path can safely await Task.WhenAll + // even if the synchronous task-creation loop below throws partway through. + var checkpointTasks = new Task[pausedCount]; + for (var i = 0; i < pausedCount; i++) checkpointTasks[i] = Task.CompletedTask; - var lockAcquired = TryGetDatabasesContentReadLock(token); - if (!lockAcquired) return false; - try { + // Force async so that the entry point can return synchronously to the caller. + await Task.Yield(); + var databaseMapSnapshot = databases.Map; - for (var currIdx = 0; currIdx < dbIdsCount; currIdx++) + try { - var dbId = dbIdsToCheckpoint[currIdx]; - - // If a checkpoint is already in progress for this database, skip it - if (!TryPauseCheckpoints(dbId)) - continue; + for (var i = 0; i < pausedCount; i++) + checkpointTasks[i] = TakeOneCheckpointAsync(databaseMapSnapshot[pausedDbIds[i]], pausedDbIds[i]); - checkpointTasks[currIdx] = TakeCheckpointHelperAsync(databaseMapSnapshot, dbId); + await Task.WhenAll(checkpointTasks).ConfigureAwait(false); } + catch (Exception ex) + { + logger?.LogError(ex, "Checkpointing threw exception"); - await Task.WhenAll(checkpointTasks).ConfigureAwait(false); - } - catch (Exception ex) - { - logger?.LogError(ex, "Checkpointing threw exception"); + // Make sure any tasks already started are observed before we resume the per-DB + // locks in the outer finally (otherwise we could resume a lock while its + // checkpoint is still running). + try { await Task.WhenAll(checkpointTasks).ConfigureAwait(false); } + catch { /* already logged above */ } + } } finally { + for (var i = 0; i < pausedCount; i++) + ResumeCheckpoints(pausedDbIds[i]); + + if (multiDbLockHeld) + multiDbCheckpointingLock.WriteUnlock(); + databasesContentLock.ReadUnlock(); } return true; - async Task TakeCheckpointHelperAsync(GarnetDatabase[] databaseMapSnapshot, int dbId) + // Local function: take one per-DB checkpoint and update LASTSAVE. Does NOT resume the + // per-DB lock — the outer finally above resumes all paused DBs after WhenAll completes. + async Task TakeOneCheckpointAsync(GarnetDatabase db, int dbId) { - var needsResume = true; - - try - { - var (storeTailAddress, objectStoreTailAddress) = await TakeCheckpointAsync(databaseMapSnapshot[dbId], logger: logger, token: token).ConfigureAwait(false); - - ResumeCheckpoints(dbId); - needsResume = false; - - UpdateLastSaveData(dbId, storeTailAddress, objectStoreTailAddress); - } - finally - { - if (needsResume) - { - ResumeCheckpoints(dbId); - } - } + var storeTailAddress = await TakeCheckpointAsync(db, logger: logger, token: token).ConfigureAwait(false); + UpdateLastSaveData(dbId, storeTailAddress); } } - private void UpdateLastSaveData(int dbId, long? storeTailAddress, long? objectStoreTailAddress) + private void UpdateLastSaveData(int dbId, long? storeTailAddress) { var databasesMapSnapshot = databases.Map; @@ -1071,9 +1049,6 @@ private void UpdateLastSaveData(int dbId, long? storeTailAddress, long? objectSt if (storeTailAddress.HasValue) { db.LastSaveStoreTailAddress = storeTailAddress.Value; - - if (db.ObjectStore != null && objectStoreTailAddress.HasValue) - db.LastSaveObjectStoreTailAddress = objectStoreTailAddress.Value; } } @@ -1113,11 +1088,7 @@ public override void Dispose() } public override (long numExpiredKeysFound, long totalRecordsScanned) ExpiredKeyDeletionScan(int dbId) - { - var (k1, t1) = MainStoreExpiredKeyDeletionScan(GetDbById(dbId)); - var (k2, t2) = StoreWrapper.serverOptions.DisableObjects ? (0, 0) : ObjectStoreExpiredKeyDeletionScan(GetDbById(dbId)); - return (k1 + k2, t1 + t2); - } + => StoreExpiredKeyDeletionScan(GetDbById(dbId)); private GarnetDatabase GetDbById(int dbId) { diff --git a/libs/server/Databases/SingleDatabaseManager.cs b/libs/server/Databases/SingleDatabaseManager.cs index 755e35f7c83..9112b88acd0 100644 --- a/libs/server/Databases/SingleDatabaseManager.cs +++ b/libs/server/Databases/SingleDatabaseManager.cs @@ -4,7 +4,6 @@ using System; using System.Threading; using System.Threading.Tasks; -using Garnet.common; using Garnet.server.Metrics; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -54,9 +53,9 @@ public override GarnetDatabase TryGetOrAddDatabase(int dbId, out bool success, o } /// - public override void RecoverCheckpoint(bool replicaRecover = false, bool recoverMainStoreFromToken = false, bool recoverObjectStoreFromToken = false, CheckpointMetadata metadata = null) + public override void RecoverCheckpoint(bool replicaRecover = false, bool recoverFromToken = false, CheckpointMetadata metadata = null) { - long storeVersion = 0, objectStoreVersion = 0; + long storeVersion = 0; try { if (replicaRecover) @@ -64,50 +63,30 @@ public override void RecoverCheckpoint(bool replicaRecover = false, bool recover // Note: Since replicaRecover only pertains to cluster-mode, we can use the default store pointers (since multi-db mode is disabled in cluster-mode) if (metadata!.storeIndexToken != default && metadata.storeHlogToken != default) { - storeVersion = !recoverMainStoreFromToken ? MainStore.Recover() : MainStore.Recover(metadata.storeIndexToken, metadata.storeHlogToken); + storeVersion = !recoverFromToken ? Store.Recover() : Store.Recover(metadata.storeIndexToken, metadata.storeHlogToken); } - if (ObjectStore != null) - { - if (metadata.objectStoreIndexToken != default && metadata.objectStoreHlogToken != default) - { - objectStoreVersion = !recoverObjectStoreFromToken ? ObjectStore.Recover() : ObjectStore.Recover(metadata.objectStoreIndexToken, metadata.objectStoreHlogToken); - } - } - - if (storeVersion > 0 || objectStoreVersion > 0) + if (storeVersion > 0) defaultDatabase.LastSaveTime = DateTimeOffset.UtcNow; } else { - RecoverDatabaseCheckpoint(defaultDatabase, out storeVersion, out objectStoreVersion); + RecoverDatabaseCheckpoint(defaultDatabase, out storeVersion); } } catch (TsavoriteNoHybridLogException ex) { // No hybrid log being found is not the same as an error in recovery. e.g. fresh start - Logger?.LogInformation(ex, - "No Hybrid Log found for recovery; storeVersion = {storeVersion}; objectStoreVersion = {objectStoreVersion}", - storeVersion, objectStoreVersion); + Logger?.LogInformation(ex, "No Hybrid Log found for recovery; storeVersion = {storeVersion};", storeVersion); } catch (Exception ex) { - Logger?.LogInformation(ex, - "Error during recovery of store; storeVersion = {storeVersion}; objectStoreVersion = {objectStoreVersion}", - storeVersion, objectStoreVersion); + Logger?.LogInformation(ex, "Error during recovery of store; storeVersion = {storeVersion};", storeVersion); if (StoreWrapper.serverOptions.FailOnRecoveryError) throw; } - // After recovery, we check if store versions match - if (ObjectStore != null && storeVersion != objectStoreVersion) - { - Logger?.LogInformation("Main store and object store checkpoint versions do not match; storeVersion = {storeVersion}; objectStoreVersion = {objectStoreVersion}", storeVersion, objectStoreVersion); - if (StoreWrapper.serverOptions.FailOnRecoveryError) - throw new GarnetException("Main store and object store checkpoint versions do not match"); - } - // Once everything is setup, initialize the VectorManager defaultDatabase.VectorManager.Initialize(); } @@ -129,8 +108,11 @@ public override void ResumeCheckpoints(int dbId) } /// - public override async Task TakeCheckpointAsync(bool background, ILogger logger = null, CancellationToken token = default) + public override async Task TakeCheckpointAsync(bool background, int dbId = -1, CancellationToken token = default, ILogger logger = null) { + if (dbId != -1 && dbId != 0) + throw new ArgumentOutOfRangeException(nameof(dbId), dbId, "SingleDatabaseManager only supports dbId 0."); + // Check if checkpoint already in progress if (!TryPauseCheckpoints(defaultDatabase.Id)) return false; @@ -146,12 +128,10 @@ async Task TakeCheckpointHelperAsync(GarnetDatabase defaultDatabase, ILogger log { try { - var (storeTailAddress, objectStoreTailAddress) = await TakeCheckpointAsync(defaultDatabase, logger: logger, token: token).ConfigureAwait(false); + var storeTailAddress = await TakeCheckpointAsync(defaultDatabase, logger: logger, token: token).ConfigureAwait(false); if (storeTailAddress.HasValue) defaultDatabase.LastSaveStoreTailAddress = storeTailAddress.Value; - if (ObjectStore != null && objectStoreTailAddress.HasValue) - defaultDatabase.LastSaveObjectStoreTailAddress = objectStoreTailAddress.Value; defaultDatabase.LastSaveTime = DateTimeOffset.UtcNow; } @@ -162,14 +142,6 @@ async Task TakeCheckpointHelperAsync(GarnetDatabase defaultDatabase, ILogger log } } - /// - public override Task TakeCheckpointAsync(bool background, int dbId, ILogger logger = null, CancellationToken token = default) - { - ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0); - - return TakeCheckpointAsync(background, logger, token); - } - /// public override async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, int dbId = 0) { @@ -186,15 +158,12 @@ public override async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, return; // Necessary to take a checkpoint because the latest checkpoint is before entryTime - var result = await TakeCheckpointAsync(defaultDatabase, logger: Logger); + var result = await TakeCheckpointAsync(defaultDatabase, logger: Logger).ConfigureAwait(false); - var storeTailAddress = result.Item1; - var objectStoreTailAddress = result.Item2; + var storeTailAddress = result; if (storeTailAddress.HasValue) defaultDatabase.LastSaveStoreTailAddress = storeTailAddress.Value; - if (ObjectStore != null && objectStoreTailAddress.HasValue) - defaultDatabase.LastSaveObjectStoreTailAddress = objectStoreTailAddress.Value; defaultDatabase.LastSaveTime = DateTimeOffset.UtcNow; } @@ -208,7 +177,7 @@ public override async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, public override async Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLimit, CancellationToken token = default, ILogger logger = null) { - var aofSize = AppendOnlyFile.TailAddress - AppendOnlyFile.BeginAddress; + var aofSize = StoreWrapper.AofSize(); if (aofSize <= aofSizeLimit) return; if (!await TryPauseCheckpointsContinuousAsync(defaultDatabase.Id, token: token).ConfigureAwait(false)) @@ -226,12 +195,9 @@ public override async Task TaskCheckpointBasedOnAofSizeLimitAsync(long aofSizeLi logger?.LogInformation("Enforcing AOF size limit currentAofSize: {aofSize} > AofSizeLimit: {aofSizeLimit}", aofSize, aofSizeLimit); - var (storeTailAddress, objectStoreTailAddress) = await TakeCheckpointAsync(defaultDatabase, logger: logger, token: token).ConfigureAwait(false); - + var storeTailAddress = await TakeCheckpointAsync(defaultDatabase, logger: logger, token: token).ConfigureAwait(false); if (storeTailAddress.HasValue) defaultDatabase.LastSaveStoreTailAddress = storeTailAddress.Value; - if (ObjectStore != null && objectStoreTailAddress.HasValue) - defaultDatabase.LastSaveObjectStoreTailAddress = objectStoreTailAddress.Value; defaultDatabase.LastSaveTime = DateTimeOffset.UtcNow; } @@ -246,13 +212,13 @@ public override async Task CommitToAofAsync(CancellationToken token = default, I { try { - await AppendOnlyFile.CommitAsync(token: token); + await AppendOnlyFile.Log.CommitAsync(token: token).ConfigureAwait(false); } catch (Exception ex) { logger?.LogError(ex, "Exception raised while committing to AOF. AOF tail address = {tailAddress}; AOF committed until address = {commitAddress}; ", - AppendOnlyFile.TailAddress, AppendOnlyFile.CommittedUntilAddress); + AppendOnlyFile.Log.TailAddress, AppendOnlyFile.Log.CommittedUntilAddress); throw; } } @@ -262,27 +228,27 @@ public override async Task CommitToAofAsync(int dbId, CancellationToken token = { ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0); - await CommitToAofAsync(token, logger); + await CommitToAofAsync(token, logger).ConfigureAwait(false); } /// public override async Task WaitForCommitToAofAsync(CancellationToken token = default, ILogger logger = null) { - await AppendOnlyFile.WaitForCommitAsync(token: token); + await AppendOnlyFile.Log.WaitForCommitAsync(token: token).ConfigureAwait(false); } /// public override void RecoverAOF() => RecoverDatabaseAOF(defaultDatabase); /// - public override long ReplayAOF(long untilAddress = -1) + public override AofAddress ReplayAOF(AofAddress untilAddress) { if (!StoreWrapper.serverOptions.EnableAOF) - return -1; + return default; // When replaying AOF we do not want to write record again to AOF. // So initialize local AofProcessor with recordToAof: false. - var aofProcessor = new AofProcessor(StoreWrapper, recordToAof: false, logger: Logger); + var aofProcessor = new AofProcessor(StoreWrapper, clusterProvider: StoreWrapper.clusterProvider, recordToAof: false, logger: Logger); try { @@ -295,7 +261,7 @@ public override long ReplayAOF(long untilAddress = -1) } /// - public override void DoCompaction(CancellationToken token = default, ILogger logger = null) => DoCompaction(defaultDatabase); + public override ValueTask DoCompactionAsync(CancellationToken token = default, ILogger logger = null) => DoCompactionAsync(defaultDatabase); /// public override ValueTask GrowIndexesIfNeededAsync(CancellationToken token = default) => @@ -310,8 +276,8 @@ public override void ExpiredKeyDeletionScan() => ExpiredKeyDeletionScan(defaultDatabase); /// - public override void StartObjectSizeTrackers(CancellationToken token = default) => - ObjectStoreSizeTracker?.Start(token); + public override void StartSizeTrackers(CancellationToken token = default) => + SizeTracker?.Start(token); /// public override void Reset(int dbId = 0) @@ -323,10 +289,7 @@ public override void Reset(int dbId = 0) /// public override void ResetRevivificationStats() - { - MainStore.ResetRevivificationStats(); - ObjectStore?.ResetRevivificationStats(); - } + => Store.ResetRevivificationStats(); /// public override void EnqueueCommit(AofEntryType entryType, long version, int dbId = 0) @@ -356,8 +319,8 @@ public override void FlushDatabase(bool unsafeTruncateLog, int dbId = 0) FlushDatabase(defaultDatabase, unsafeTruncateLog, !safeTruncateAof); - if (safeTruncateAof) - SafeTruncateAOF(AofEntryType.FlushDb, unsafeTruncateLog); + if (safeTruncateAof && StoreWrapper.serverOptions.EnableAOF) + SafeFlushAOF(AofEntryType.FlushDb, unsafeTruncateLog); } /// @@ -367,8 +330,10 @@ public override void FlushAllDatabases(bool unsafeTruncateLog) FlushDatabase(defaultDatabase, unsafeTruncateLog, !safeTruncateAof); + // We truncate AOF safely only in the cluster case. + // For standalone FlushDatabase will take care of the AOF truncation if (safeTruncateAof) - SafeTruncateAOF(AofEntryType.FlushAll, unsafeTruncateLog); + SafeFlushAOF(AofEntryType.FlushAll, unsafeTruncateLog); } /// @@ -382,8 +347,7 @@ public override FunctionsState CreateFunctionsState(int dbId = 0, byte respProto { ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0); - return new(AppendOnlyFile, VersionMap, StoreWrapper.customCommandManager, null, ObjectStoreSizeTracker, - StoreWrapper.GarnetObjectSerializer, DefaultDatabase.VectorManager, respProtocolVersion); + return new(AppendOnlyFile, VersionMap, StoreWrapper, null, SizeTracker, DefaultDatabase.VectorManager, Logger, respProtocolVersion); } private async Task TryPauseCheckpointsContinuousAsync(int dbId, @@ -405,27 +369,20 @@ private async Task TryPauseCheckpointsContinuousAsync(int dbId, public override (long numExpiredKeysFound, long totalRecordsScanned) ExpiredKeyDeletionScan(int dbId) { ArgumentOutOfRangeException.ThrowIfNotEqual(dbId, 0); - var (k1, t1) = MainStoreExpiredKeyDeletionScan(DefaultDatabase); - var (k2, t2) = StoreWrapper.serverOptions.DisableObjects ? (0, 0) : ObjectStoreExpiredKeyDeletionScan(DefaultDatabase); - return (k1 + k2, t1 + t2); + return StoreExpiredKeyDeletionScan(DefaultDatabase); } public override (HybridLogScanMetrics mainStore, HybridLogScanMetrics objectStore)[] CollectHybridLogStats() => [CollectHybridLogStatsForDb(defaultDatabase)]; - private void SafeTruncateAOF(AofEntryType entryType, bool unsafeTruncateLog) + private unsafe void SafeFlushAOF(AofEntryType entryType, bool unsafeTruncateLog) { - StoreWrapper.clusterProvider.SafeTruncateAOF(AppendOnlyFile.TailAddress); + // Safe truncate up to tail for botth primary and replica + StoreWrapper.clusterProvider.SafeTruncateAOF(AppendOnlyFile.Log.TailAddress); + + // Only enqueue operation if this is a primary if (StoreWrapper.clusterProvider.IsPrimary()) { - AofHeader header = new() - { - opType = entryType, - storeVersion = 0, - sessionID = -1, - unsafeTruncateLog = unsafeTruncateLog ? (byte)0 : (byte)1, - databaseId = (byte)defaultDatabase.Id - }; - AppendOnlyFile?.Enqueue(header, out _); + AppendOnlyFile.Log.EnqueueSafeFlushAOF(entryType, unsafeTruncateLog, defaultDatabase.Id); } } diff --git a/libs/server/Garnet.server.csproj b/libs/server/Garnet.server.csproj index de66b80f463..61d12536ede 100644 --- a/libs/server/Garnet.server.csproj +++ b/libs/server/Garnet.server.csproj @@ -13,6 +13,7 @@ + @@ -22,8 +23,17 @@ - + + + + $(DefineConstants);SERVER_PROJECT + + + + + + \ No newline at end of file diff --git a/libs/server/GarnetCheckpointManager.cs b/libs/server/GarnetCheckpointManager.cs index 64e4ac014cb..e6980c41e4c 100644 --- a/libs/server/GarnetCheckpointManager.cs +++ b/libs/server/GarnetCheckpointManager.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.IO; using System.Text; using Microsoft.Extensions.Logging; using Tsavorite.core; @@ -15,40 +16,73 @@ public class GarnetCheckpointManager : DeviceLogCommitCheckpointManager { public string CurrentHistoryId { get; set; } public string RecoveredHistoryId { get; set; } - public long CurrentSafeAofAddress { get; set; } - public long RecoveredSafeAofAddress { get; set; } + public AofAddress CurrentSafeAofAddress { get; private set; } + public AofAddress RecoveredSafeAofAddress { get; private set; } /// /// Create new instance of Garnet checkpoint manager /// + /// Number of sublog for Aof /// Factory for getting devices /// Checkpoint naming helper /// Remove older Tsavorite log commits /// FastCommit throttle frequency - use only in FastCommit mode /// Logger - public GarnetCheckpointManager(INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool removeOutdated = true, int fastCommitThrottleFreq = 0, ILogger logger = null) + public GarnetCheckpointManager(int AofPhysicalSublogCount, INamedDeviceFactoryCreator deviceFactoryCreator, ICheckpointNamingScheme checkpointNamingScheme, bool removeOutdated = true, int fastCommitThrottleFreq = 0, ILogger logger = null) : base(deviceFactoryCreator, checkpointNamingScheme, removeOutdated, fastCommitThrottleFreq, logger) { CurrentHistoryId = null; RecoveredHistoryId = null; - CurrentSafeAofAddress = 0; - RecoveredSafeAofAddress = 0; + CurrentSafeAofAddress = AofAddress.Create(AofPhysicalSublogCount, 0); + RecoveredSafeAofAddress = AofAddress.Create(AofPhysicalSublogCount, 0); } + /// + /// Set current AOF address + /// + /// + public void SetCurrentSafeAofAddress(ref AofAddress safeAofTailAddress) => CurrentSafeAofAddress = safeAofTailAddress; + + /// + /// Set recovered AOF address + /// + /// + public void SetRecoveredSafeAofAddress(ref AofAddress recoveredSafeAofAddress) => RecoveredSafeAofAddress = recoveredSafeAofAddress; + /// public override unsafe byte[] GetCookie() { if (CurrentHistoryId == null) return null; - var cookie = new byte[sizeof(int) + sizeof(long) + CurrentHistoryId.Length]; - var primaryReplIdBytes = Encoding.ASCII.GetBytes(CurrentHistoryId); - fixed (byte* ptr = cookie) - fixed (byte* pridPtr = primaryReplIdBytes) + + if (CurrentSafeAofAddress.Length == 1) + { + // Legacy single log serialization + var cookie = new byte[sizeof(int) + sizeof(long) + CurrentHistoryId.Length]; + var primaryReplIdBytes = Encoding.ASCII.GetBytes(CurrentHistoryId); + fixed (byte* ptr = cookie) + fixed (byte* pridPtr = primaryReplIdBytes) + { + *(int*)ptr = sizeof(long) + CurrentHistoryId.Length; + *(long*)(ptr + 4) = CurrentSafeAofAddress[0]; + Buffer.MemoryCopy(pridPtr, ptr + 12, primaryReplIdBytes.Length, primaryReplIdBytes.Length); + } + return cookie; + } + else { - *(int*)ptr = sizeof(long) + CurrentHistoryId.Length; - *(long*)(ptr + 4) = CurrentSafeAofAddress; - Buffer.MemoryCopy(pridPtr, ptr + 12, primaryReplIdBytes.Length, primaryReplIdBytes.Length); + // Multi-log serialization + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms, Encoding.ASCII); + + //1. Write history-Id + writer.Write(CurrentHistoryId == null ? 0 : 1); + if (CurrentHistoryId != null) writer.Write(CurrentHistoryId); + //2. Write checkpoint covered aof address + CurrentSafeAofAddress.Serialize(writer); + + var byteArray = ms.ToArray(); + return byteArray; } - return cookie; } } } \ No newline at end of file diff --git a/libs/server/GarnetDatabase.cs b/libs/server/GarnetDatabase.cs index 7de6d6f923f..141d42a78de 100644 --- a/libs/server/GarnetDatabase.cs +++ b/libs/server/GarnetDatabase.cs @@ -8,12 +8,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Represents a logical database in Garnet /// @@ -31,14 +25,9 @@ public class GarnetDatabase : IDisposable public int Id { get; } /// - /// Main Store + /// Store /// - public TsavoriteKV MainStore { get; } - - /// - /// Object Store - /// - public TsavoriteKV ObjectStore { get; } + public TsavoriteKV Store { get; } /// /// Epoch instance used by server @@ -51,19 +40,14 @@ public class GarnetDatabase : IDisposable public StateMachineDriver StateMachineDriver { get; } /// - /// Size Tracker for Object Store + /// Size Tracker /// - public CacheSizeTracker ObjectStoreSizeTracker { get; } - - /// - /// Device used for AOF logging - /// - public IDevice AofDevice { get; } + public CacheSizeTracker SizeTracker { get; } /// /// AOF log /// - public TsavoriteLog AppendOnlyFile { get; } + public GarnetAppendOnlyFile AppendOnlyFile { get; } /// /// Version map @@ -71,29 +55,19 @@ public class GarnetDatabase : IDisposable public WatchVersionMap VersionMap { get; } /// - /// Tail address of main store log at last save + /// Tail address of store log at last save /// public long LastSaveStoreTailAddress; - /// - /// Tail address of object store log at last save - /// - public long LastSaveObjectStoreTailAddress; - /// /// Last time checkpoint of database was taken /// public DateTimeOffset LastSaveTime; /// - /// True if database's main store index has maxed-out + /// True if database's store index has maxed-out /// - public bool MainStoreIndexMaxedOut; - - /// - /// True if database's object store index has maxed-out - /// - public bool ObjectStoreIndexMaxedOut; + public bool StoreIndexMaxedOut; /// /// Reader-Writer lock for database checkpointing @@ -109,71 +83,60 @@ public class GarnetDatabase : IDisposable public readonly VectorManager VectorManager; /// - /// Storage session intended for store-wide object collection operations + /// RangeIndex (BfTree) manager for this database. + /// Created early and passed to the store's GarnetRecordDisposer for eviction cleanup. /// - internal StorageSession ObjectStoreCollectionDbStorageSession; + public readonly RangeIndexManager RangeIndexManager; /// - /// Storage session intended for main-store expired key deletion operations + /// Storage session intended for store-wide object collection operations /// - internal StorageSession MainStoreExpiredKeyDeletionDbStorageSession; + internal StorageSession StoreCollectionDbStorageSession; /// - /// Storage session intended for object-store expired key deletion operations + /// Storage session intended for store expired key deletion operations /// - internal StorageSession ObjectStoreExpiredKeyDeletionDbStorageSession; - + internal StorageSession StoreExpiredKeyDeletionDbStorageSession; internal StorageSession HybridLogStatScanStorageSession; - readonly KVSettings KvSettings; - readonly KVSettings ObjKvSettings; + private KVSettings KvSettings; bool disposed = false; - public GarnetDatabase(int id, TsavoriteKV mainStore, - TsavoriteKV objectStore, - KVSettings kvSettings, KVSettings objKvSettings, - LightEpoch epoch, StateMachineDriver stateMachineDriver, - CacheSizeTracker objectStoreSizeTracker, IDevice aofDevice, TsavoriteLog appendOnlyFile, - bool mainStoreIndexMaxedOut, bool objectStoreIndexMaxedOut, VectorManager vectorManager) : this() + public GarnetDatabase(int id, TsavoriteKV store, KVSettings kvSettings, LightEpoch epoch, StateMachineDriver stateMachineDriver, + CacheSizeTracker sizeTracker, GarnetAppendOnlyFile appendOnlyFile, bool storeIndexMaxedOut, VectorManager vectorManager, RangeIndexManager rangeIndexManager) + : this() { Id = id; - MainStore = mainStore; - ObjectStore = objectStore; + Store = store; KvSettings = kvSettings; - ObjKvSettings = objKvSettings; Epoch = epoch; StateMachineDriver = stateMachineDriver; - ObjectStoreSizeTracker = objectStoreSizeTracker; - AofDevice = aofDevice; + SizeTracker = sizeTracker; AppendOnlyFile = appendOnlyFile; - MainStoreIndexMaxedOut = mainStoreIndexMaxedOut; - ObjectStoreIndexMaxedOut = objectStoreIndexMaxedOut; + StoreIndexMaxedOut = storeIndexMaxedOut; VectorManager = vectorManager; + RangeIndexManager = rangeIndexManager; } public GarnetDatabase(int id, GarnetDatabase srcDb, bool enableAof, bool copyLastSaveData = false) : this() { Id = id; - MainStore = srcDb.MainStore; - ObjectStore = srcDb.ObjectStore; + Store = srcDb.Store; KvSettings = srcDb.KvSettings; - ObjKvSettings = srcDb.ObjKvSettings; Epoch = srcDb.Epoch; StateMachineDriver = srcDb.StateMachineDriver; - ObjectStoreSizeTracker = srcDb.ObjectStoreSizeTracker; - AofDevice = enableAof ? srcDb.AofDevice : null; + SizeTracker = srcDb.SizeTracker; AppendOnlyFile = enableAof ? srcDb.AppendOnlyFile : null; - MainStoreIndexMaxedOut = srcDb.MainStoreIndexMaxedOut; - ObjectStoreIndexMaxedOut = srcDb.ObjectStoreIndexMaxedOut; + StoreIndexMaxedOut = srcDb.StoreIndexMaxedOut; VectorManager = srcDb.VectorManager; + RangeIndexManager = srcDb.RangeIndexManager; if (copyLastSaveData) { LastSaveTime = srcDb.LastSaveTime; LastSaveStoreTailAddress = srcDb.LastSaveStoreTailAddress; - LastSaveObjectStoreTailAddress = srcDb.LastSaveObjectStoreTailAddress; } } @@ -181,7 +144,6 @@ public GarnetDatabase() { VersionMap = new WatchVersionMap(DefaultVersionMapSize); LastSaveStoreTailAddress = 0; - LastSaveObjectStoreTailAddress = 0; LastSaveTime = DateTimeOffset.FromUnixTimeSeconds(0); } @@ -190,7 +152,9 @@ public GarnetDatabase() /// public void Dispose() { - if (disposed) return; + if (disposed) + return; + disposed = true; // Shutdown vector replays and cleanup operations VectorManager?.Dispose(); @@ -199,32 +163,14 @@ public void Dispose() while (!CheckpointingLock.TryWriteLock()) _ = Thread.Yield(); - MainStore?.Dispose(); - ObjectStore?.Dispose(); - + Store?.Dispose(); KvSettings?.LogDevice?.Dispose(); - if (ObjKvSettings != null) - { - ObjKvSettings.LogDevice?.Dispose(); - ObjKvSettings.ObjectLogDevice?.Dispose(); - } - - AofDevice?.Dispose(); + KvSettings?.ObjectLogDevice?.Dispose(); AppendOnlyFile?.Dispose(); - ObjectStoreCollectionDbStorageSession?.Dispose(); - MainStoreExpiredKeyDeletionDbStorageSession?.Dispose(); - ObjectStoreExpiredKeyDeletionDbStorageSession?.Dispose(); + StoreCollectionDbStorageSession?.Dispose(); + StoreExpiredKeyDeletionDbStorageSession?.Dispose(); - if (ObjectStoreSizeTracker != null) - { - // If tracker has previously started, wait for it to stop - if (!ObjectStoreSizeTracker.TryPreventStart()) - { - while (!ObjectStoreSizeTracker.Stopped) - Thread.Yield(); - } - } - disposed = true; + SizeTracker?.Stop(); } } } \ No newline at end of file diff --git a/libs/server/InputHeader.cs b/libs/server/InputHeader.cs index e2af4615f9d..deebfb6aa94 100644 --- a/libs/server/InputHeader.cs +++ b/libs/server/InputHeader.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Tsavorite.core; @@ -11,18 +12,12 @@ namespace Garnet.server { /// /// Flags used by append-only file (AOF/WAL) - /// The byte representation only use the last 3 bits of the byte since the lower 5 bits of the field used to store the flag stores other data in the case of Object types. + /// The byte representation only use the last 3 bits of the byte since the lower 5 bits of the "union" field that is used to store the flag stores other data (see RespInputHeader.FlagMask). /// In the case of a Rawstring, the last 4 bits are used for flags, and the other 4 bits are unused of the byte. - /// NOTE: This will soon be expanded as a part of a breaking change to make WithEtag bit compatible with object store as well. /// [Flags] public enum RespInputFlags : byte { - /// - /// Flag indicating an operation intending to add an etag for a RAWSTRING command. - /// - WithEtag = 16, - /// /// Flag indicating a SET operation that returns the previous value (for strings). /// @@ -49,8 +44,7 @@ public struct RespInputHeader /// public const int Size = 3; - // Since we know WithEtag is not used with any Object types, we keep the flag mask to work with the last 3 bits as flags, - // and the other 5 bits for storing object associated flags. However, in the case of Rawstring we use the last 4 bits for flags, and let the others remain unused. + // Flag mask separates the lower bits (used for object-associated sub-operation IDs) from the upper bits (used for RespInputFlags). internal const byte FlagMask = (byte)RespInputFlags.SetGet - 1; [FieldOffset(0)] @@ -67,6 +61,7 @@ public struct RespInputHeader /// /// Command /// Flags + [MethodImpl(MethodImplOptions.AggressiveInlining)] public RespInputHeader(RespCommand cmd, RespInputFlags flags = 0) { this.cmd = cmd; @@ -78,6 +73,7 @@ public RespInputHeader(RespCommand cmd, RespInputFlags flags = 0) /// /// Object type /// Flags + [MethodImpl(MethodImplOptions.AggressiveInlining)] public RespInputHeader(GarnetObjectType type, RespInputFlags flags = 0) { this.type = type; @@ -135,44 +131,16 @@ internal ListOperation ListOp /// internal unsafe void SetSetGetFlag() => flags |= RespInputFlags.SetGet; - /// - /// Set "WithEtag" flag for the input header - /// - internal void SetWithEtagFlag() => flags |= RespInputFlags.WithEtag; - - /// - /// Check if the WithEtag flag is set - /// - /// - internal bool CheckWithEtagFlag() => (flags & RespInputFlags.WithEtag) != 0; - - /// - /// Check that neither SetGet nor WithEtag flag is set - /// - internal bool NotSetGetNorCheckWithEtag() => (flags & (RespInputFlags.SetGet | RespInputFlags.WithEtag)) == 0; - /// /// Check if record is expired, either deterministically during log replay, /// or based on current time in normal operation. /// /// Expiration time /// - internal unsafe bool CheckExpiry(long expireTime) - { - if ((flags & RespInputFlags.Deterministic) != 0) - { - if ((flags & RespInputFlags.Expired) != 0) - return true; - } - else - { - if (expireTime < DateTimeOffset.Now.UtcTicks) - { - return true; - } - } - return false; - } + internal readonly unsafe bool CheckExpiry(long expireTime) + => (flags & RespInputFlags.Deterministic) != 0 + ? (flags & RespInputFlags.Expired) != 0 + : expireTime < DateTimeOffset.Now.UtcTicks; /// /// Check the SetGet flag @@ -188,9 +156,9 @@ internal unsafe bool CheckSetGetFlag() => (byte*)Unsafe.AsPointer(ref cmd); /// - /// Get header as SpanByte + /// Get header as PinnedSpanByte /// - public unsafe SpanByte SpanByte => new(Length, (nint)ToPointer()); + public unsafe PinnedSpanByte SpanByte => PinnedSpanByte.FromPinnedPointer(ToPointer(), Length); /// /// Get header length @@ -276,7 +244,7 @@ public unsafe int CopyTo(byte* dest, int length) var curr = dest; // Serialize header - header.SpanByte.CopyTo(curr); + header.SpanByte.SerializeTo(curr); curr += header.SpanByte.TotalSize; // Serialize arg1 @@ -289,7 +257,7 @@ public unsafe int CopyTo(byte* dest, int length) // Serialize parse state var remainingLength = length - (int)(curr - dest); - var len = parseState.CopyTo(curr, remainingLength); + var len = parseState.SerializeTo(curr, remainingLength); curr += len; // Number of serialized bytes @@ -302,10 +270,10 @@ public unsafe int DeserializeFrom(byte* src) var curr = src; // Deserialize header - ref var sbHeader = ref Unsafe.AsRef(curr); - ref var h = ref Unsafe.AsRef(sbHeader.ToPointer()); - curr += sbHeader.TotalSize; - header = h; + var header = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + ref var h = ref Unsafe.AsRef(header.ToPointer()); + curr += header.TotalSize; + this.header = h; // Deserialize arg1 arg1 = *(int*)curr; @@ -326,7 +294,7 @@ public unsafe int DeserializeFrom(byte* src) /// /// Header for Garnet Main Store inputs /// - public struct RawStringInput : IStoreInput + public struct StringInput : IStoreInput { /// /// Common input header for Garnet @@ -344,50 +312,176 @@ public struct RawStringInput : IStoreInput public SessionParseState parseState; /// - /// Create a new instance of RawStringInput + /// Create a new instance of StringInput /// /// Command /// Flags /// General-purpose argument - public RawStringInput(RespCommand cmd, RespInputFlags flags = 0, long arg1 = 0) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StringInput(RespCommand cmd, RespInputFlags flags = 0, long arg1 = 0) { this.header = new RespInputHeader(cmd, flags); this.arg1 = arg1; } /// - /// Create a new instance of RawStringInput + /// Create a new instance of StringInput + /// + /// Command + /// Flags + /// General-purpose argument + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StringInput(ushort cmd, byte flags = 0, long arg1 = 0) + : this((RespCommand)cmd, (RespInputFlags)flags, arg1) + { + } + + /// + /// Create a new instance of StringInput /// /// Command + /// Parse state + /// General-purpose argument /// Flags + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StringInput(RespCommand cmd, ref SessionParseState parseState, long arg1 = 0, RespInputFlags flags = 0) + : this(cmd, flags, arg1) + { + this.parseState = parseState; + } + + /// + /// Create a new instance of StringInput + /// + /// Command + /// Parse state + /// First command argument index in parse state /// General-purpose argument - public RawStringInput(ushort cmd, byte flags = 0, long arg1 = 0) : + /// Flags + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public StringInput(RespCommand cmd, ref SessionParseState parseState, int startIdx, long arg1 = 0, RespInputFlags flags = 0) + : this(cmd, flags, arg1) + { + this.parseState = parseState.Slice(startIdx); + } + + /// + public int SerializedLength => header.SpanByte.TotalSize + + sizeof(long) // arg1 + + parseState.GetSerializedLength(); + + /// + public unsafe int CopyTo(byte* dest, int length) + { + Debug.Assert(length >= this.SerializedLength); + + var curr = dest; + + // Serialize header + header.SpanByte.SerializeTo(curr); + curr += header.SpanByte.TotalSize; + + // Serialize arg1 + *(long*)curr = arg1; + curr += sizeof(long); + + // Serialize parse state + var remainingLength = length - (int)(curr - dest); + var len = parseState.SerializeTo(curr, remainingLength); + curr += len; + + // Serialize length + return (int)(curr - dest); + } + + /// + public unsafe int DeserializeFrom(byte* src) + { + var curr = src; + + // Deserialize header + var header = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + ref var h = ref Unsafe.AsRef(header.ToPointer()); + curr += header.TotalSize; + this.header = h; + + // Deserialize arg1 + arg1 = *(long*)curr; + curr += sizeof(long); + + // Deserialize parse state + var len = parseState.DeserializeFrom(curr); + curr += len; + + return (int)(curr - src); + } + } + + /// + /// Header for Garnet Unified Store inputs + /// + public struct UnifiedInput : IStoreInput + { + /// + /// Common input header for Garnet + /// + public RespInputHeader header; + + /// + /// Argument for generic usage by command implementation + /// + public long arg1; + + /// + /// Session parse state + /// + public SessionParseState parseState; + + /// + /// Create a new instance of UnifiedInput + /// + /// Command + /// Flags + /// General-purpose argument + public UnifiedInput(RespCommand cmd, RespInputFlags flags = 0, long arg1 = 0) + { + this.header = new RespInputHeader(cmd, flags); + this.arg1 = arg1; + } + + /// + /// Create a new instance of UnifiedInput + /// + /// Command + /// Flags + /// General-purpose argument + public UnifiedInput(ushort cmd, byte flags = 0, long arg1 = 0) : this((RespCommand)cmd, (RespInputFlags)flags, arg1) { } /// - /// Create a new instance of RawStringInput + /// Create a new instance of UnifiedInput /// /// Command /// Parse state /// General-purpose argument /// Flags - public RawStringInput(RespCommand cmd, ref SessionParseState parseState, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) + public UnifiedInput(RespCommand cmd, ref SessionParseState parseState, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) { this.parseState = parseState; } /// - /// Create a new instance of RawStringInput + /// Create a new instance of UnifiedInput /// /// Command /// Parse state /// First command argument index in parse state /// General-purpose argument /// Flags - public RawStringInput(RespCommand cmd, ref SessionParseState parseState, int startIdx, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) + public UnifiedInput(RespCommand cmd, ref SessionParseState parseState, int startIdx, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) { this.parseState = parseState.Slice(startIdx); } @@ -405,7 +499,7 @@ public unsafe int CopyTo(byte* dest, int length) var curr = dest; // Serialize header - header.SpanByte.CopyTo(curr); + header.SpanByte.SerializeTo(curr); curr += header.SpanByte.TotalSize; // Serialize arg1 @@ -414,7 +508,7 @@ public unsafe int CopyTo(byte* dest, int length) // Serialize parse state var remainingLength = length - (int)(curr - dest); - var len = parseState.CopyTo(curr, remainingLength); + var len = parseState.SerializeTo(curr, remainingLength); curr += len; // Serialize length @@ -427,10 +521,10 @@ public unsafe int DeserializeFrom(byte* src) var curr = src; // Deserialize header - ref var sbHeader = ref Unsafe.AsRef(curr); - ref var h = ref Unsafe.AsRef(sbHeader.ToPointer()); - curr += sbHeader.TotalSize; - header = h; + var header = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + ref var h = ref Unsafe.AsRef(header.ToPointer()); + curr += header.TotalSize; + this.header = h; // Deserialize arg1 arg1 = *(long*)curr; @@ -462,7 +556,7 @@ public struct CustomProcedureInput : IStoreInput public byte RespVersion { get; } /// - /// Create a new instance of RawStringInput + /// Create a new instance of StringInput /// /// Parse state /// RESP version for the session @@ -473,7 +567,7 @@ public CustomProcedureInput(ref SessionParseState parseState, byte respVersion) } /// - /// Create a new instance of RawStringInput + /// Create a new instance of StringInput /// /// Parse state /// First command argument index in parse state @@ -496,7 +590,7 @@ public unsafe int CopyTo(byte* dest, int length) // Serialize parse state var remainingLength = (int)(curr - dest); - var len = parseState.CopyTo(curr, remainingLength); + var len = parseState.SerializeTo(curr, remainingLength); curr += len; return (int)(curr - dest); @@ -512,24 +606,6 @@ public unsafe int DeserializeFrom(byte* src) } } - /// - /// Object output header (sometimes used as footer) - /// - [StructLayout(LayoutKind.Explicit, Size = Size)] - public struct ObjectOutputHeader - { - /// - /// Expected size of this object - /// - public const int Size = 4; - - /// - /// Some result of operation (e.g., number of items added successfully) - /// - [FieldOffset(0)] - public int result1; - } - /// /// Header for Garnet Main Store inputs but for Vector element r/w/d ops /// @@ -545,6 +621,13 @@ public struct VectorInput : IStoreInput public nint CallbackContext { get; set; } public nint Callback { get; set; } + public bool AlignmentExpected { get; set; } + + [MemberNotNullWhen(returnValue: true, member: nameof(MaxMigrationHeapAllocationSize))] + public bool IsMigrationRead => MaxMigrationHeapAllocationSize != null; + + public int? MaxMigrationHeapAllocationSize { get; set; } + public VectorInput() { } diff --git a/libs/server/LogCompactionType.cs b/libs/server/LogCompactionType.cs index 70774fdc688..500d0d52dfa 100644 --- a/libs/server/LogCompactionType.cs +++ b/libs/server/LogCompactionType.cs @@ -20,21 +20,18 @@ public enum LogCompactionType Shift, /// - /// Shift the begin address without compacting active records (data loss) - /// Immediately deletes files - do not use if you plan to recover after failure. + /// Lookup each record in compaction range, for record liveness checking using hash chain - no data loss + /// (to delete actual data files from disk, take a checkpoint after compaction). + /// Recommended for production use. /// - ShiftForced, + Lookup, /// /// Scan from untilAddress to read-only address to check for record liveness checking - no data loss - /// (to delete actual data files from disk, take a checkpoint after compaction) + /// (to delete actual data files from disk, take a checkpoint after compaction). + /// NOT RECOMMENDED: this strategy builds a temporary parallel KV index proportional to the keyspace, + /// causing significant transient memory use. Prefer Lookup. /// Scan, - - /// - /// Lookup each record in compaction range, for record liveness checking using hash chain - no data loss - /// (to delete actual data files from disk, take a checkpoint after compaction) - /// - Lookup, } } \ No newline at end of file diff --git a/libs/server/Lua/LuaCommands.cs b/libs/server/Lua/LuaCommands.cs index 07d6dcd41ab..59a3f47e78f 100644 --- a/libs/server/Lua/LuaCommands.cs +++ b/libs/server/Lua/LuaCommands.cs @@ -39,7 +39,7 @@ private unsafe bool TryEVALSHA() ScriptHashKey scriptKey = default; // Length check is mandatory, as ScriptHashKey assumes correct length - if (digest.length == SessionScriptCache.SHA1Len) + if (digest.Length == SessionScriptCache.SHA1Len) { tryAgain: scriptKey = new ScriptHashKey(digest.Span); @@ -208,7 +208,7 @@ private bool NetworkScriptExists() var exists = 0; // Length check is required, as ScriptHashKey makes a hard assumption - if (sha1.length == SessionScriptCache.SHA1Len) + if (sha1.Length == SessionScriptCache.SHA1Len) { AsciiUtils.ToLowerInPlace(sha1.Span); diff --git a/libs/server/Lua/LuaRunner.Functions.Struct.cs b/libs/server/Lua/LuaRunner.Functions.Struct.cs index d6d3c83649d..da67aac327d 100644 --- a/libs/server/Lua/LuaRunner.Functions.Struct.cs +++ b/libs/server/Lua/LuaRunner.Functions.Struct.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft Corporation. // The code is based on, not copied from, the Lua functions below. +// Licensed under the MIT license. + /****************************************************************************** * Copyright (C) 2010-2018 Lua.org, PUC-Rio. All rights reserved. * diff --git a/libs/server/Lua/LuaRunner.Functions.cs b/libs/server/Lua/LuaRunner.Functions.cs index e5b36a83dd8..56e230a6b21 100644 --- a/libs/server/Lua/LuaRunner.Functions.cs +++ b/libs/server/Lua/LuaRunner.Functions.cs @@ -16,6 +16,7 @@ using Garnet.common; using KeraLua; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.server { @@ -226,9 +227,7 @@ internal int UnsafeRunPreambleForSession(nint luaStatePtr) if (txnMode) { - txnKeyEntries.AddKey(key, false, Tsavorite.core.LockType.Exclusive); - if (!respServerSession.storageSession.objectStoreLockableContext.IsNull) - txnKeyEntries.AddKey(key, true, Tsavorite.core.LockType.Exclusive); + txnKeyEntries.AddKey(key, LockType.Exclusive); } // Equivalent to KEYS[i+1] = key @@ -317,7 +316,7 @@ internal int NoSessionResponse(nint luaStatePtr) /// Entry point for redis.call method from a Lua script (transactional mode) /// public int GarnetCallWithTransaction(nint luaStatePtr) - => ProcessCommandFromScripting(luaStatePtr, ref respServerSession.lockableGarnetApi); + => ProcessCommandFromScripting(luaStatePtr, ref respServerSession.transactionalGarnetApi); /// /// Entry point for redis.call method from a Lua script (non-transactional mode) @@ -3005,7 +3004,7 @@ internal int AclCheckCommand(nint luaStatePtr) static (RespCommand Parsed, bool BadArg) PrepareAndCheckRespRequest( ref LuaStateWrapper state, RespServerSession respServerSession, - ScratchBufferBuilder scratchBufferManager, + ScratchBufferBuilder scratchBufferBuilder, RespCommandsInfo cmdInfo, ReadOnlySpan cmdSpan, int luaArgCount @@ -3020,8 +3019,8 @@ int luaArgCount // RESP format the args so we can parse the command (and sub-command, and maybe keys down the line?) - scratchBufferManager.Reset(); - scratchBufferManager.StartCommand(cmdSpan, actualRespArgCount); + scratchBufferBuilder.Reset(); + scratchBufferBuilder.StartCommand(cmdSpan, actualRespArgCount); for (var i = 0; i < actualRespArgCount; i++) { @@ -3033,7 +3032,7 @@ int luaArgCount var argType = state.Type(stackIx); if (argType == LuaType.Nil) { - scratchBufferManager.WriteNullArgument(); + scratchBufferBuilder.WriteNullArgument(); } else if (argType is LuaType.String or LuaType.Number) { @@ -3043,7 +3042,7 @@ int luaArgCount state.KnownStringToBuffer(stackIx, out var span); // Span remains pinned so long as we don't pop the stack - scratchBufferManager.WriteArgument(span); + scratchBufferBuilder.WriteArgument(span); } else { @@ -3053,11 +3052,11 @@ int luaArgCount else { // For args we don't have, shove in an empty string - scratchBufferManager.WriteArgument(default); + scratchBufferBuilder.WriteArgument(default); } } - var request = scratchBufferManager.ViewFullArgSlice(); + var request = scratchBufferBuilder.ViewFullArgSlice(); var parsedCommand = respServerSession.ParseRespCommandBuffer(request.ReadOnlySpan); return (parsedCommand, false); @@ -3209,8 +3208,8 @@ private unsafe int ProcessCommandFromScripting(nint luaStatePtr, ref } // Note these spans are implicitly pinned, as they're actually on the Lua stack - var key = ArgSlice.FromPinnedSpan(keySpan); - var value = ArgSlice.FromPinnedSpan(valSpan); + var key = PinnedSpanByte.FromPinnedSpan(keySpan); + var value = PinnedSpanByte.FromPinnedSpan(valSpan); _ = api.SET(key, value); @@ -3243,8 +3242,8 @@ private unsafe int ProcessCommandFromScripting(nint luaStatePtr, ref } // Span is (implicitly) pinned since it's actually on the Lua stack - var key = ArgSlice.FromPinnedSpan(keySpan); - var status = api.GET(key, out var value); + var key = PinnedSpanByte.FromPinnedSpan(keySpan); + var status = api.GET(key, out PinnedSpanByte value); if (status == GarnetStatus.OK) { diff --git a/libs/server/Lua/LuaRunner.cs b/libs/server/Lua/LuaRunner.cs index cd5aea8dc2c..4b713322d33 100644 --- a/libs/server/Lua/LuaRunner.cs +++ b/libs/server/Lua/LuaRunner.cs @@ -263,7 +263,8 @@ public unsafe LuaRunner( delegate* unmanaged[Cdecl] garnetCall; if (txnMode) { - txnKeyEntries = new TxnKeyEntries(16, respServerSession.storageSession.lockableContext, respServerSession.storageSession.objectStoreLockableContext); + txnKeyEntries = new TxnKeyEntries(16, + respServerSession.storageSession.unifiedTransactionalContext); garnetCall = &LuaRunnerTrampolines.GarnetCallWithTransaction; } @@ -1237,9 +1238,8 @@ public unsafe object RunForRunner(string[] keys = null, string[] argv = null) foreach (var key in keys) { var _key = scratchBufferBuilder.CreateArgSlice(key); - txnKeyEntries.AddKey(_key, false, Tsavorite.core.LockType.Exclusive); - if (!respServerSession.storageSession.objectStoreLockableContext.IsNull) - txnKeyEntries.AddKey(_key, true, Tsavorite.core.LockType.Exclusive); + txnKeyEntries.AddKey(_key, Tsavorite.core.LockType.Exclusive); + scratchBufferBuilder.RewindScratchBuffer(_key); } adapter = new(scratchBufferBuilder); @@ -1337,25 +1337,31 @@ private void RunInTransaction(ref TResponse response) var txnVersion = respServerSession.storageSession.stateMachineDriver.AcquireTransactionVersion(); try { - respServerSession.storageSession.lockableContext.BeginLockable(); - if (!respServerSession.storageSession.objectStoreLockableContext.IsNull) - respServerSession.storageSession.objectStoreLockableContext.BeginLockable(); + respServerSession.storageSession.stringTransactionalContext.BeginTransaction(); + if (!respServerSession.storageSession.objectTransactionalContext.IsNull) + respServerSession.storageSession.objectTransactionalContext.BeginTransaction(); + if (!respServerSession.storageSession.unifiedTransactionalContext.IsNull) + respServerSession.storageSession.unifiedTransactionalContext.BeginTransaction(); respServerSession.SetTransactionMode(true); txnKeyEntries.LockAllKeys(); txnVersion = respServerSession.storageSession.stateMachineDriver.VerifyTransactionVersion(txnVersion); - respServerSession.storageSession.lockableContext.LocksAcquired(txnVersion); - if (!respServerSession.storageSession.objectStoreLockableContext.IsNull) - respServerSession.storageSession.objectStoreLockableContext.LocksAcquired(txnVersion); + respServerSession.storageSession.stringTransactionalContext.LocksAcquired(txnVersion); + if (!respServerSession.storageSession.objectTransactionalContext.IsNull) + respServerSession.storageSession.objectTransactionalContext.LocksAcquired(txnVersion); + if (!respServerSession.storageSession.unifiedTransactionalContext.IsNull) + respServerSession.storageSession.unifiedTransactionalContext.LocksAcquired(txnVersion); RunCommon(ref response); } finally { txnKeyEntries.UnlockAllKeys(); respServerSession.SetTransactionMode(false); - respServerSession.storageSession.lockableContext.EndLockable(); - if (!respServerSession.storageSession.objectStoreLockableContext.IsNull) - respServerSession.storageSession.objectStoreLockableContext.EndLockable(); + respServerSession.storageSession.stringTransactionalContext.EndTransaction(); + if (!respServerSession.storageSession.objectTransactionalContext.IsNull) + respServerSession.storageSession.objectTransactionalContext.EndTransaction(); + if (!respServerSession.storageSession.unifiedTransactionalContext.IsNull) + respServerSession.storageSession.unifiedTransactionalContext.EndTransaction(); respServerSession.storageSession.stateMachineDriver.EndTransaction(txnVersion); } } diff --git a/libs/server/Lua/ScratchBufferNetworkSender.cs b/libs/server/Lua/ScratchBufferNetworkSender.cs index bccd0cdcda6..a39933f5d56 100644 --- a/libs/server/Lua/ScratchBufferNetworkSender.cs +++ b/libs/server/Lua/ScratchBufferNetworkSender.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using Garnet.networking; +using Tsavorite.core; namespace Garnet.server { @@ -32,7 +33,7 @@ public ScratchBufferNetworkSender() scratchBufferBuilder = new(); } - public ArgSlice GetResponse() + public PinnedSpanByte GetResponse() => scratchBufferBuilder.ViewFullArgSlice(); public void Reset() diff --git a/libs/server/Metrics/GarnetServerMonitor.cs b/libs/server/Metrics/GarnetServerMonitor.cs index ed7131b8c02..4f03adcb9c0 100644 --- a/libs/server/Metrics/GarnetServerMonitor.cs +++ b/libs/server/Metrics/GarnetServerMonitor.cs @@ -83,7 +83,7 @@ public void Start() if (monitorSamplingFrequency > TimeSpan.Zero) { done.Reset(); - _ = MainMonitorTaskAsync(cts.Token); + _ = Task.Run(() => MainMonitorTaskAsync(cts.Token)); } } @@ -267,14 +267,11 @@ private void CleanupGlobalLatencyMetrics() private async Task MainMonitorTaskAsync(CancellationToken token) { - // Force async - await Task.Yield(); - try { while (true) { - await Task.Delay(monitorSamplingFrequency, token); + await Task.Delay(monitorSamplingFrequency, token).ConfigureAwait(false); // Reset the session level latency metrics for the prior version, as we are // about to make that the current version. diff --git a/libs/server/Metrics/Info/GarnetInfoMetrics.cs b/libs/server/Metrics/Info/GarnetInfoMetrics.cs index 5804b287adf..a014e0f3792 100644 --- a/libs/server/Metrics/Info/GarnetInfoMetrics.cs +++ b/libs/server/Metrics/Info/GarnetInfoMetrics.cs @@ -19,9 +19,7 @@ class GarnetInfoMetrics .Where(e => e switch { InfoMetricsType.STOREHASHTABLE => false, - InfoMetricsType.OBJECTSTOREHASHTABLE => false, InfoMetricsType.STOREREVIV => false, - InfoMetricsType.OBJECTSTOREREVIV => false, InfoMetricsType.HLOGSCAN => false, InfoMetricsType.COMMANDSTATS => false, _ => true @@ -38,11 +36,8 @@ class GarnetInfoMetrics MetricsItem[] replicationInfo = null; MetricsItem[] statsInfo = null; MetricsItem[][] storeInfo = null; - MetricsItem[][] objectStoreInfo = null; MetricsItem[][] storeHashDistrInfo = null; - MetricsItem[][] objectStoreHashDistrInfo = null; MetricsItem[][] storeRevivInfo = null; - MetricsItem[][] objectStoreRevivInfo = null; MetricsItem[][] persistenceInfo = null; MetricsItem[] clientsInfo = null; MetricsItem[] keyspaceInfo = null; @@ -74,21 +69,12 @@ private void PopulateServerInfo(StoreWrapper storeWrapper) private void PopulateMemoryInfo(StoreWrapper storeWrapper) { - var main_store_index_size = 0L; - var main_store_log_memory_size = 0L; - var main_store_read_cache_size = 0L; - long total_main_store_size; - - var disableObj = storeWrapper.serverOptions.DisableObjects; - - var initialSize = disableObj ? -1L : 0L; - var object_store_index_size = initialSize; - var object_store_log_memory_size = initialSize; - var object_store_read_cache_log_memory_size = initialSize; - var object_store_heap_memory_target_size = initialSize; - var object_store_heap_memory_size = initialSize; - var object_store_read_cache_heap_memory_size = initialSize; - var total_object_store_size = initialSize; + var store_index_size = 0L; + var store_mainlog_memory_target_size = 0L; + var store_mainlog_memory_size = 0L; + var store_readcache_memory_size = 0L; + var store_readcache_memory_target_size = 0L; + long total_store_size; var enableAof = storeWrapper.serverOptions.EnableAOF; var aof_log_memory_size = enableAof ? 0 : -1L; @@ -97,31 +83,27 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper) foreach (var db in databases) { - main_store_index_size += db.MainStore.IndexSize * 64; - main_store_log_memory_size += db.MainStore.Log.MemorySizeBytes; - main_store_read_cache_size += db.MainStore.ReadCache?.MemorySizeBytes ?? 0; + store_index_size += db.Store.IndexSize * 64; + aof_log_memory_size += db.AppendOnlyFile != null ? db.AppendOnlyFile.Log.MemorySizeBytes.AggregateDiff(0) : 0; - aof_log_memory_size += db.AppendOnlyFile?.MemorySizeBytes ?? 0; + if (db.SizeTracker?.mainLogTracker is null) + store_mainlog_memory_size += db.Store.Log.MemorySizeBytes; + else + { + store_mainlog_memory_target_size += db.SizeTracker?.mainLogTracker?.TargetSize ?? 0; + store_mainlog_memory_size += db.SizeTracker?.mainLogTracker.TotalSize ?? 0; + } - if (!disableObj) + if (db.SizeTracker?.mainLogTracker is null) + store_readcache_memory_size += db.Store.ReadCache?.MemorySizeBytes ?? 0; + else { - object_store_index_size += db.ObjectStore.IndexSize * 64; - object_store_log_memory_size += db.ObjectStore.Log.MemorySizeBytes; - object_store_read_cache_log_memory_size += db.ObjectStore.ReadCache?.MemorySizeBytes ?? 0; - object_store_heap_memory_target_size += db.ObjectStoreSizeTracker?.mainLogTracker.TargetSize ?? 0; - object_store_heap_memory_size += db.ObjectStoreSizeTracker?.mainLogTracker.LogHeapSizeBytes ?? 0; - object_store_read_cache_heap_memory_size += db.ObjectStoreSizeTracker?.readCacheTracker?.LogHeapSizeBytes ?? 0; + store_readcache_memory_target_size += db.SizeTracker?.readCacheTracker?.TargetSize ?? 0; + store_readcache_memory_size += db.SizeTracker?.readCacheTracker?.TotalSize ?? 0; } } - total_main_store_size = main_store_index_size + main_store_log_memory_size + main_store_read_cache_size; - - if (!disableObj) - { - total_object_store_size = object_store_index_size + object_store_log_memory_size + - object_store_read_cache_log_memory_size + object_store_heap_memory_size + - object_store_read_cache_heap_memory_size; - } + total_store_size = store_index_size + store_mainlog_memory_size + store_readcache_memory_size; var gcMemoryInfo = GC.GetGCMemoryInfo(); var gcAvailableMemory = gcMemoryInfo.TotalCommittedBytes - gcMemoryInfo.HeapSizeBytes; @@ -153,17 +135,11 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper) new("gc_heap_bytes", gcMemoryInfo.HeapSizeBytes.ToString()), new("gc_managed_memory_bytes_excluding_heap", gcAvailableMemory.ToString()), new("gc_fragmented_bytes", gcMemoryInfo.FragmentedBytes.ToString()), - new("main_store_index_size", main_store_index_size.ToString()), - new("main_store_log_memory_size", main_store_log_memory_size.ToString()), - new("main_store_read_cache_size", main_store_read_cache_size.ToString()), - new("total_main_store_size", total_main_store_size.ToString()), - new("object_store_index_size", object_store_index_size.ToString()), - new("object_store_log_memory_size", object_store_log_memory_size.ToString()), - new("object_store_heap_memory_target_size", object_store_heap_memory_target_size.ToString()), - new("object_store_heap_memory_size", object_store_heap_memory_size.ToString()), - new("object_store_read_cache_log_memory_size", object_store_read_cache_log_memory_size.ToString()), - new("object_store_read_cache_heap_memory_size", object_store_read_cache_heap_memory_size.ToString()), - new("total_object_store_size", total_object_store_size.ToString()), + new("store_index_size", store_index_size.ToString()), + new("store_mainlog_memory_size", store_mainlog_memory_size.ToString()), + new("store_readcache_memory_size", store_readcache_memory_size.ToString()), + new("total_main_store_size", total_store_size.ToString()), + new("store_heap_memory_target_size", store_mainlog_memory_target_size.ToString()), new("aof_memory_size", aof_log_memory_size.ToString()) ]; } @@ -188,8 +164,6 @@ private void PopulateReplicationInfo(StoreWrapper storeWrapper) new("second_repl_offset", "N/A"), new("store_current_safe_aof_address", "N/A"), new("store_recovered_safe_aof_address", "N/A"), - new("object_store_current_safe_aof_address", "N/A"), - new("object_store_recovered_safe_aof_address", "N/A") ]; } else @@ -317,62 +291,26 @@ private void PopulateStoreStats(StoreWrapper storeWrapper) private MetricsItem[] GetDatabaseStoreStats(StoreWrapper storeWrapper, GarnetDatabase db) => [ - new($"CurrentVersion", db.MainStore.CurrentVersion.ToString()), - new($"LastCheckpointedVersion", db.MainStore.LastCheckpointedVersion.ToString()), - new($"SystemState", db.MainStore.SystemState.ToString()), - new($"IndexSize", db.MainStore.IndexSize.ToString()), + new($"CurrentVersion", db.Store.CurrentVersion.ToString()), + new($"LastCheckpointedVersion", db.Store.LastCheckpointedVersion.ToString()), + new($"SystemState", db.Store.SystemState.ToString()), + new($"IndexMemorySize", db.Store.IndexSize.ToString()), new($"LogDir", storeWrapper.serverOptions.LogDir), - new($"Log.BeginAddress", db.MainStore.Log.BeginAddress.ToString()), - new($"Log.BufferSize", db.MainStore.Log.BufferSize.ToString()), - new($"Log.EmptyPageCount", db.MainStore.Log.EmptyPageCount.ToString()), - new($"Log.MinEmptyPageCount", db.MainStore.Log.MinEmptyPageCount.ToString()), - new($"Log.FixedRecordSize", db.MainStore.Log.FixedRecordSize.ToString()), - new($"Log.HeadAddress", db.MainStore.Log.HeadAddress.ToString()), - new($"Log.MemorySizeBytes", db.MainStore.Log.MemorySizeBytes.ToString()), - new($"Log.SafeReadOnlyAddress", db.MainStore.Log.SafeReadOnlyAddress.ToString()), - new($"Log.TailAddress", db.MainStore.Log.TailAddress.ToString()), - new($"ReadCache.BeginAddress", db.MainStore.ReadCache?.BeginAddress.ToString() ?? "N/A"), - new($"ReadCache.BufferSize", db.MainStore.ReadCache?.BufferSize.ToString() ?? "N/A"), - new($"ReadCache.EmptyPageCount", db.MainStore.ReadCache?.EmptyPageCount.ToString() ?? "N/A"), - new($"ReadCache.HeadAddress", db.MainStore.ReadCache?.HeadAddress.ToString() ?? "N/A"), - new($"ReadCache.MemorySizeBytes", db.MainStore.ReadCache?.MemorySizeBytes.ToString() ?? "N/A"), - new($"ReadCache.TailAddress", db.MainStore.ReadCache?.TailAddress.ToString() ?? "N/A"), - ]; - - private void PopulateObjectStoreStats(StoreWrapper storeWrapper) - { - var databases = storeWrapper.GetDatabasesSnapshot(); - - objectStoreInfo = new MetricsItem[storeWrapper.MaxDatabaseId + 1][]; - foreach (var db in databases) - { - var storeStats = GetDatabaseObjectStoreStats(storeWrapper, db); - objectStoreInfo[db.Id] = storeStats; - } - } - - private MetricsItem[] GetDatabaseObjectStoreStats(StoreWrapper storeWrapper, GarnetDatabase db) => - [ - new($"CurrentVersion", db.ObjectStore.CurrentVersion.ToString()), - new($"LastCheckpointedVersion", db.ObjectStore.LastCheckpointedVersion.ToString()), - new($"SystemState", db.ObjectStore.SystemState.ToString()), - new($"IndexSize", db.ObjectStore.IndexSize.ToString()), - new($"LogDir", storeWrapper.serverOptions.LogDir), - new($"Log.BeginAddress", db.ObjectStore.Log.BeginAddress.ToString()), - new($"Log.BufferSize", db.ObjectStore.Log.BufferSize.ToString()), - new($"Log.EmptyPageCount", db.ObjectStore.Log.EmptyPageCount.ToString()), - new($"Log.MinEmptyPageCount", db.ObjectStore.Log.MinEmptyPageCount.ToString()), - new($"Log.FixedRecordSize", db.ObjectStore.Log.FixedRecordSize.ToString()), - new($"Log.HeadAddress", db.ObjectStore.Log.HeadAddress.ToString()), - new($"Log.MemorySizeBytes", db.ObjectStore.Log.MemorySizeBytes.ToString()), - new($"Log.SafeReadOnlyAddress", db.ObjectStore.Log.SafeReadOnlyAddress.ToString()), - new($"Log.TailAddress", db.ObjectStore.Log.TailAddress.ToString()), - new($"ReadCache.BeginAddress", db.ObjectStore.ReadCache?.BeginAddress.ToString() ?? "N/A"), - new($"ReadCache.BufferSize", db.ObjectStore.ReadCache?.BufferSize.ToString() ?? "N/A"), - new($"ReadCache.EmptyPageCount", db.ObjectStore.ReadCache?.EmptyPageCount.ToString() ?? "N/A"), - new($"ReadCache.HeadAddress", db.ObjectStore.ReadCache?.HeadAddress.ToString() ?? "N/A"), - new($"ReadCache.MemorySizeBytes", db.ObjectStore.ReadCache?.MemorySizeBytes.ToString() ?? "N/A"), - new($"ReadCache.TailAddress", db.ObjectStore.ReadCache?.TailAddress.ToString() ?? "N/A"), + new($"Log.BeginAddress", db.Store.Log.BeginAddress.ToString()), + new($"Log.BufferSize", db.Store.Log.BufferSize.ToString()), + new($"Log.AllocatedPageCount", db.Store.Log.AllocatedPageCount.ToString()), + new($"Log.HeadAddress", db.Store.Log.HeadAddress.ToString()), + new($"Log.MemorySizeBytes", db.Store.Log.MemorySizeBytes.ToString()), + new($"Log.HeapSizeBytes", db.Store.Log.HeapSizeBytes.ToString()), + new($"Log.SafeReadOnlyAddress", db.Store.Log.SafeReadOnlyAddress.ToString()), + new($"Log.TailAddress", db.Store.Log.TailAddress.ToString()), + new($"ReadCache.BeginAddress", db.Store.ReadCache?.BeginAddress.ToString() ?? "N/A"), + new($"ReadCache.BufferSize", db.Store.ReadCache?.BufferSize.ToString() ?? "N/A"), + new($"ReadCache.AllocatedPageCount", db.Store.ReadCache?.AllocatedPageCount.ToString() ?? "N/A"), + new($"ReadCache.HeadAddress", db.Store.ReadCache?.HeadAddress.ToString() ?? "N/A"), + new($"ReadCache.MemorySizeBytes", db.Store.ReadCache?.MemorySizeBytes.ToString() ?? "N/A"), + new($"ReadCache.HeapSizeBytes", db.Store.ReadCache?.HeapSizeBytes.ToString() ?? "N/A"), + new($"ReadCache.TailAddress", db.Store.ReadCache?.TailAddress.ToString() ?? "N/A"), ]; private void PopulateStoreHashDistribution(StoreWrapper storeWrapper) @@ -382,18 +320,7 @@ private void PopulateStoreHashDistribution(StoreWrapper storeWrapper) storeHashDistrInfo = new MetricsItem[storeWrapper.MaxDatabaseId + 1][]; foreach (var db in databases) { - storeHashDistrInfo[db.Id] = [new("", db.MainStore.DumpDistribution())]; - } - } - - private void PopulateObjectStoreHashDistribution(StoreWrapper storeWrapper) - { - var databases = storeWrapper.GetDatabasesSnapshot(); - - objectStoreHashDistrInfo = new MetricsItem[storeWrapper.MaxDatabaseId + 1][]; - foreach (var db in databases) - { - objectStoreHashDistrInfo[db.Id] = [new("", db.ObjectStore.DumpDistribution())]; + storeHashDistrInfo[db.Id] = [new("", db.Store.DumpDistribution())]; } } @@ -404,18 +331,7 @@ private void PopulateStoreRevivInfo(StoreWrapper storeWrapper) storeRevivInfo = new MetricsItem[storeWrapper.MaxDatabaseId + 1][]; foreach (var db in databases) { - storeRevivInfo[db.Id] = [new("", db.MainStore.DumpRevivificationStats())]; - } - } - - private void PopulateObjectStoreRevivInfo(StoreWrapper storeWrapper) - { - var databases = storeWrapper.GetDatabasesSnapshot(); - - objectStoreRevivInfo = new MetricsItem[storeWrapper.MaxDatabaseId + 1][]; - foreach (var db in databases) - { - objectStoreRevivInfo[db.Id] = [new("", db.ObjectStore.DumpRevivificationStats())]; + storeRevivInfo[db.Id] = [new("", db.Store.DumpRevivificationStats())]; } } @@ -437,11 +353,11 @@ private MetricsItem[] GetDatabasePersistenceStats(StoreWrapper storeWrapper, Gar return [ - new($"CommittedBeginAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.CommittedBeginAddress.ToString()), - new($"CommittedUntilAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.CommittedUntilAddress.ToString()), - new($"FlushedUntilAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.FlushedUntilAddress.ToString()), - new($"BeginAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.BeginAddress.ToString()), - new($"TailAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.TailAddress.ToString()), + new($"CommittedBeginAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.Log.CommittedBeginAddress.ToString()), + new($"CommittedUntilAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.Log.CommittedUntilAddress.ToString()), + new($"FlushedUntilAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.Log.FlushedUntilAddress.ToString()), + new($"BeginAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.Log.BeginAddress.ToString()), + new($"TailAddress", !aofEnabled ? "N/A" : db.AppendOnlyFile.Log.TailAddress.ToString()), new($"SafeAofAddress", !aofEnabled ? "N/A" : storeWrapper.safeAofAddress.ToString()) ]; } @@ -504,12 +420,9 @@ public static string GetSectionHeader(InfoMetricsType infoType, int dbId) InfoMetricsType.CLUSTER => "Cluster", InfoMetricsType.REPLICATION => "Replication", InfoMetricsType.STATS => "Stats", - InfoMetricsType.STORE => $"MainStore_DB_{dbId}", - InfoMetricsType.OBJECTSTORE => $"ObjectStore_DB_{dbId}", - InfoMetricsType.STOREHASHTABLE => $"MainStoreHashTableDistribution_DB_{dbId}", - InfoMetricsType.OBJECTSTOREHASHTABLE => $"ObjectStoreHashTableDistribution_DB_{dbId}", - InfoMetricsType.STOREREVIV => $"MainStoreDeletedRecordRevivification_DB_{dbId}", - InfoMetricsType.OBJECTSTOREREVIV => $"ObjectStoreDeletedRecordRevivification_DB_{dbId}", + InfoMetricsType.STORE => $"Store_DB_{dbId}", + InfoMetricsType.STOREHASHTABLE => $"StoreHashTableDistribution_DB_{dbId}", + InfoMetricsType.STOREREVIV => $"StoreDeletedRecordRevivification_DB_{dbId}", InfoMetricsType.PERSISTENCE => $"Persistence_DB_{dbId}", InfoMetricsType.CLIENTS => "Clients", InfoMetricsType.KEYSPACE => "Keyspace", @@ -571,32 +484,14 @@ private void GetRespInfo(InfoMetricsType section, int dbId, StoreWrapper storeWr PopulateStoreStats(storeWrapper); GetSectionRespInfo(header, storeInfo[dbId], sbResponse); return; - case InfoMetricsType.OBJECTSTORE: - if (storeWrapper.serverOptions.DisableObjects) - return; - PopulateObjectStoreStats(storeWrapper); - GetSectionRespInfo(header, objectStoreInfo[dbId], sbResponse); - return; case InfoMetricsType.STOREHASHTABLE: PopulateStoreHashDistribution(storeWrapper); GetSectionRespInfo(header, storeHashDistrInfo[dbId], sbResponse); return; - case InfoMetricsType.OBJECTSTOREHASHTABLE: - if (storeWrapper.serverOptions.DisableObjects) - return; - PopulateObjectStoreHashDistribution(storeWrapper); - GetSectionRespInfo(header, objectStoreHashDistrInfo[dbId], sbResponse); - return; case InfoMetricsType.STOREREVIV: PopulateStoreRevivInfo(storeWrapper); GetSectionRespInfo(header, storeRevivInfo[dbId], sbResponse); return; - case InfoMetricsType.OBJECTSTOREREVIV: - if (storeWrapper.serverOptions.DisableObjects) - return; - PopulateObjectStoreRevivInfo(storeWrapper); - GetSectionRespInfo(header, objectStoreRevivInfo[dbId], sbResponse); - return; case InfoMetricsType.PERSISTENCE: if (!storeWrapper.serverOptions.EnableAOF) return; @@ -671,27 +566,12 @@ private MetricsItem[] GetMetricInternal(InfoMetricsType section, int dbId, Store case InfoMetricsType.STORE: PopulateStoreStats(storeWrapper); return storeInfo[dbId]; - case InfoMetricsType.OBJECTSTORE: - if (storeWrapper.serverOptions.DisableObjects) - return null; - PopulateObjectStoreStats(storeWrapper); - return objectStoreInfo[dbId]; case InfoMetricsType.STOREHASHTABLE: PopulateStoreHashDistribution(storeWrapper); return storeHashDistrInfo[dbId]; - case InfoMetricsType.OBJECTSTOREHASHTABLE: - if (storeWrapper.serverOptions.DisableObjects) - return null; - PopulateObjectStoreHashDistribution(storeWrapper); - return objectStoreHashDistrInfo[dbId]; case InfoMetricsType.STOREREVIV: PopulateStoreRevivInfo(storeWrapper); return storeRevivInfo[dbId]; - case InfoMetricsType.OBJECTSTOREREVIV: - if (storeWrapper.serverOptions.DisableObjects) - return null; - PopulateObjectStoreRevivInfo(storeWrapper); - return objectStoreRevivInfo[dbId]; case InfoMetricsType.PERSISTENCE: if (!storeWrapper.serverOptions.EnableAOF) return null; diff --git a/libs/server/Metrics/Info/InfoCommand.cs b/libs/server/Metrics/Info/InfoCommand.cs index 684fa80a343..951f64ac876 100644 --- a/libs/server/Metrics/Info/InfoCommand.cs +++ b/libs/server/Metrics/Info/InfoCommand.cs @@ -80,7 +80,6 @@ private bool NetworkINFO() } } return true; - } private void GetHelpMessage() diff --git a/libs/server/Metrics/Info/InfoHelp.cs b/libs/server/Metrics/Info/InfoHelp.cs index f2de3e79bed..501b81b6c32 100644 --- a/libs/server/Metrics/Info/InfoHelp.cs +++ b/libs/server/Metrics/Info/InfoHelp.cs @@ -25,11 +25,8 @@ public static List GetInfoTypeHelpMessage() $"{nameof(InfoMetricsType.REPLICATION)}: Replication info.", $"{nameof(InfoMetricsType.STATS)}: General server operational stats.", $"{nameof(InfoMetricsType.STORE)}: Main store operational information.", - $"{nameof(InfoMetricsType.OBJECTSTORE)}: Object store operational information.", $"{nameof(InfoMetricsType.STOREHASHTABLE)}: Hash table distribution info for main store (expensive, not returned by default).", - $"{nameof(InfoMetricsType.OBJECTSTOREHASHTABLE)}: Hash table distribution info for object store (expensive, not returned by default).", $"{nameof(InfoMetricsType.STOREREVIV)}: Revivification info for deleted records in main store (not returned by default).", - $"{nameof(InfoMetricsType.OBJECTSTOREREVIV)}: Record revivification info for deleted records in object store (not returned by default).", $"{nameof(InfoMetricsType.PERSISTENCE)}: Persistence related information (i.e. Checkpoint and AOF).", $"{nameof(InfoMetricsType.CLIENTS)}: Information related to client connections.", $"{nameof(InfoMetricsType.KEYSPACE)}: Database related statistics.", diff --git a/libs/server/Metrics/Slowlog/RespSlowlogCommands.cs b/libs/server/Metrics/Slowlog/RespSlowlogCommands.cs index 8dc86db2ee7..13c19de8202 100644 --- a/libs/server/Metrics/Slowlog/RespSlowlogCommands.cs +++ b/libs/server/Metrics/Slowlog/RespSlowlogCommands.cs @@ -166,7 +166,7 @@ void HandleSlowLog(RespCommand cmd) byte[] args = new byte[len]; fixed (byte* argsPtr = args) { - parseState.CopyTo(argsPtr, len); + parseState.SerializeTo(argsPtr, len); } entry.Arguments = args; } diff --git a/libs/server/Module/ModuleUtils.cs b/libs/server/Module/ModuleUtils.cs index cb9eb922806..f8b048fc3a7 100644 --- a/libs/server/Module/ModuleUtils.cs +++ b/libs/server/Module/ModuleUtils.cs @@ -8,12 +8,124 @@ using System.Reflection; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; +using System.Runtime.Loader; using Garnet.common; namespace Garnet.server { public class ModuleUtils { + /// + /// Loads only the assemblies that the module at references (transitively) + /// from , instead of loading every DLL in the directory. + /// + /// Full path to the module assembly file + /// Directory containing the module and its dependencies + /// List of allowed paths for loading assemblies from + /// True if loading unsigned assemblies is allowed + /// Assemblies that were loaded + /// Error message on failure + /// True if all required dependencies were loaded successfully + public static bool LoadModuleDependencies( + string modulePath, + string binPath, + string[] allowedExtensionPaths, + bool allowUnsignedAssemblies, + out IEnumerable loadedAssemblies, + out ReadOnlySpan errorMessage) + { + loadedAssemblies = null; + errorMessage = default; + + // Read referenced assembly names from the module without loading it into the runtime + AssemblyName[] referencedNames; + try + { + referencedNames = GetReferencedAssemblyNames(modulePath); + } + catch + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_LOADING_ASSEMBLIES; + return false; + } + + if (referencedNames.Length == 0) + return true; + + // Build a map of available DLLs in binPath (filename without extension → full path) + var availableFiles = new Dictionary(StringComparer.Ordinal); + foreach (var file in Directory.GetFiles(binPath, "*.dll", SearchOption.TopDirectoryOnly)) + _ = availableFiles.TryAdd(Path.GetFileNameWithoutExtension(file), file); + + // Collect the set of dependency files to load (transitive closure) + var alreadyLoaded = new HashSet( + AssemblyLoadContext.Default.Assemblies + .Where(a => !a.IsDynamic && a.GetName().Name != null) + .Select(a => a.GetName().Name!), + StringComparer.Ordinal); + + var toLoad = new List(); + var visited = new HashSet(StringComparer.Ordinal); + var queue = new Queue(referencedNames); + + while (queue.TryDequeue(out var asmName)) + { + var name = asmName.Name; + if (name == null || !visited.Add(name)) + continue; + + // Skip if already loaded in the runtime or not available in binPath (it may be a framework assembly) + if (alreadyLoaded.Contains(name) || !availableFiles.TryGetValue(name, out var filePath)) + continue; + + toLoad.Add(filePath); + + // Walk transitive dependencies + try + { + foreach (var t in GetReferencedAssemblyNames(filePath)) + queue.Enqueue(t); + } + catch + { + // If we can't read metadata, we'll still try to load the file + } + } + + if (toLoad.Count == 0) + return true; + + return LoadAssemblies(toLoad, allowedExtensionPaths, allowUnsignedAssemblies, + out loadedAssemblies, out errorMessage, ignoreAssemblyLoadErrors: true, ignorePathCheckWhenUndefined: true); + } + + /// + /// Reads referenced assembly names from an assembly file using metadata, without loading it into the runtime. + /// + private static AssemblyName[] GetReferencedAssemblyNames(string assemblyPath) + { + using var fs = File.OpenRead(assemblyPath); + using var peReader = new PEReader(fs); + + if (!peReader.HasMetadata) + return []; + + var metadataReader = peReader.GetMetadataReader(); + var refs = new List(); + + foreach (var refHandle in metadataReader.AssemblyReferences) + { + var asmRef = metadataReader.GetAssemblyReference(refHandle); + refs.Add(new AssemblyName + { + Name = metadataReader.GetString(asmRef.Name), + Version = asmRef.Version + }); + } + + return [.. refs]; + } + /// /// Load assemblies from specified binary paths /// diff --git a/libs/server/Objects/Hash/HashObject.cs b/libs/server/Objects/Hash/HashObject.cs index c2b1286281b..b97a64ce0b3 100644 --- a/libs/server/Objects/Hash/HashObject.cs +++ b/libs/server/Objects/Hash/HashObject.cs @@ -75,8 +75,8 @@ private bool HasExpirableItems /// /// Constructor /// - public HashObject(long expiration = 0) - : base(expiration, MemoryUtils.DictionaryOverhead) + public HashObject() + : base(MemoryUtils.DictionaryOverhead) { hash = new Dictionary(ByteArrayComparer.Instance); #if NET9_0_OR_GREATER @@ -113,7 +113,7 @@ public HashObject(BinaryReader reader) InitializeExpirationStructures(); expirationTimes.Add(item, expiration); expirationQueue.Enqueue(item, expiration); - UpdateExpirationSize(item, true); + UpdateExpirationSize(add: true); } } else @@ -121,15 +121,16 @@ public HashObject(BinaryReader reader) hash.Add(item, value); } - this.UpdateSize(item, value); + // Expiration has already been added via UpdateExpirationSize if hasExpiration + UpdateSize(item, value, add: true); } } /// /// Copy constructor /// - public HashObject(Dictionary hash, Dictionary expirationTimes, PriorityQueue expirationQueue, long expiration, long size) - : base(expiration, size) + public HashObject(Dictionary hash, Dictionary expirationTimes, PriorityQueue expirationQueue, long heapMemorySize) + : base(heapMemorySize) { this.hash = hash; this.expirationTimes = expirationTimes; @@ -149,7 +150,7 @@ public override void DoSerialize(BinaryWriter writer) DeleteExpiredItems(); - int count = hash.Count; // Since expired items are already deleted, no need to worry about expiring items + var count = hash.Count; // Since expired items are already deleted, no need to worry about expiring items writer.Write(count); foreach (var kvp in hash) { @@ -178,23 +179,19 @@ public override void DoSerialize(BinaryWriter writer) public override void Dispose() { } /// - public override GarnetObjectBase Clone() => new HashObject(hash, expirationTimes, expirationQueue, Expiration, Size); + public override GarnetObjectBase Clone() => new HashObject(hash, expirationTimes, expirationQueue, HeapMemorySize); /// - public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, - byte respProtocolVersion, out long sizeChange) + public override bool Operate(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { - sizeChange = 0; - if (input.header.type != GarnetObjectType.Hash) { //Indicates when there is an incorrect type - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; output.SpanByteAndMemory.Length = 0; return true; } - var previousSize = this.Size; switch (input.header.HashOp) { case HashOperation.HSET: @@ -261,20 +258,24 @@ public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput throw new GarnetException($"Unsupported operation {input.header.HashOp} in HashObject.Operate"); } - sizeChange = this.Size - previousSize; - if (hash.Count == 0) - output.OutputFlags |= ObjectStoreOutputFlags.RemoveKey; + output.OutputFlags |= ObjectOutputFlags.RemoveKey; return true; } - private void UpdateSize(ReadOnlySpan key, ReadOnlySpan value, bool add = true) + private void UpdateSize(ReadOnlySpan key, ReadOnlySpan value, bool add) { - var size = Utility.RoundUp(key.Length, IntPtr.Size) + Utility.RoundUp(value.Length, IntPtr.Size) + var memorySize = Utility.RoundUp(key.Length, IntPtr.Size) + Utility.RoundUp(value.Length, IntPtr.Size) + (2 * MemoryUtils.ByteArrayOverhead) + MemoryUtils.DictionaryEntryOverhead; - this.Size += add ? size : -size; - Debug.Assert(this.Size >= MemoryUtils.DictionaryOverhead); + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.DictionaryOverhead); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -287,26 +288,34 @@ private void InitializeExpirationStructures() #if NET9_0_OR_GREATER expirationTimeSpanLookup = expirationTimes.GetAlternateLookup>(); #endif - this.Size += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + HeapMemorySize += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void UpdateExpirationSize(ReadOnlySpan key, bool add = true) + private void UpdateExpirationSize(bool add, bool includePQ = true) { // Account for dictionary entry and priority queue entry - var size = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead - + IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; - this.Size += add ? size : -size; + var memorySize = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; + if (includePQ) + memorySize += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(this.HeapMemorySize >= MemoryUtils.DictionaryOverhead); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void CleanupExpirationStructures() + private void CleanupExpirationStructuresIfEmpty() { if (expirationTimes.Count == 0) { - this.Size -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead) * expirationQueue.Count; - this.Size -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + HeapMemorySize -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead) * expirationQueue.Count; + HeapMemorySize -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; expirationTimes = null; expirationQueue = null; #if NET9_0_OR_GREATER @@ -319,7 +328,7 @@ private void CleanupExpirationStructures() public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false) { cursor = start; - items = new List(); + items = []; if (hash.Count < start) { @@ -329,7 +338,7 @@ public override unsafe void Scan(long start, out List items, out long cu // Hashset has key and value, so count is multiplied by 2 count = isNoValue ? count : count * 2; - int index = 0; + var index = 0; var expiredKeysCount = 0; foreach (var item in hash) { @@ -349,9 +358,7 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); if (!isNoValue) - { items.Add(item.Value); - } } else { @@ -361,9 +368,7 @@ public override unsafe void Scan(long start, out List items, out long cu { items.Add(item.Key); if (!isNoValue) - { items.Add(item.Value); - } } } } @@ -372,7 +377,6 @@ public override unsafe void Scan(long start, out List items, out long cu if (items.Count == count) break; - } // Indicates end of collection has been reached. @@ -398,28 +402,30 @@ private void DeleteExpiredItems() private void DeleteExpiredItemsWorker() { + // The PQ is ordered such that oldest items are dequeued first while (expirationQueue.TryPeek(out var key, out var expiration) && expiration < DateTimeOffset.UtcNow.Ticks) { // expirationTimes and expirationQueue will be out of sync when user is updating the expire time of key which already has some TTL. // PriorityQueue Doesn't have update option, so we will just enqueue the new expiration and already treat expirationTimes as the source of truth if (expirationTimes.TryGetValue(key, out var actualExpiration) && actualExpiration == expiration) { - expirationTimes.Remove(key); - expirationQueue.Dequeue(); - UpdateExpirationSize(key, false); + _ = expirationTimes.Remove(key); + _ = expirationQueue.Dequeue(); + UpdateExpirationSize(add: false); if (hash.Remove(key, out var value)) - { - UpdateSize(key, value, false); - } + UpdateSize(key, value, add: false); } else { - expirationQueue.Dequeue(); - this.Size -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); + // The key was not in expirationTimes. It may have been Remove()d. + _ = expirationQueue.Dequeue(); + + // Adjust memory size for the priority queue entry removal. No DiskSize change needed as it was not in expirationTimes. + HeapMemorySize -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); } } - CleanupExpirationStructures(); + CleanupExpirationStructuresIfEmpty(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -427,9 +433,7 @@ private bool TryGetValue(ByteSpan key, out byte[] value) { value = default; if (IsExpired(key)) - { return false; - } #if NET9_0_OR_GREATER return hashSpanLookup.TryGetValue(key, out value); @@ -448,7 +452,17 @@ private bool Remove(ByteSpan key, out byte[] value) #endif if (result) { - UpdateSize(key, value, false); + if (HasExpirableItems) + { + // We cannot remove from the PQ so just remove from expirationTimes, let the next call to DeleteExpiredItems() clean it up, and don't adjust PQ sizes. +#if NET9_0_OR_GREATER + _ = expirationTimeSpanLookup.Remove(key); +#else + _ = expirationTimes.Remove(key); +#endif + UpdateExpirationSize(add: false, includePQ: false); + } + UpdateSize(key, value, add: false); } return result; } @@ -456,19 +470,14 @@ private bool Remove(ByteSpan key, out byte[] value) private int Count() { if (!HasExpirableItems) - { return hash.Count; - } var expiredKeysCount = 0; foreach (var item in expirationTimes) { if (IsExpired(item.Key)) - { expiredKeysCount++; - } } - return hash.Count - expiredKeysCount; } @@ -480,10 +489,7 @@ private bool ContainsKey(ByteSpan key) var result = hash.ContainsKey(key); #endif if (result && IsExpired(key)) - { return false; - } - return result; } @@ -492,9 +498,7 @@ private bool ContainsKey(ByteSpan key, out byte[] keyArray) { var result = hashSpanLookup.TryGetValue(key, out keyArray, out _); if (result && IsExpired(key)) - { return false; - } return result; } @@ -503,6 +507,7 @@ private bool ContainsKey(ByteSpan key, out byte[] keyArray) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void Add(ByteSpan key, byte[] value) { + // Called only when we have verified the key exists DeleteExpiredItems(); #if NET9_0_OR_GREATER var success = hashSpanLookup.TryAdd(key, value); @@ -511,7 +516,7 @@ private void Add(ByteSpan key, byte[] value) #endif Debug.Assert(success); - UpdateSize(key, value); + UpdateSize(key, value, add: true); } private ExpireResult SetExpiration(ByteSpan key, long expiration, ExpireOption expireOption) @@ -521,13 +526,11 @@ private ExpireResult SetExpiration(ByteSpan key, long expiration, ExpireOption e #else if (!ContainsKey(key)) #endif - { return ExpireResult.KeyNotFound; - } if (expiration <= DateTimeOffset.UtcNow.Ticks) { - Remove(key, out _); + _ = Remove(key, out _); return ExpireResult.KeyAlreadyExpired; } @@ -556,16 +559,15 @@ private ExpireResult SetExpiration(ByteSpan key, long expiration, ExpireOption e #else expirationQueue.Enqueue(key, expiration); #endif - // Size of dictionary entry already accounted for as the key already exists - this.Size += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + + // LogMemorySize of dictionary entry already accounted for as the key already exists. + // SerializedSize of expiration is already accounted for as the key already exists in expirationTimes. + HeapMemorySize += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; } else { - if ((expireOption & ExpireOption.XX) == ExpireOption.XX || - (expireOption & ExpireOption.GT) == ExpireOption.GT) - { + if ((expireOption & ExpireOption.XX) == ExpireOption.XX || (expireOption & ExpireOption.GT) == ExpireOption.GT) return ExpireResult.ExpireConditionNotMet; - } expirationTimeRef = expiration; #if NET9_0_OR_GREATER @@ -573,7 +575,7 @@ private ExpireResult SetExpiration(ByteSpan key, long expiration, ExpireOption e #else expirationQueue.Enqueue(key, expiration); #endif - UpdateExpirationSize(key); + UpdateExpirationSize(add: true, includePQ: true); } return ExpireResult.ExpireUpdated; @@ -592,8 +594,8 @@ private int Persist(ByteSpan key) if (HasExpirableItems && expirationTimes.Remove(key, out var currentExpiration)) #endif { - this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; - CleanupExpirationStructures(); + HeapMemorySize -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; + CleanupExpirationStructuresIfEmpty(); return (int)ExpireResult.ExpireUpdated; } @@ -603,19 +605,14 @@ private int Persist(ByteSpan key) private long GetExpiration(ByteSpan key) { if (!ContainsKey(key)) - { return (long)ExpireResult.KeyNotFound; - } #if NET9_0_OR_GREATER if (HasExpirableItems && expirationTimeSpanLookup.TryGetValue(key, out var expiration)) #else if (HasExpirableItems && expirationTimes.TryGetValue(key, out var expiration)) #endif - { return expiration; - } - return -1; } @@ -627,14 +624,10 @@ private KeyValuePair ElementAt(int index) foreach (var item in hash) { if (IsExpired(item.Key)) - { continue; - } if (currIndex++ == index) - { return item; - } } throw new ArgumentOutOfRangeException("index is outside the bounds of the source sequence."); diff --git a/libs/server/Objects/Hash/HashObjectImpl.cs b/libs/server/Objects/Hash/HashObjectImpl.cs index 76632d51d56..0fdacc49360 100644 --- a/libs/server/Objects/Hash/HashObjectImpl.cs +++ b/libs/server/Objects/Hash/HashObjectImpl.cs @@ -23,24 +23,20 @@ namespace Garnet.server /// public partial class HashObject : IGarnetObject { - private void HashGet(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashGet(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); var key = GetByteSpanFromInput(ref input, 0); if (TryGetValue(key, out var hashValue)) - { writer.WriteBulkString(hashValue); - } else - { writer.WriteNull(); - } - output.Header.result1++; + output.result1++; } - private void HashMultipleGet(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashMultipleGet(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -58,11 +54,11 @@ private void HashMultipleGet(ref ObjectInput input, ref GarnetObjectStoreOutput writer.WriteNull(); } - output.Header.result1++; + output.result1++; } } - private void HashGetAll(ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashGetAll(ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -82,36 +78,34 @@ private void HashGetAll(ref GarnetObjectStoreOutput output, byte respProtocolVer } } - private void HashDelete(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void HashDelete(ref ObjectInput input, ref ObjectOutput output) { for (var i = 0; i < input.parseState.Count; i++) { var key = GetByteSpanFromInput(ref input, i); - if (Remove(key, out var hashValue)) - { - output.Header.result1++; - } + if (Remove(key, out _)) + output.result1++; } } - private void HashLength(ref GarnetObjectStoreOutput output) + private void HashLength(ref ObjectOutput output) { - output.Header.result1 = Count(); + output.result1 = Count(); } - private void HashStrLength(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void HashStrLength(ref ObjectInput input, ref ObjectOutput output) { var key = GetByteSpanFromInput(ref input, 0); - output.Header.result1 = TryGetValue(key, out var hashValue) ? hashValue.Length : 0; + output.result1 = TryGetValue(key, out var hashValue) ? hashValue.Length : 0; } - private void HashExists(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void HashExists(ref ObjectInput input, ref ObjectOutput output) { var field = GetByteSpanFromInput(ref input, 0); - output.Header.result1 = ContainsKey(field) ? 1 : 0; + output.result1 = ContainsKey(field) ? 1 : 0; } - private void HashRandomField(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashRandomField(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // HRANDFIELD key [count [WITHVALUES]] var countParameter = input.arg1 >> 2; @@ -130,7 +124,7 @@ private void HashRandomField(ref ObjectInput input, ref GarnetObjectStoreOutput if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet { writer.WriteEmptyArray(); - output.Header.result1 = 0; + output.result1 = 0; return; } @@ -171,7 +165,7 @@ private void HashRandomField(ref ObjectInput input, ref GarnetObjectStoreOutput if (count == 0) // This can happen because of expiration but RMW operation haven't applied yet { writer.WriteNull(); - output.Header.result1 = 0; + output.result1 = 0; return; } @@ -181,10 +175,10 @@ private void HashRandomField(ref ObjectInput input, ref GarnetObjectStoreOutput countDone = 1; } - output.Header.result1 = countDone; + output.result1 = countDone; } - private void HashSet(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void HashSet(ref ObjectInput input, ref ObjectOutput output) { DeleteExpiredItems(); @@ -192,7 +186,7 @@ private void HashSet(ref ObjectInput input, ref GarnetObjectStoreOutput output) for (var i = 0; i < input.parseState.Count; i += 2) { var key = GetByteSpanFromInput(ref input, i); - var value = input.parseState.GetArgSliceByRef(i + 1).SpanByte.AsReadOnlySpan(); + var value = input.parseState.GetArgSliceByRef(i + 1).ReadOnlySpan; // Avoid multiple hash calculations by acquiring ref to the dictionary value. // The ref is unsafe to read/write to if the hash dictionary is mutated. @@ -206,9 +200,9 @@ private void HashSet(ref ObjectInput input, ref GarnetObjectStoreOutput output) if (!exists || IsExpired(key)) { hashValueRef = value.ToArray(); - UpdateSize(key, value); + UpdateSize(key, value, add: true); - output.Header.result1++; + output.result1++; } else if (exists && (hashOp is HashOperation.HSET or HashOperation.HMSET)) { @@ -219,9 +213,7 @@ private void HashSet(ref ObjectInput input, ref GarnetObjectStoreOutput output) else { // Adjust the size to account for the new value replacing the old one. - this.Size += Utility.RoundUp(value.Length, IntPtr.Size) - - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); - + HeapMemorySize += Utility.RoundUp(value.Length, IntPtr.Size) - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); hashValueRef = value.ToArray(); } @@ -233,21 +225,20 @@ private void HashSet(ref ObjectInput input, ref GarnetObjectStoreOutput output) expirationTimes.Remove(key)) #endif { - this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; - CleanupExpirationStructures(); + HeapMemorySize -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; + CleanupExpirationStructuresIfEmpty(); } } } } - private void HashCollect(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void HashCollect(ref ObjectInput input, ref ObjectOutput output) { DeleteExpiredItems(); - - output.Header.result1 = 1; + output.result1 = 1; } - private void HashGetKeysOrValues(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashGetKeysOrValues(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var count = Count(); var op = input.header.HashOp; @@ -274,19 +265,19 @@ private void HashGetKeysOrValues(ref ObjectInput input, ref GarnetObjectStoreOut writer.WriteBulkString(item.Value); } - output.Header.result1++; + output.result1++; } } [SkipLocalsInit] // avoid zeroing the stackalloc buffer - private void HashIncrement(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashIncrement(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var op = input.header.HashOp; using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); // This value is used to indicate partial command execution - output.Header.result1 = int.MinValue; + output.result1 = int.MinValue; var key = GetByteSpanFromInput(ref input, 0); var incrSlice = input.parseState.GetArgSliceByRef(1); @@ -311,7 +302,7 @@ private void HashIncrement(ref ObjectInput input, ref GarnetObjectStoreOutput ou if (!exists || IsExpired(key)) { hashValueRef = incrSlice.ToArray(); - UpdateSize(key, hashValueRef); + UpdateSize(key, hashValueRef, add: true); } else { @@ -337,27 +328,25 @@ private void HashIncrement(ref ObjectInput input, ref GarnetObjectStoreOutput ou else { // Adjust the size to account for the new value replacing the old one. - this.Size += Utility.RoundUp(formattedValue.Length, IntPtr.Size) - - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); - + HeapMemorySize += Utility.RoundUp(formattedValue.Length, IntPtr.Size) - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); hashValueRef = formattedValue.ToArray(); } } writer.WriteIntegerFromBytes(hashValueRef); - output.Header.result1 = 1; + output.result1 = 1; } [SkipLocalsInit] // avoid zeroing the stackalloc buffer - private void HashIncrementFloat(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashIncrementFloat(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var op = input.header.HashOp; using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); // This value is used to indicate partial command execution - output.Header.result1 = int.MinValue; + output.result1 = int.MinValue; var key = GetByteSpanFromInput(ref input, 0); var incrSlice = input.parseState.GetArgSliceByRef(1); @@ -388,7 +377,7 @@ private void HashIncrementFloat(ref ObjectInput input, ref GarnetObjectStoreOutp if (!exists || IsExpired(key)) { hashValueRef = incrSlice.ToArray(); - UpdateSize(key, hashValueRef); + UpdateSize(key, hashValueRef, add: true); } else { @@ -420,19 +409,17 @@ private void HashIncrementFloat(ref ObjectInput input, ref GarnetObjectStoreOutp else { // Adjust the size to account for the new value replacing the old one. - this.Size += Utility.RoundUp(formattedValue.Length, IntPtr.Size) - - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); - + HeapMemorySize += Utility.RoundUp(formattedValue.Length, IntPtr.Size) - Utility.RoundUp(hashValueRef.Length, IntPtr.Size); hashValueRef = formattedValue.ToArray(); } } writer.WriteBulkString(hashValueRef); - output.Header.result1 = 1; + output.result1 = 1; } - private void HashExpire(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashExpire(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -450,11 +437,11 @@ private void HashExpire(ref ObjectInput input, ref GarnetObjectStoreOutput outpu var result = SetExpiration(item.ToArray(), expirationWithOption.ExpirationTimeInTicks, expirationWithOption.ExpireOption); #endif writer.WriteInt32((int)result); - output.Header.result1++; + output.result1++; } } - private void HashTimeToLive(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashTimeToLive(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -495,11 +482,11 @@ private void HashTimeToLive(ref ObjectInput input, ref GarnetObjectStoreOutput o } writer.WriteInt64(result); - output.Header.result1++; + output.result1++; } } - private void HashPersist(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void HashPersist(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -517,7 +504,7 @@ private void HashPersist(ref ObjectInput input, ref GarnetObjectStoreOutput outp var result = Persist(item.ToArray()); #endif writer.WriteInt32(result); - output.Header.result1++; + output.result1++; } } diff --git a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs index a6504ed1032..ca12518eeb1 100644 --- a/libs/server/Objects/ItemBroker/CollectionItemBroker.cs +++ b/libs/server/Objects/ItemBroker/CollectionItemBroker.cs @@ -91,10 +91,10 @@ internal bool TryGetObserver(int sessionId, out CollectionItemObserver observer) /// Additional arguments for command /// Result of operation internal async Task GetCollectionItemAsync(RespCommand command, byte[][] keys, - RespServerSession session, double timeoutInSeconds, ArgSlice[] cmdArgs = null) + RespServerSession session, double timeoutInSeconds, PinnedSpanByte[] cmdArgs = null) { var observer = new CollectionItemObserver(session, command, cmdArgs); - return await GetCollectionItemAsync(observer, keys, timeoutInSeconds); + return await GetCollectionItemAsync(observer, keys, timeoutInSeconds).ConfigureAwait(false); } /// @@ -108,10 +108,10 @@ internal async Task GetCollectionItemAsync(RespCommand com /// Additional arguments for command /// Result of operation internal async Task MoveCollectionItemAsync(RespCommand command, byte[] srcKey, - RespServerSession session, double timeoutInSeconds, ArgSlice[] cmdArgs) + RespServerSession session, double timeoutInSeconds, PinnedSpanByte[] cmdArgs) { var observer = new CollectionItemObserver(session, command, cmdArgs); - return await GetCollectionItemAsync(observer, [srcKey], timeoutInSeconds); + return await GetCollectionItemAsync(observer, [srcKey], timeoutInSeconds).ConfigureAwait(false); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -466,7 +466,7 @@ private static bool TryMoveNextListItem(ListObject srcListObj, ListObject dstLis /// BZPOPMIN and BZPOPMAX share same implementation since Dictionary.First() and Last() /// handle the ordering automatically based on sorted set scores /// - private static unsafe bool TryGetNextSortedSetItem(byte[] key, SortedSetObject sortedSetObj, int count, RespCommand command, ArgSlice[] cmdArgs, out CollectionItemResult result) + private static unsafe bool TryGetNextSortedSetItem(byte[] key, SortedSetObject sortedSetObj, int count, RespCommand command, PinnedSpanByte[] cmdArgs, out CollectionItemResult result) { result = default; @@ -482,8 +482,8 @@ private static unsafe bool TryGetNextSortedSetItem(byte[] key, SortedSetObject s return true; case RespCommand.BZMPOP: - var lowScoresFirst = *(bool*)cmdArgs[0].ptr; - var popCount = *(int*)cmdArgs[1].ptr; + var lowScoresFirst = *(bool*)cmdArgs[0].ToPointer(); + var popCount = *(int*)cmdArgs[1].ToPointer(); popCount = Math.Min(popCount, count); var scores = new double[popCount]; @@ -505,7 +505,7 @@ private static unsafe bool TryGetNextSortedSetItem(byte[] key, SortedSetObject s } private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, RespCommand command, - ArgSlice[] cmdArgs, bool failOnSrcTypeMismatch, out int currCount, out CollectionItemResult result) + PinnedSpanByte[] cmdArgs, bool failOnSrcTypeMismatch, out int currCount, out CollectionItemResult result) { currCount = default; result = default; @@ -518,11 +518,9 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp _ => throw new NotSupportedException() }; - ArgSlice dstKey = default; + PinnedSpanByte dstKey = default; if (command == RespCommand.BLMOVE) - { dstKey = cmdArgs[0]; - } var asKey = storageSession.scratchBufferBuilder.CreateArgSlice(key); @@ -531,23 +529,22 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp { Debug.Assert(storageSession.txnManager.state == TxnState.None); createTransaction = true; - storageSession.txnManager.SaveKeyEntryToLock(asKey, true, LockType.Exclusive); + storageSession.txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + storageSession.txnManager.SaveKeyEntryToLock(asKey, LockType.Exclusive); if (command == RespCommand.BLMOVE) - { - storageSession.txnManager.SaveKeyEntryToLock(dstKey, true, LockType.Exclusive); - } + storageSession.txnManager.SaveKeyEntryToLock(dstKey, LockType.Exclusive); _ = storageSession.txnManager.Run(true); } - var lockableContext = storageSession.txnManager.LockableContext; - var objectLockableContext = storageSession.txnManager.ObjectStoreLockableContext; + var objectTransactionalContext = storageSession.txnManager.ObjectTransactionalContext; + var unifiedTransactionalContext = storageSession.txnManager.UnifiedTransactionalContext; try { // Get the object stored at key - var statusOp = storageSession.GET(key, out var osObject, ref objectLockableContext); + var statusOp = storageSession.GET(asKey, out var osObject, ref objectTransactionalContext); if (statusOp == GarnetStatus.NOTFOUND) return false; @@ -565,11 +562,9 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp } IGarnetObject dstObj = null; - byte[] arrDstKey = default; if (command == RespCommand.BLMOVE) { - arrDstKey = dstKey.ToArray(); - var dstStatusOp = storageSession.GET(arrDstKey, out var osDstObject, ref objectLockableContext); + var dstStatusOp = storageSession.GET(dstKey, out var osDstObject, ref objectTransactionalContext); if (dstStatusOp != GarnetStatus.NOTFOUND) { dstObj = osDstObject.GarnetObject; @@ -621,14 +616,13 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp if (isSuccessful && newObj) { - isSuccessful = storageSession.SET(arrDstKey, dstList, ref objectLockableContext) == - GarnetStatus.OK; + isSuccessful = storageSession.SET(dstKey, dstList, ref objectTransactionalContext) == GarnetStatus.OK; } break; case RespCommand.BLMPOP: var popDirection = (OperationDirection)cmdArgs[0].ReadOnlySpan[0]; - var popCount = *(int*)(cmdArgs[1].ptr); + var popCount = *(int*)(cmdArgs[1].ToPointer()); popCount = Math.Min(popCount, listObj.LnkList.Count); var items = new byte[popCount][]; @@ -648,8 +642,8 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp if (isSuccessful && listObj.LnkList.Count == 0) { - _ = storageSession.EXPIRE(asKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectLockableContext); + _ = storageSession.EXPIRE(asKey, TimeSpan.Zero, out _, ExpireOption.None, + ref unifiedTransactionalContext); } return isSuccessful; @@ -662,8 +656,8 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp if (isSuccessful && sortedSetObj.Count() == 0) { - _ = storageSession.EXPIRE(asKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectLockableContext); + _ = storageSession.EXPIRE(asKey, TimeSpan.Zero, out _, ExpireOption.None, + ref unifiedTransactionalContext); } return isSuccessful; @@ -674,6 +668,7 @@ private unsafe bool TryGetResult(byte[] key, StorageSession storageSession, Resp } finally { + storageSession.scratchBufferBuilder.RewindScratchBuffer(asKey); if (createTransaction) storageSession.txnManager.Commit(true); } @@ -698,7 +693,7 @@ private async Task StartAsync() // once event is dequeued successfully, call handler method try { - nextEvent = await brokerEventsQueue.DequeueAsync(cts.Token); + nextEvent = await brokerEventsQueue.DequeueAsync(cts.Token).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/libs/server/Objects/ItemBroker/CollectionItemObserver.cs b/libs/server/Objects/ItemBroker/CollectionItemObserver.cs index e9c8d420d1e..79f9bac601a 100644 --- a/libs/server/Objects/ItemBroker/CollectionItemObserver.cs +++ b/libs/server/Objects/ItemBroker/CollectionItemObserver.cs @@ -3,6 +3,7 @@ using System.Threading; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -24,7 +25,7 @@ internal class CollectionItemObserver /// /// Additional arguments for the command /// - internal ArgSlice[] CommandArgs { get; } + internal PinnedSpanByte[] CommandArgs { get; } /// /// Status of the observer @@ -51,7 +52,7 @@ internal class CollectionItemObserver /// internal CancellationTokenSource CancellationTokenSource { get; } = new(); - internal CollectionItemObserver(RespServerSession session, RespCommand command, ArgSlice[] commandArgs = null) + internal CollectionItemObserver(RespServerSession session, RespCommand command, PinnedSpanByte[] commandArgs = null) { Session = session; Command = command; diff --git a/libs/server/Objects/List/ListObject.cs b/libs/server/Objects/List/ListObject.cs index ec174a0e3ba..745a0a4e1d6 100644 --- a/libs/server/Objects/List/ListObject.cs +++ b/libs/server/Objects/List/ListObject.cs @@ -64,8 +64,8 @@ public partial class ListObject : GarnetObjectBase /// /// Constructor /// - public ListObject(long expiration = 0) - : base(expiration, MemoryUtils.ListOverhead) + public ListObject() + : base(MemoryUtils.ListOverhead) { list = new LinkedList(); } @@ -78,21 +78,20 @@ public ListObject(BinaryReader reader) { list = new LinkedList(); - int count = reader.ReadInt32(); - for (int i = 0; i < count; i++) + var count = reader.ReadInt32(); + for (var i = 0; i < count; i++) { var item = reader.ReadBytes(reader.ReadInt32()); - list.AddLast(item); - - this.UpdateSize(item); + _ = list.AddLast(item); + UpdateSize(item); } } /// /// Copy constructor /// - public ListObject(LinkedList list, long expiration, long size) - : base(expiration, size) + public ListObject(LinkedList list, long heapMemorySize) + : base(heapMemorySize) { this.list = list; } @@ -110,7 +109,7 @@ public override void DoSerialize(BinaryWriter writer) { base.DoSerialize(writer); - int count = list.Count; + var count = list.Count; writer.Write(count); foreach (var item in list) { @@ -125,23 +124,20 @@ public override void DoSerialize(BinaryWriter writer) public override void Dispose() { } /// - public override GarnetObjectBase Clone() => new ListObject(list, Expiration, Size); + public override GarnetObjectBase Clone() => new ListObject(list, HeapMemorySize); /// - public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, - byte respProtocolVersion, out long sizeChange) + public override bool Operate(ref ObjectInput input, ref ObjectOutput output, + byte respProtocolVersion) { - sizeChange = 0; - if (input.header.type != GarnetObjectType.List) { // Indicates an incorrect type of key - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; output.SpanByteAndMemory.Length = 0; return true; } - var previousSize = this.Size; switch (input.header.ListOp) { case ListOperation.LPUSH: @@ -187,19 +183,23 @@ public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput throw new GarnetException($"Unsupported operation {input.header.ListOp} in ListObject.Operate"); } - sizeChange = this.Size - previousSize; - if (list.Count == 0) - output.OutputFlags |= ObjectStoreOutputFlags.RemoveKey; + output.OutputFlags |= ObjectOutputFlags.RemoveKey; return true; } internal void UpdateSize(byte[] item, bool add = true) { - var size = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + MemoryUtils.ListEntryOverhead; - this.Size += add ? size : -size; - Debug.Assert(this.Size >= MemoryUtils.ListOverhead); + var memorySize = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + MemoryUtils.ListEntryOverhead; + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.ListOverhead); + } } /// diff --git a/libs/server/Objects/List/ListObjectImpl.cs b/libs/server/Objects/List/ListObjectImpl.cs index bf1b8fc6a22..06afc658ad4 100644 --- a/libs/server/Objects/List/ListObjectImpl.cs +++ b/libs/server/Objects/List/ListObjectImpl.cs @@ -13,18 +13,18 @@ namespace Garnet.server /// public partial class ListObject : IGarnetObject { - private void ListRemove(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void ListRemove(ref ObjectInput input, ref ObjectOutput output) { var count = input.arg1; //indicates partial execution - output.Header.result1 = int.MinValue; + output.result1 = int.MinValue; // get the source string to remove var itemSpan = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var removedCount = 0; - output.Header.result1 = 0; + output.result1 = 0; //remove all equals to item if (count == 0) @@ -64,13 +64,13 @@ private void ListRemove(ref ObjectInput input, ref GarnetObjectStoreOutput outpu currentNode = nextNode; } } - output.Header.result1 = removedCount; + output.result1 = removedCount; } - private void ListInsert(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void ListInsert(ref ObjectInput input, ref ObjectOutput output) { //indicates partial execution - output.Header.result1 = int.MinValue; + output.result1 = int.MinValue; if (list.Count > 0) { @@ -81,11 +81,11 @@ private void ListInsert(ref ObjectInput input, ref GarnetObjectStoreOutput outpu var pivot = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; // get the string to INSERT into the list - var item = input.parseState.GetArgSliceByRef(2).SpanByte.ToByteArray(); + var item = input.parseState.GetArgSliceByRef(2).ToArray(); var insertBefore = position.EqualsUpperCaseSpanIgnoringCase(CmdStrings.BEFORE); - output.Header.result1 = -1; + output.result1 = -1; // find the first ocurrence of the pivot element var currentNode = list.First; @@ -99,7 +99,7 @@ private void ListInsert(ref ObjectInput input, ref GarnetObjectStoreOutput outpu list.AddAfter(currentNode, item); UpdateSize(item); - output.Header.result1 = list.Count; + output.result1 = list.Count; break; } } @@ -107,23 +107,23 @@ private void ListInsert(ref ObjectInput input, ref GarnetObjectStoreOutput outpu } } - private void ListIndex(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void ListIndex(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var index = input.arg1; using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); - output.Header.result1 = -1; + output.result1 = -1; index = index < 0 ? list.Count + index : index; var item = list.ElementAtOrDefault(index); if (item != default) { writer.WriteBulkString(item); - output.Header.result1 = 1; + output.result1 = 1; } } - private void ListRange(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void ListRange(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var start = input.arg1; var stop = input.arg2; @@ -164,12 +164,12 @@ private void ListRange(ref ObjectInput input, ref GarnetObjectStoreOutput output writer.WriteBulkString(bytes); } - output.Header.result1 = count; + output.result1 = count; } } } - private void ListTrim(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void ListTrim(ref ObjectInput input, ref ObjectOutput output) { var start = input.arg1; var end = input.arg2; @@ -198,7 +198,7 @@ private void ListTrim(ref ObjectInput input, ref GarnetObjectStoreOutput output) list.RemoveLast(); this.UpdateSize(value, false); } - output.Header.result1 = numDeletes; + output.result1 = numDeletes; } else { @@ -213,23 +213,23 @@ private void ListTrim(ref ObjectInput input, ref GarnetObjectStoreOutput output) } i++; } - output.Header.result1 = i; + output.result1 = i; } } } } - private void ListLength(ref GarnetObjectStoreOutput output) + private void ListLength(ref ObjectOutput output) { - output.Header.result1 = list.Count; + output.result1 = list.Count; } - private void ListPush(ref ObjectInput input, ref GarnetObjectStoreOutput output, bool fAddAtHead) + private void ListPush(ref ObjectInput input, ref ObjectOutput output, bool fAddAtHead) { - output.Header.result1 = 0; + output.result1 = 0; for (var i = 0; i < input.parseState.Count; i++) { - var value = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var value = input.parseState.GetArgSliceByRef(i).ToArray(); // Add the value to the top of the list if (fAddAtHead) @@ -239,10 +239,10 @@ private void ListPush(ref ObjectInput input, ref GarnetObjectStoreOutput output, UpdateSize(value); } - output.Header.result1 = list.Count; + output.result1 = list.Count; } - private void ListPop(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion, bool fDelAtHead) + private void ListPop(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion, bool fDelAtHead) { var count = input.arg1; @@ -279,11 +279,11 @@ private void ListPop(ref ObjectInput input, ref GarnetObjectStoreOutput output, writer.WriteBulkString(node.Value); count--; - output.Header.result1++; + output.result1++; } } - private void ListSet(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void ListSet(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -309,7 +309,7 @@ private void ListSet(ref ObjectInput input, ref GarnetObjectStoreOutput output, } // element - var element = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); + var element = input.parseState.GetArgSliceByRef(1).ToArray(); var targetNode = index == 0 ? list.First : (index == list.Count - 1 ? list.Last @@ -320,10 +320,10 @@ private void ListSet(ref ObjectInput input, ref GarnetObjectStoreOutput output, UpdateSize(targetNode.Value); writer.WriteDirect(CmdStrings.RESP_OK); - output.Header.result1 = 1; + output.result1 = 1; } - private void ListPosition(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void ListPosition(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var element = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; @@ -444,7 +444,7 @@ private void ListPosition(ref ObjectInput input, ref GarnetObjectStoreOutput out writer.DecreaseArrayLength(noOfFoundItem, totalArrayHeaderLen); } - output.Header.result1 = noOfFoundItem; + output.result1 = noOfFoundItem; } private static bool ReadListPositionInput(ref ObjectInput input, out int rank, out int count, out bool isDefaultCount, out int maxlen, out ReadOnlySpan error) diff --git a/libs/server/Objects/Set/SetObject.cs b/libs/server/Objects/Set/SetObject.cs index 4e9ee621c23..f46e68ab65f 100644 --- a/libs/server/Objects/Set/SetObject.cs +++ b/libs/server/Objects/Set/SetObject.cs @@ -50,8 +50,8 @@ public partial class SetObject : GarnetObjectBase /// /// Constructor /// - public SetObject(long expiration = 0) - : base(expiration, MemoryUtils.HashSetOverhead) + public SetObject() + : base(MemoryUtils.HashSetOverhead) { Set = new HashSet(ByteArrayComparer.Instance); @@ -66,15 +66,14 @@ public SetObject(long expiration = 0) public SetObject(BinaryReader reader) : base(reader, MemoryUtils.HashSetOverhead) { - int count = reader.ReadInt32(); + var count = reader.ReadInt32(); Set = new HashSet(count, ByteArrayComparer.Instance); - for (int i = 0; i < count; i++) + for (var i = 0; i < count; i++) { var item = reader.ReadBytes(reader.ReadInt32()); Set.Add(item); - - this.UpdateSize(item); + UpdateSize(item); } #if NET9_0_OR_GREATER @@ -85,8 +84,8 @@ public SetObject(BinaryReader reader) /// /// Copy constructor /// - public SetObject(HashSet set, long expiration, long size) - : base(expiration, size) + public SetObject(HashSet set, long heapMemorySize) + : base(heapMemorySize) { Set = set; @@ -104,7 +103,7 @@ public override void DoSerialize(BinaryWriter writer) { base.DoSerialize(writer); - int count = Set.Count; + var count = Set.Count; writer.Write(count); foreach (var item in Set) { @@ -119,23 +118,20 @@ public override void DoSerialize(BinaryWriter writer) public override void Dispose() { } /// - public override GarnetObjectBase Clone() => new SetObject(Set, Expiration, Size); + public override GarnetObjectBase Clone() => new SetObject(Set, HeapMemorySize); /// - public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, - byte respProtocolVersion, out long sizeChange) + public override bool Operate(ref ObjectInput input, ref ObjectOutput output, + byte respProtocolVersion) { - sizeChange = 0; - if (input.header.type != GarnetObjectType.Set) { // Indicates an incorrect type of key - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; output.SpanByteAndMemory.Length = 0; return true; } - var prevSize = this.Size; switch (input.header.SetOp) { case SetOperation.SADD: @@ -169,26 +165,30 @@ public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput throw new GarnetException($"Unsupported operation {input.header.SetOp} in SetObject.Operate"); } - sizeChange = this.Size - prevSize; - if (Set.Count == 0) - output.OutputFlags |= ObjectStoreOutputFlags.RemoveKey; + output.OutputFlags |= ObjectOutputFlags.RemoveKey; return true; } internal void UpdateSize(ReadOnlySpan item, bool add = true) { - var size = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + MemoryUtils.HashSetEntryOverhead; - this.Size += add ? size : -size; - Debug.Assert(this.Size >= MemoryUtils.HashSetOverhead); + var memorySize = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + MemoryUtils.HashSetEntryOverhead; + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.HashSetOverhead); + } } /// public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false) { cursor = start; - items = new List(); + items = []; if (Set.Count < start) { @@ -196,7 +196,7 @@ public override unsafe void Scan(long start, out List items, out long cu return; } - int index = 0; + var index = 0; foreach (var item in Set) { if (index < start) diff --git a/libs/server/Objects/Set/SetObjectImpl.cs b/libs/server/Objects/Set/SetObjectImpl.cs index 434ffaa1361..e3d5264f107 100644 --- a/libs/server/Objects/Set/SetObjectImpl.cs +++ b/libs/server/Objects/Set/SetObjectImpl.cs @@ -13,7 +13,7 @@ namespace Garnet.server /// public partial class SetObject : IGarnetObject { - private void SetAdd(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void SetAdd(ref ObjectInput input, ref ObjectOutput output) { for (var i = 0; i < input.parseState.Count; i++) { @@ -25,13 +25,13 @@ private void SetAdd(ref ObjectInput input, ref GarnetObjectStoreOutput output) if (Set.Add(member.ToArray())) #endif { - output.Header.result1++; + output.result1++; UpdateSize(member); } } } - private void SetMembers(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SetMembers(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -40,11 +40,11 @@ private void SetMembers(ref ObjectInput input, ref GarnetObjectStoreOutput outpu foreach (var item in Set) { writer.WriteBulkString(item); - output.Header.result1++; + output.result1++; } } - private void SetIsMember(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SetIsMember(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -55,10 +55,10 @@ private void SetIsMember(ref ObjectInput input, ref GarnetObjectStoreOutput outp var isMember = Set.Contains(member.ToArray()); #endif writer.WriteInt32(isMember ? 1 : 0); - output.Header.result1 = 1; + output.result1 = 1; } - private void SetMultiIsMember(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SetMultiIsMember(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -75,10 +75,10 @@ private void SetMultiIsMember(ref ObjectInput input, ref GarnetObjectStoreOutput writer.WriteInt32(isMember ? 1 : 0); } - output.Header.result1 = input.parseState.Count; + output.result1 = input.parseState.Count; } - private void SetRemove(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void SetRemove(ref ObjectInput input, ref ObjectOutput output) { for (var i = 0; i < input.parseState.Count; i++) { @@ -90,19 +90,19 @@ private void SetRemove(ref ObjectInput input, ref GarnetObjectStoreOutput output if (Set.Remove(field.ToArray())) #endif { - output.Header.result1++; + output.result1++; UpdateSize(field, false); } } } - private void SetLength(ref GarnetObjectStoreOutput output) + private void SetLength(ref ObjectOutput output) { // SCARD key - output.Header.result1 = Set.Count; + output.result1 = Set.Count; } - private void SetPop(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SetPop(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // SPOP key [count] var count = input.arg1; @@ -151,10 +151,10 @@ private void SetPop(ref ObjectInput input, ref GarnetObjectStoreOutput output, b countDone++; } - output.Header.result1 = countDone; + output.result1 = countDone; } - private void SetRandomMember(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SetRandomMember(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var count = input.arg1; var seed = input.arg2; @@ -230,7 +230,7 @@ private void SetRandomMember(ref ObjectInput input, ref GarnetObjectStoreOutput } } - output.Header.result1 = countDone; + output.result1 = countDone; } } } \ No newline at end of file diff --git a/libs/server/Objects/SortedSet/SortedSetObject.cs b/libs/server/Objects/SortedSet/SortedSetObject.cs index 75915f6fd5c..e4bc43c1713 100644 --- a/libs/server/Objects/SortedSet/SortedSetObject.cs +++ b/libs/server/Objects/SortedSet/SortedSetObject.cs @@ -150,8 +150,8 @@ public partial class SortedSetObject : GarnetObjectBase /// /// Constructor /// - public SortedSetObject(long expiration = 0) - : base(expiration, MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead) + public SortedSetObject() + : base(MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead) { sortedSet = new(SortedSetComparer.Instance); sortedSetDict = new Dictionary(ByteArrayComparer.Instance); @@ -166,8 +166,8 @@ public SortedSetObject(BinaryReader reader) sortedSet = new(SortedSetComparer.Instance); sortedSetDict = new Dictionary(ByteArrayComparer.Instance); - int count = reader.ReadInt32(); - for (int i = 0; i < count; i++) + var count = reader.ReadInt32(); + for (var i = 0; i < count; i++) { var keyLength = reader.ReadInt32(); var hasExpiration = (keyLength & ExpirationBitMask) != 0; @@ -186,15 +186,15 @@ public SortedSetObject(BinaryReader reader) if (canAddItem) { sortedSetDict.Add(item, score); - sortedSet.Add((score, item)); - this.UpdateSize(item); + _ = sortedSet.Add((score, item)); + UpdateSize(item); if (expiration > 0) { InitializeExpirationStructures(); expirationTimes.Add(item, expiration); expirationQueue.Enqueue(item, expiration); - UpdateExpirationSize(item, true); + UpdateExpirationSize(add: true); } } } @@ -204,12 +204,12 @@ public SortedSetObject(BinaryReader reader) /// Copy constructor /// public SortedSetObject(SortedSetObject sortedSetObject) - : base(sortedSetObject.Expiration, sortedSetObject.Size) + : base(sortedSetObject.HeapMemorySize) { - this.sortedSet = sortedSetObject.sortedSet; - this.sortedSetDict = sortedSetObject.sortedSetDict; - this.expirationTimes = sortedSetObject.expirationTimes; - this.expirationQueue = sortedSetObject.expirationQueue; + sortedSet = sortedSetObject.sortedSet; + sortedSetDict = sortedSetObject.sortedSetDict; + expirationTimes = sortedSetObject.expirationTimes; + expirationQueue = sortedSetObject.expirationQueue; } /// @@ -248,7 +248,7 @@ public override void DoSerialize(BinaryWriter writer) DeleteExpiredItems(); - int count = sortedSetDict.Count; // Since expired items are already deleted, no need to worry about expiring items + var count = sortedSetDict.Count; // Since expired items are already deleted, no need to worry about expiring items writer.Write(count); foreach (var kvp in sortedSetDict) { @@ -280,9 +280,9 @@ public void Add(byte[] item, double score) DeleteExpiredItems(); sortedSetDict.Add(item, score); - sortedSet.Add((score, item)); + _ = sortedSet.Add((score, item)); - this.UpdateSize(item); + UpdateSize(item); } /// @@ -290,19 +290,16 @@ public void Add(byte[] item, double score) /// public bool Equals(SortedSetObject other) { - if (sortedSetDict.Count() != other.sortedSetDict.Count()) return false; + if (sortedSetDict.Count != other.sortedSetDict.Count) + return false; foreach (var key in sortedSetDict) { if (IsExpired(key.Key) && IsExpired(key.Key)) - { continue; - } if (IsExpired(key.Key) || IsExpired(key.Key)) - { return false; - } if (!other.sortedSetDict.TryGetValue(key.Key, out var otherValue) || key.Value != otherValue) return false; @@ -318,21 +315,18 @@ public override void Dispose() { } public override GarnetObjectBase Clone() => new SortedSetObject(this); /// - public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, - byte respProtocolVersion, out long sizeChange) + public override bool Operate(ref ObjectInput input, ref ObjectOutput output, + byte respProtocolVersion) { - sizeChange = 0; - var header = input.header; if (header.type != GarnetObjectType.SortedSet) { // Indicates an incorrect type of key - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; output.SpanByteAndMemory.Length = 0; return true; } - var prevSize = this.Size; var op = header.SortedSetOp; switch (op) { @@ -418,10 +412,8 @@ public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput throw new GarnetException($"Unsupported operation {op} in SortedSetObject.Operate"); } - sizeChange = this.Size - prevSize; - if (sortedSetDict.Count == 0) - output.OutputFlags |= ObjectStoreOutputFlags.RemoveKey; + output.OutputFlags |= ObjectOutputFlags.RemoveKey; return true; } @@ -430,7 +422,7 @@ public override bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = default, int patternLength = 0, bool isNoValue = false) { cursor = start; - items = new List(); + items = []; // Allocation for score to string conversion // Based on the reference https://en.wikipedia.org/wiki/IEEE_754-1985 @@ -438,8 +430,7 @@ public override unsafe void Scan(long start, out List items, out long cu const int DOUBLE_MAX_STRING_LENGTH = 38; Span doubleValueToByteSpan = stackalloc byte[DOUBLE_MAX_STRING_LENGTH]; - int index = 0; - + var index = 0; if (sortedSetDict.Count < start) { cursor = 0; @@ -461,7 +452,7 @@ public override unsafe void Scan(long start, out List items, out long cu continue; } - bool addToList = false; + var addToList = false; if (patternLength == 0) { items.Add(item.Key); @@ -482,7 +473,7 @@ public override unsafe void Scan(long start, out List items, out long cu if (addToList) { // Double.TryFormat was prefered to convert the value to UTF8 byte array, but is not available before .net 8 - if (Utf8Formatter.TryFormat(item.Value, doubleValueToByteSpan, out int bytesWritten, default)) + if (Utf8Formatter.TryFormat(item.Value, doubleValueToByteSpan, out var bytesWritten, default)) items.Add(doubleValueToByteSpan.Slice(0, bytesWritten).ToArray()); else items.Add(null); @@ -550,7 +541,7 @@ public static void InPlaceDiff(Dictionary dict1, SortedSetObject foreach (var item in dict1) { if (!sortedSetObject2.IsExpired(item.Key) && sortedSetObject2.sortedSetDict.ContainsKey(item.Key)) - dict1.Remove(item.Key); + _ = dict1.Remove(item.Key); } } } @@ -566,9 +557,7 @@ public bool TryGetScore(byte[] key, out double value) { value = default; if (IsExpired(key)) - { return false; - } return sortedSetDict.TryGetValue(key, out value); } @@ -580,17 +569,13 @@ public bool TryGetScore(byte[] key, out double value) public int Count() { if (!HasExpirableItems()) - { return sortedSetDict.Count; - } - var expiredKeysCount = 0; + var expiredKeysCount = 0; foreach (var item in expirationTimes) { if (IsExpired(item.Key)) - { expiredKeysCount++; - } } return sortedSetDict.Count - expiredKeysCount; } @@ -608,10 +593,7 @@ public int Count() /// /// True if the sorted set has expirable items; otherwise, false. [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool HasExpirableItems() - { - return expirationTimes is not null; - } + public bool HasExpirableItems() => expirationTimes is not null; #endregion private void InitializeExpirationStructures() @@ -620,28 +602,38 @@ private void InitializeExpirationStructures() { expirationTimes = new Dictionary(ByteArrayComparer.Instance); expirationQueue = new PriorityQueue(); - this.Size += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + HeapMemorySize += MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + // No DiskSize adjustment needed yet; wait until keys are added or removed } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void UpdateExpirationSize(ReadOnlySpan key, bool add = true) + private void UpdateExpirationSize(bool add, bool includePQ = true) { - var size = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead - + IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; - this.Size += add ? size : -size; + // Account for dictionary entry and priority queue entry + var memorySize = IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; + if (includePQ) + memorySize += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.DictionaryOverhead); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void CleanupExpirationStructures() + private void CleanupExpirationStructuresIfEmpty() { - if (expirationTimes.Count == 0) - { - this.Size -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead) * expirationQueue.Count; - this.Size -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; - expirationTimes = null; - expirationQueue = null; - } + if (expirationTimes.Count != 0) + return; + + HeapMemorySize -= (IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead) * expirationQueue.Count; + HeapMemorySize -= MemoryUtils.DictionaryOverhead + MemoryUtils.PriorityQueueOverhead; + expirationTimes = null; + expirationQueue = null; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -658,38 +650,39 @@ private void DeleteExpiredItemsWorker() { if (expirationTimes.TryGetValue(key, out var actualExpiration) && actualExpiration == expiration) { - expirationTimes.Remove(key); - expirationQueue.Dequeue(); - UpdateExpirationSize(key, false); + _ = expirationTimes.Remove(key); + _ = expirationQueue.Dequeue(); + UpdateExpirationSize(add: false); if (sortedSetDict.TryGetValue(key, out var value)) { - sortedSetDict.Remove(key); - sortedSet.Remove((value, key)); - UpdateSize(key, false); + _ = sortedSetDict.Remove(key); + _ = sortedSet.Remove((value, key)); + UpdateSize(key, add: false); } } else { - expirationQueue.Dequeue(); - this.Size -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); + // The key was not in expirationTimes. It may have been Remove()d. + _ = expirationQueue.Dequeue(); + + // Adjust memory size for the priority queue entry removal. No DiskSize change needed as it was not in expirationTimes. + HeapMemorySize -= MemoryUtils.PriorityQueueEntryOverhead + IntPtr.Size + sizeof(long); } } - CleanupExpirationStructures(); + CleanupExpirationStructuresIfEmpty(); } private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption) { if (!sortedSetDict.ContainsKey(key)) - { return (int)SortedSetExpireResult.KeyNotFound; - } if (expiration <= DateTimeOffset.UtcNow.Ticks) { - sortedSetDict.Remove(key, out var value); - sortedSet.Remove((value, key)); - UpdateSize(key, false); + _ = sortedSetDict.Remove(key, out var value); + _ = sortedSet.Remove((value, key)); + UpdateSize(key, add: false); return (int)SortedSetExpireResult.KeyAlreadyExpired; } @@ -706,18 +699,19 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption expirationTimes[key] = expiration; expirationQueue.Enqueue(key, expiration); - this.Size += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; + + // LogMemorySize of dictionary entry already accounted for as the key already exists. + // DiskSize of expiration already accounted for as the key already exists in expirationTimes. + HeapMemorySize += IntPtr.Size + sizeof(long) + MemoryUtils.PriorityQueueEntryOverhead; } else { - if (expireOption.HasFlag(ExpireOption.XX) || expireOption.HasFlag(ExpireOption.GT)) - { + if ((expireOption & ExpireOption.XX) == ExpireOption.XX || (expireOption & ExpireOption.GT) == ExpireOption.GT) return (int)SortedSetExpireResult.ExpireConditionNotMet; - } expirationTimes[key] = expiration; expirationQueue.Enqueue(key, expiration); - UpdateExpirationSize(key); + UpdateExpirationSize(add: true); } return (int)SortedSetExpireResult.ExpireUpdated; @@ -726,10 +720,7 @@ private int SetExpiration(byte[] key, long expiration, ExpireOption expireOption private int Persist(byte[] key) { if (!sortedSetDict.ContainsKey(key)) - { return -2; - } - return TryRemoveExpiration(key) ? 1 : -1; } @@ -744,28 +735,21 @@ private bool TryRemoveExpiration(byte[] key) private bool TryRemoveExpirationWorker(byte[] key) { if (!expirationTimes.TryGetValue(key, out _)) - { return false; - } - expirationTimes.Remove(key); - this.Size -= IntPtr.Size + sizeof(long) + MemoryUtils.DictionaryEntryOverhead; - CleanupExpirationStructures(); + _ = expirationTimes.Remove(key); + + UpdateExpirationSize(add: false, includePQ: false); + CleanupExpirationStructuresIfEmpty(); return true; } private long GetExpiration(byte[] key) { if (!sortedSetDict.ContainsKey(key)) - { return -2; - } - if (expirationTimes is not null && expirationTimes.TryGetValue(key, out var expiration)) - { return expiration; - } - return -1; } @@ -777,17 +761,12 @@ private KeyValuePair ElementAt(int index) foreach (var item in sortedSetDict) { if (IsExpired(item.Key)) - { continue; - } - if (currIndex++ == index) - { return item; - } } - throw new ArgumentOutOfRangeException("index is outside the bounds of the source sequence."); + throw new ArgumentOutOfRangeException(nameof(index), "index is outside the bounds of the source sequence."); } return sortedSetDict.ElementAt(index); @@ -796,10 +775,16 @@ private KeyValuePair ElementAt(int index) private void UpdateSize(ReadOnlySpan item, bool add = true) { // item's length + overhead to store item + value of type double added to sorted set and dictionary + overhead for those datastructures - var size = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + (2 * sizeof(double)) + var memorySize = Utility.RoundUp(item.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + (2 * sizeof(double)) + MemoryUtils.SortedSetEntryOverhead + MemoryUtils.DictionaryEntryOverhead; - this.Size += add ? size : -size; - Debug.Assert(this.Size >= MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead); + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.SortedSetOverhead + MemoryUtils.DictionaryOverhead); + } } /// diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs index 067bb2b2b5d..51fbb325d4e 100644 --- a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs +++ b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs @@ -87,7 +87,7 @@ bool GetOptions(ref ObjectInput input, ref int currTokenIdx, out SortedSetAddOpt return true; } - private void SortedSetAdd(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetAdd(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -208,7 +208,7 @@ private void SortedSetAdd(ref ObjectInput input, ref GarnetObjectStoreOutput out } } - private void SortedSetRemove(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void SortedSetRemove(ref ObjectInput input, ref ObjectOutput output) { DeleteExpiredItems(); @@ -220,7 +220,7 @@ private void SortedSetRemove(ref ObjectInput input, ref GarnetObjectStoreOutput if (!sortedSetDict.Remove(valueArray, out var key)) continue; - output.Header.result1++; + output.result1++; sortedSet.Remove((key, valueArray)); _ = TryRemoveExpiration(valueArray); @@ -228,19 +228,19 @@ private void SortedSetRemove(ref ObjectInput input, ref GarnetObjectStoreOutput } } - private void SortedSetLength(ref GarnetObjectStoreOutput output) + private void SortedSetLength(ref ObjectOutput output) { // Check both objects Debug.Assert(sortedSetDict.Count == sortedSet.Count, "SortedSet object is not in sync."); - output.Header.result1 = Count(); + output.result1 = Count(); } - private void SortedSetScore(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetScore(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // ZSCORE key member using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); - var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(0).ToArray(); if (!TryGetScore(member, out var score)) { @@ -250,10 +250,10 @@ private void SortedSetScore(ref ObjectInput input, ref GarnetObjectStoreOutput o { writer.WriteDoubleNumeric(score); } - output.Header.result1 = 1; + output.result1 = 1; } - private void SortedSetScores(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetScores(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // ZMSCORE key member var count = input.parseState.Count; @@ -264,7 +264,7 @@ private void SortedSetScores(ref ObjectInput input, ref GarnetObjectStoreOutput for (var i = 0; i < count; i++) { - var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).ToArray(); if (!TryGetScore(member, out var score)) { @@ -276,10 +276,10 @@ private void SortedSetScores(ref ObjectInput input, ref GarnetObjectStoreOutput } } - output.Header.result1 = count; + output.result1 = count; } - private void SortedSetCount(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetCount(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // Read min & max var minParamSpan = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; @@ -301,9 +301,12 @@ private void SortedSetCount(ref ObjectInput input, ref GarnetObjectStoreOutput o { foreach (var item in sortedSet.GetViewBetween((minValue, null), sortedSet.Max)) { - if (IsExpired(item.Element)) continue; - if (item.Score > maxValue || (maxExclusive && item.Score == maxValue)) break; - if (minExclusive && item.Score == minValue) continue; + if (IsExpired(item.Element)) + continue; + if (item.Score > maxValue || (maxExclusive && item.Score == maxValue)) + break; + if (minExclusive && item.Score == minValue) + continue; count++; } } @@ -311,7 +314,7 @@ private void SortedSetCount(ref ObjectInput input, ref GarnetObjectStoreOutput o writer.WriteInt32(count); } - private void SortedSetIncrement(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetIncrement(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -330,7 +333,7 @@ private void SortedSetIncrement(ref ObjectInput input, ref GarnetObjectStoreOutp } // Read member - var member = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(1).ToArray(); if (sortedSetDict.TryGetValue(member, out var score)) { @@ -358,7 +361,7 @@ private void SortedSetIncrement(ref ObjectInput input, ref GarnetObjectStoreOutp writer.WriteDoubleNumeric(sortedSetDict[member]); } - private void SortedSetRange(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetRange(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { //ZRANGE key min max [BYSCORE|BYLEX] [REV] [LIMIT offset count] [WITHSCORES] //ZRANGEBYSCORE key min max [WITHSCORES] [LIMIT offset count] @@ -560,7 +563,7 @@ void WriteSortedSetResult(bool withScores, int count, byte respProtocolVersion, } } - private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -602,7 +605,7 @@ private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref GarnetObjectS writer.WriteInt32(elementCount); } - private void SortedSetRemoveRangeByScore(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetRemoveRangeByScore(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -627,7 +630,7 @@ private void SortedSetRemoveRangeByScore(ref ObjectInput input, ref GarnetObject writer.WriteInt32(elementCount); } - private void SortedSetRandomMember(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetRandomMember(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var count = input.arg1 >> 2; var withScores = (input.arg1 & 1) == 1; @@ -669,16 +672,16 @@ private void SortedSetRandomMember(ref ObjectInput input, ref GarnetObjectStoreO } // Write count done into output footer - output.Header.result1 = count; + output.result1 = count; } - private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, ref GarnetObjectStoreOutput output, SortedSetOperation op) + private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, ref ObjectOutput output, SortedSetOperation op) { // ZREMRANGEBYLEX key min max // ZLEXCOUNT key min max // Using minValue for partial execution detection - output.Header.result1 = int.MinValue; + output.result1 = int.MinValue; var minParamBytes = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var maxParamBytes = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; @@ -692,9 +695,9 @@ private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, ref GarnetO var rem = GetElementsInRangeByLex(minParamBytes, maxParamBytes, false, false, isRemove, out int errorCode); - output.Header.result1 = errorCode; + output.result1 = errorCode; if (errorCode == 0) - output.Header.result1 = rem.Count; + output.result1 = rem.Count; } /// @@ -704,12 +707,12 @@ private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, ref GarnetO /// /// /// - private void SortedSetRank(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion, bool ascending = true) + private void SortedSetRank(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion, bool ascending = true) { //ZRANK key member var withScore = input.arg1 == 1; - var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(0).ToArray(); using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -775,7 +778,7 @@ private void SortedSetRank(ref ObjectInput input, ref GarnetObjectStoreOutput ou /// /// /// - private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion, SortedSetOperation op) + private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion, SortedSetOperation op) { DeleteExpiredItems(); @@ -801,7 +804,7 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref GarnetObjectSt if (count == 0) { writer.WriteEmptyArray(); - output.Header.result1 = 0; + output.result1 = 0; return; } @@ -832,10 +835,10 @@ private void SortedSetPopMinOrMaxCount(ref ObjectInput input, ref GarnetObjectSt count--; } - output.Header.result1 = countDone; + output.result1 = countDone; } - private void SortedSetPersist(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetPersist(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -849,11 +852,11 @@ private void SortedSetPersist(ref ObjectInput input, ref GarnetObjectStoreOutput { var result = Persist(item.ToArray()); writer.WriteInt32(result); - output.Header.result1++; + output.result1++; } } - private void SortedSetTimeToLive(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetTimeToLive(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { var isMilliseconds = input.arg1 == 1; var isTimestamp = input.arg2 == 1; @@ -895,11 +898,11 @@ private void SortedSetTimeToLive(ref ObjectInput input, ref GarnetObjectStoreOut } writer.WriteInt64(result); - output.Header.result1++; + output.result1++; } } - private void SortedSetExpire(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void SortedSetExpire(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -912,15 +915,15 @@ private void SortedSetExpire(ref ObjectInput input, ref GarnetObjectStoreOutput { var result = SetExpiration(item.ToArray(), expirationWithOption.ExpirationTimeInTicks, expirationWithOption.ExpireOption); writer.WriteInt32(result); - output.Header.result1++; + output.result1++; } } - private void SortedSetCollect(ref ObjectInput input, ref GarnetObjectStoreOutput output) + private void SortedSetCollect(ref ObjectInput input, ref ObjectOutput output) { DeleteExpiredItems(); - output.Header.result1 = 1; + output.result1 = 1; } #region CommonMethods diff --git a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs index 107b97bc4f4..27ac174eff7 100644 --- a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs +++ b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs @@ -27,7 +27,7 @@ private struct GeoSearchData public (double Latitude, double Longitude) Coordinates; } - private void GeoAdd(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void GeoAdd(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { DeleteExpiredItems(); @@ -81,7 +81,7 @@ private void GeoAdd(ref ObjectInput input, ref GarnetObjectStoreOutput output, b writer.WriteInt32((options & GeoAddOptions.CH) == 0 ? elementsAdded : elementsChanged); } - private void GeoHash(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void GeoHash(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -90,7 +90,7 @@ private void GeoHash(ref ObjectInput input, ref GarnetObjectStoreOutput output, for (var i = 0; i < input.parseState.Count; i++) { // Read member - var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).ToArray(); if (sortedSetDict.TryGetValue(member, out var value52Int)) { @@ -104,13 +104,13 @@ private void GeoHash(ref ObjectInput input, ref GarnetObjectStoreOutput output, } } - private void GeoDistance(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void GeoDistance(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { // Read 1st member - var member1 = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var member1 = input.parseState.GetArgSliceByRef(0).ToArray(); // Read 2nd member - var member2 = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); + var member2 = input.parseState.GetArgSliceByRef(1).ToArray(); // Read units var units = GeoDistanceUnitType.M; @@ -140,7 +140,7 @@ private void GeoDistance(ref ObjectInput input, ref GarnetObjectStoreOutput outp } } - private void GeoPosition(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + private void GeoPosition(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -149,7 +149,7 @@ private void GeoPosition(ref ObjectInput input, ref GarnetObjectStoreOutput outp for (var i = 0; i < input.parseState.Count; i++) { // read member - var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).ToArray(); if (sortedSetDict.TryGetValue(member, out var scoreMember1)) { diff --git a/libs/server/Objects/Types/ByteArrayBinaryObjectSerializer.cs b/libs/server/Objects/Types/ByteArrayBinaryObjectSerializer.cs deleted file mode 100644 index a3140797580..00000000000 --- a/libs/server/Objects/Types/ByteArrayBinaryObjectSerializer.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using Tsavorite.core; - -namespace Garnet.server -{ - /// - /// Byte array serializer - /// - public sealed class ByteArrayBinaryObjectSerializer : BinaryObjectSerializer - { - /// - public override void Deserialize(out byte[] obj) => obj = reader.ReadBytes(reader.ReadInt32()); - /// - public override void Serialize(ref byte[] obj) - { - writer.Write(obj.Length); - writer.Write(obj); - } - } -} \ No newline at end of file diff --git a/libs/server/Objects/Types/GarnetObject.cs b/libs/server/Objects/Types/GarnetObject.cs index bdd50ee5459..1c9ee0f7d34 100644 --- a/libs/server/Objects/Types/GarnetObject.cs +++ b/libs/server/Objects/Types/GarnetObject.cs @@ -15,14 +15,14 @@ public static class GarnetObject /// /// /// - internal static IGarnetObject Create(GarnetObjectType garnetObjectType, long expiration = 0) + internal static IGarnetObject Create(GarnetObjectType garnetObjectType) { return garnetObjectType switch { - GarnetObjectType.SortedSet => new SortedSetObject(expiration), - GarnetObjectType.List => new ListObject(expiration), - GarnetObjectType.Hash => new HashObject(expiration), - GarnetObjectType.Set => new SetObject(expiration), + GarnetObjectType.SortedSet => new SortedSetObject(), + GarnetObjectType.List => new ListObject(), + GarnetObjectType.Hash => new HashObject(), + GarnetObjectType.Set => new SetObject(), _ => throw new Exception("Unsupported data type"), }; } @@ -74,8 +74,6 @@ internal static bool NeedToCreate(RespInputHeader header) HashOperation.HCOLLECT => false, _ => true, }, - GarnetObjectType.Expire => false, - GarnetObjectType.Persist => false, _ => true, }; } diff --git a/libs/server/Objects/Types/GarnetObjectBase.cs b/libs/server/Objects/Types/GarnetObjectBase.cs index 3eb85e99b20..dd56b8584ca 100644 --- a/libs/server/Objects/Types/GarnetObjectBase.cs +++ b/libs/server/Objects/Types/GarnetObjectBase.cs @@ -3,10 +3,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Text; -using System.Threading; using Garnet.common; using Tsavorite.core; @@ -15,139 +12,36 @@ namespace Garnet.server /// /// Base class for Garnet heap objects /// - public abstract class GarnetObjectBase : IGarnetObject + public abstract class GarnetObjectBase : HeapObjectBase, IGarnetObject { - int serializationState; - public byte[] serialized; - /// public abstract byte Type { get; } - /// - public long Expiration { get; set; } - - /// - public long Size { get; set; } - - protected GarnetObjectBase(long expiration, long size) + protected GarnetObjectBase(long heapMemorySize) { - Debug.Assert(size >= 0); - this.Expiration = expiration; - this.Size = size; + HeapMemorySize = heapMemorySize; } - protected GarnetObjectBase(BinaryReader reader, long size) - : this(expiration: reader.ReadInt64(), size: size) - { - } - - /// - public void Serialize(BinaryWriter writer) - { - while (true) - { - if (serializationState == (int)SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZING)) - { - // Directly serialize to wire, do not cache serialized state - writer.Write(Type); - DoSerialize(writer); - serializationState = (int)SerializationPhase.REST; - return; - } - - if (serializationState == (int)SerializationPhase.SERIALIZED) - { - // If serialized state is cached, use that - var _serialized = serialized; - if (_serialized != null) - { - writer.Write(Type); - writer.Write(_serialized); - } - else - { - // Write null object to stream - writer.Write((byte)GarnetObjectType.Null); - } - return; - } - - Thread.Yield(); - } - } - - /// - public void CopyUpdate(ref IGarnetObject oldValue, ref IGarnetObject newValue, bool isInNewVersion) + protected GarnetObjectBase(BinaryReader reader, long heapMemorySize) + : this(heapMemorySize) { - newValue = Clone(); - newValue.Expiration = Expiration; - - // If we are not currently taking a checkpoint, we can delete the old version - // since the new version of the object is already created. - if (!isInNewVersion) - { - // Wait for any concurrent ongoing serialization of oldValue to complete - while (true) - { - if (serializationState == (int)SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZED)) - break; - - if (serializationState >= (int)SerializationPhase.SERIALIZED) - break; - - _ = Thread.Yield(); - } - oldValue = null; - return; - } - - // Create a serialized version for checkpoint version (v) - while (true) - { - if (serializationState == (int)SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZING)) - { - using var ms = new MemoryStream(); - using var writer = new BinaryWriter(ms, Encoding.UTF8); - DoSerialize(writer); - serialized = ms.ToArray(); - - serializationState = (int)SerializationPhase.SERIALIZED; - return; - } - - if (serializationState >= (int)SerializationPhase.SERIALIZED) - return; - - Thread.Yield(); - } + // Add anything here that should match DoSerialize() } - /// - /// Clone object (shallow copy) - /// - /// - public abstract GarnetObjectBase Clone(); - /// - public abstract bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion, out long sizeChange); - - /// - public abstract void Dispose(); + public abstract bool Operate(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion); /// /// Serialize to given writer /// NOTE: Make sure to first call base.DoSerialize(writer) in all derived classes. /// - public virtual void DoSerialize(BinaryWriter writer) + public override void DoSerialize(BinaryWriter writer) { - writer.Write(Expiration); + // Add anything here that needs to be in front of the derived object data } - private bool MakeTransition(SerializationPhase expectedPhase, SerializationPhase nextPhase) - { - if (Interlocked.CompareExchange(ref serializationState, (int)nextPhase, (int)expectedPhase) != (int)expectedPhase) return false; - return true; - } + /// + public override void WriteType(BinaryWriter writer, bool isNull) => writer.Write(isNull ? (byte)GarnetObjectType.Null : Type); /// /// Scan the items of the collection @@ -167,7 +61,7 @@ private bool MakeTransition(SerializationPhase expectedPhase, SerializationPhase /// /// /// - protected unsafe void Scan(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion) + protected unsafe void Scan(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion) { using var writer = new RespMemoryWriter(respProtocolVersion, ref output.SpanByteAndMemory); @@ -203,7 +97,7 @@ protected unsafe void Scan(ref ObjectInput input, ref GarnetObjectStoreOutput ou } } - output.Header.result1 = items.Count; + output.result1 = items.Count; } else { @@ -253,7 +147,7 @@ private static unsafe bool ReadScanInput(ref ObjectInput input, ref SpanByteAndM if (sbParam.EqualsUpperCaseSpanIgnoringCase(CmdStrings.MATCH)) { // Read pattern for keys filter - var sbPattern = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte; + var sbPattern = input.parseState.GetArgSliceByRef(currTokenIdx++); pattern = sbPattern.ToPointer(); patternLength = sbPattern.Length; } diff --git a/libs/server/Objects/Types/GarnetObjectSerializer.cs b/libs/server/Objects/Types/GarnetObjectSerializer.cs index 498dec7e289..14b92834d8d 100644 --- a/libs/server/Objects/Types/GarnetObjectSerializer.cs +++ b/libs/server/Objects/Types/GarnetObjectSerializer.cs @@ -11,7 +11,8 @@ namespace Garnet.server /// /// Serializer for IGarnetObject /// - public sealed class GarnetObjectSerializer : BinaryObjectSerializer + /// Implements for Tsavorite + public sealed class GarnetObjectSerializer : BinaryObjectSerializer, IObjectSerializer { readonly CustomCommandManager customCommandManager; @@ -29,6 +30,12 @@ public override void Deserialize(out IGarnetObject obj) obj = DeserializeInternal(base.reader); } + /// + public void Deserialize(out IHeapObject obj) + { + obj = DeserializeInternal(base.reader); + } + /// Thread-safe version of Deserialize /// /// @@ -64,21 +71,24 @@ private IGarnetObject CustomDeserialize(byte type, BinaryReader binaryReader) } /// - public override void Serialize(ref IGarnetObject obj) => SerializeInternal(base.writer, obj); + public override void Serialize(IGarnetObject obj) => SerializeInternal(base.writer, obj); /// Thread safe version of Serialize. /// /// - public static byte[] Serialize(IGarnetObject obj) + public static void Serialize(IGarnetObject obj, out byte[] bytes) { Debug.Assert(obj != null); using var ms = new MemoryStream(); using var binaryWriter = new BinaryWriter(ms, Encoding.UTF8); SerializeInternal(binaryWriter, obj); - return ms.ToArray(); + bytes = ms.ToArray(); } + /// + public void Serialize(IHeapObject obj) => SerializeInternal(base.writer, (IGarnetObject)obj); + private static void SerializeInternal(BinaryWriter binaryWriter, IGarnetObject obj) { if (obj == null) diff --git a/libs/server/Objects/Types/GarnetObjectType.cs b/libs/server/Objects/Types/GarnetObjectType.cs index 3007ff94655..20e24f6dd89 100644 --- a/libs/server/Objects/Types/GarnetObjectType.cs +++ b/libs/server/Objects/Types/GarnetObjectType.cs @@ -33,56 +33,16 @@ public enum GarnetObjectType : byte // Any new special type inserted here should update GarnetObjectTypeExtensions.FirstSpecialObjectType - /// - /// Special type indicating DELIFEXPIM command, a conditional deletion when a key is in memory and expired - /// - DelIfExpIm = 0xf7, - - /// - /// Special type indicating PEXPIRE command - /// - PExpire = 0xf8, - - /// - /// Special type indicating EXPIRETIME command - /// - ExpireTime = 0xf9, - - /// - /// Special type indicating PEXPIRETIME command - /// - PExpireTime = 0xfa, - /// /// Indicating a Custom Object command /// All = 0xfb, - - /// - /// Special type indicating PTTL command - /// - PTtl = 0xfc, - - /// - /// Special type indicating PERSIST command - /// - Persist = 0xfd, - - /// - /// Special type indicating TTL command - /// - Ttl = 0xfe, - - /// - /// Special type indicating EXPIRE command - /// - Expire = 0xff, } public static class GarnetObjectTypeExtensions { internal const GarnetObjectType LastObjectType = GarnetObjectType.Set; - internal const GarnetObjectType FirstSpecialObjectType = GarnetObjectType.DelIfExpIm; + internal const GarnetObjectType FirstSpecialObjectType = GarnetObjectType.All; } } \ No newline at end of file diff --git a/libs/server/Objects/Types/IGarnetObject.cs b/libs/server/Objects/Types/IGarnetObject.cs index e698eeb4e78..f7535c42e91 100644 --- a/libs/server/Objects/Types/IGarnetObject.cs +++ b/libs/server/Objects/Types/IGarnetObject.cs @@ -1,50 +1,28 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Collections.Generic; -using System.IO; +using Tsavorite.core; namespace Garnet.server { /// /// Interface representing Garnet object /// - public interface IGarnetObject : IDisposable + public interface IGarnetObject : IHeapObject { /// /// Type of object /// byte Type { get; } - /// - /// Expiration time of object - /// - long Expiration { get; set; } - - /// - /// Total memory size of the object - /// - long Size { get; set; } - /// /// Operator on object /// /// /// - /// /// - bool Operate(ref ObjectInput input, ref GarnetObjectStoreOutput output, byte respProtocolVersion, out long sizeChange); - - /// - /// Serializer - /// - void Serialize(BinaryWriter writer); - - /// - /// Copy update - /// - void CopyUpdate(ref IGarnetObject oldValue, ref IGarnetObject newValue, bool isInNewVersion); + bool Operate(ref ObjectInput input, ref ObjectOutput output, byte respProtocolVersion); /// /// Scan the items of the collection diff --git a/libs/server/Objects/Types/GarnetObjectStoreOutput.cs b/libs/server/Objects/Types/ObjectOutput.cs similarity index 60% rename from libs/server/Objects/Types/GarnetObjectStoreOutput.cs rename to libs/server/Objects/Types/ObjectOutput.cs index ad525e18818..ebc831fb566 100644 --- a/libs/server/Objects/Types/GarnetObjectStoreOutput.cs +++ b/libs/server/Objects/Types/ObjectOutput.cs @@ -7,10 +7,10 @@ namespace Garnet.server { /// - /// Flags for object store outputs. + /// Flags for store outputs. /// [Flags] - public enum ObjectStoreOutputFlags : byte + public enum ObjectOutputFlags : byte { /// /// No flags set @@ -23,7 +23,7 @@ public enum ObjectStoreOutputFlags : byte RemoveKey = 1, /// - /// Wrong type of object + /// Wrong type of value /// WrongType = 1 << 1, } @@ -33,7 +33,7 @@ public enum ObjectStoreOutputFlags : byte /// Any field / property added to this struct must be set in the back-end (IFunctions) and used in the front-end (GarnetApi caller). /// That is in order to justify transferring data in this struct through the Tsavorite storage layer. /// - public struct GarnetObjectStoreOutput + public struct ObjectOutput { /// /// Span byte and memory @@ -46,38 +46,37 @@ public struct GarnetObjectStoreOutput public IGarnetObject GarnetObject; /// - /// Object header + /// Some result of operation (e.g., number of items added successfully) /// - public ObjectOutputHeader Header; + public int result1; /// /// Output flags /// - public ObjectStoreOutputFlags OutputFlags; + public ObjectOutputFlags OutputFlags; /// /// True if output flag WrongType is set /// - public bool HasWrongType => (OutputFlags & ObjectStoreOutputFlags.WrongType) == ObjectStoreOutputFlags.WrongType; + public readonly bool HasWrongType => + (OutputFlags & ObjectOutputFlags.WrongType) == ObjectOutputFlags.WrongType; /// /// True if output flag RemoveKey is set /// - public bool HasRemoveKey => (OutputFlags & ObjectStoreOutputFlags.RemoveKey) == ObjectStoreOutputFlags.RemoveKey; + public readonly bool HasRemoveKey => + (OutputFlags & ObjectOutputFlags.RemoveKey) == ObjectOutputFlags.RemoveKey; - public GarnetObjectStoreOutput() - { - SpanByteAndMemory = new(null); - } + public ObjectOutput() => SpanByteAndMemory = new(null); - public GarnetObjectStoreOutput(SpanByteAndMemory spam) - { - SpanByteAndMemory = spam; - } + public ObjectOutput(SpanByteAndMemory span) => SpanByteAndMemory = span; + + public static unsafe ObjectOutput FromPinnedPointer(byte* pointer, int length) + => new(SpanByteAndMemory.FromPinnedPointer(pointer, length)); public void ConvertToHeap() { - // Does not convert to heap when going pending, because we immediately complete pending operations for object store. + // Does not convert to heap when going pending, because we complete all pending operations before releasing the pinned source bytes. } } } \ No newline at end of file diff --git a/libs/server/OperationError.cs b/libs/server/OperationError.cs deleted file mode 100644 index 0667fd347b7..00000000000 --- a/libs/server/OperationError.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -namespace Garnet.server -{ - /// - /// Operation error type - /// - public enum OperationError : byte - { - /// - /// Operation on data type succeeded - /// - SUCCESS, - /// - /// Operation failed due to incompatible type - /// - INVALID_TYPE, - /// - /// Operation failed due to NaN/infinity - /// - NAN_OR_INFINITY - } -} \ No newline at end of file diff --git a/libs/server/Properties/AssemblyInfo.cs b/libs/server/Properties/AssemblyInfo.cs index 128cca4c54c..5418a03c8b3 100644 --- a/libs/server/Properties/AssemblyInfo.cs +++ b/libs/server/Properties/AssemblyInfo.cs @@ -4,9 +4,17 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Garnet.test" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.collections" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.acl" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.scripting" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.complexstring" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.vectorset" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.rangeindex" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Garnet.test.extensions" + AssemblyRef.GarnetPublicKey)] [assembly: InternalsVisibleTo("Garnet.fuzz" + AssemblyRef.GarnetPublicKey)] [assembly: InternalsVisibleTo("Embedded.perftest" + AssemblyRef.GarnetPublicKey)] [assembly: InternalsVisibleTo("BDN.benchmark" + AssemblyRef.GarnetPublicKey)] +[assembly: InternalsVisibleTo("Resp.benchmark" + AssemblyRef.GarnetPublicKey)] /// /// Sets public key string for friend assemblies. diff --git a/libs/server/Providers/GarnetProvider.cs b/libs/server/Providers/GarnetProvider.cs index 1ccbc6ac248..c27819f53f6 100644 --- a/libs/server/Providers/GarnetProvider.cs +++ b/libs/server/Providers/GarnetProvider.cs @@ -8,14 +8,10 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// - /// Session provider for Garnet, based on - /// [K, V, I, O, C] = [SpanByte, SpanByte, SpanByte, SpanByteAndMemory, long] + /// Session provider for Garnet /// - public sealed class GarnetProvider : TsavoriteKVProviderBase, MainStoreFunctions, MainStoreAllocator, SpanByteServerSerializer> + public sealed class GarnetProvider : TsavoriteKVProviderBase { readonly StoreWrapper storeWrapper; @@ -32,10 +28,8 @@ public sealed class GarnetProvider : TsavoriteKVProviderBase /// /// - public GarnetProvider(StoreWrapper storeWrapper, - SubscribeBroker broker = null, - MaxSizeSettings maxSizeSettings = default) - : base(new SpanByteServerSerializer(), broker, maxSizeSettings) + public GarnetProvider(StoreWrapper storeWrapper, SubscribeBroker broker = null, MaxSizeSettings maxSizeSettings = default) + : base(broker, maxSizeSettings) { this.storeWrapper = storeWrapper; } @@ -60,13 +54,10 @@ public void Dispose() storeWrapper.Dispose(); } - /// - public override SpanByteFunctionsForServer GetFunctions() => new(); - /// public override IMessageConsumer GetSession(WireFormat wireFormat, INetworkSender networkSender) => (wireFormat == WireFormat.ASCII) - ? new RespServerSession(Interlocked.Increment(ref lastSessionId), networkSender, storeWrapper, broker, null, true) + ? new RespServerSession(Interlocked.Increment(ref lastSessionId), networkSender, storeWrapper, broker, authenticator: null, enableScripts: true) : throw new GarnetException($"Unsupported wireFormat {wireFormat}"); } } \ No newline at end of file diff --git a/libs/server/Providers/TsavoriteKVProviderBase.cs b/libs/server/Providers/TsavoriteKVProviderBase.cs index 85a6e84e7b3..2f805fd8363 100644 --- a/libs/server/Providers/TsavoriteKVProviderBase.cs +++ b/libs/server/Providers/TsavoriteKVProviderBase.cs @@ -7,24 +7,16 @@ namespace Garnet.server { /// - /// Abstract session provider for TsavoriteKV store based on - /// [K, V, I, O, F, P] + /// Abstract session provider for TsavoriteKV store /// - public abstract class TsavoriteKVProviderBase : ISessionProvider - where TSessionFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - where TParameterSerializer : IServerSerializer + public abstract class TsavoriteKVProviderBase : ISessionProvider + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Store /// - protected readonly TsavoriteKV store; - - /// - /// Serializer - /// - protected readonly TParameterSerializer serializer; + protected readonly TsavoriteKV store; /// /// Broker @@ -39,14 +31,11 @@ public abstract class TsavoriteKVProviderBase /// Create TsavoriteKV backend /// - /// /// /// - public TsavoriteKVProviderBase(TParameterSerializer serializer, - SubscribeBroker broker = null, MaxSizeSettings maxSizeSettings = default) + public TsavoriteKVProviderBase(SubscribeBroker broker = null, MaxSizeSettings maxSizeSettings = default) { this.broker = broker; - this.serializer = serializer; this.maxSizeSettings = maxSizeSettings ?? new MaxSizeSettings(); } @@ -55,12 +44,6 @@ public TsavoriteKVProviderBase(TParameterSerializer serializer, /// public MaxSizeSettings GetMaxSizeSettings => this.maxSizeSettings; - /// - /// GetFunctions() for custom functions provided by the client - /// - /// - public abstract TSessionFunctions GetFunctions(); - /// public abstract IMessageConsumer GetSession(WireFormat wireFormat, INetworkSender networkSender); } diff --git a/libs/server/PubSub/SubscribeBroker.cs b/libs/server/PubSub/SubscribeBroker.cs index 79c891ec7b4..ba9fc5cb318 100644 --- a/libs/server/PubSub/SubscribeBroker.cs +++ b/libs/server/PubSub/SubscribeBroker.cs @@ -22,7 +22,7 @@ public sealed class SubscribeBroker : IDisposable, ILogEntryConsumer bool initialized = false; ConcurrentDictionary> subscriptions; ReadOptimizedConcurrentSet patternSubscriptions; - readonly TsavoriteLog log; + readonly TsavoriteLog aof; readonly IDevice device; readonly CancellationTokenSource cts = new(); readonly ManualResetEvent done = new(true); @@ -36,16 +36,15 @@ public sealed class SubscribeBroker : IDisposable, ILogEntryConsumer /// /// Directory where the log will be stored /// Page size of log used for pub/sub - /// Subscriber log refresh frequency /// start the log from scratch, do not continue - public SubscribeBroker(string logDir, long pageSize, int subscriberRefreshFrequencyMs, LightEpoch epoch, bool startFresh = true, ILogger logger = null) + public SubscribeBroker(string logDir, long pageSize, LightEpoch epoch, bool startFresh = true, ILogger logger = null) { device = logDir == null ? new NullDevice() : Devices.CreateLogDevice(logDir + "/pubsubkv", preallocateFile: false); device.Initialize((long)(1 << 30) * 64); - log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, PageSize = pageSize, MemorySize = pageSize * 4, SafeTailRefreshFrequencyMs = subscriberRefreshFrequencyMs, Epoch = epoch }); - pageSizeBits = log.UnsafeGetLogPageSizeBits(); + aof = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, PageSize = pageSize, MemorySize = pageSize * 4, Epoch = epoch }); + pageSizeBits = aof.UnsafeGetLogPageSizeBits(); if (startFresh) - log.TruncateUntil(log.CommittedUntilAddress); + aof.TruncateUntil(aof.CommittedUntilAddress); this.logger = logger; } @@ -74,7 +73,7 @@ public unsafe void RemoveSubscription(IMessageConsumer session) } } - unsafe int Broadcast(ArgSlice key, ArgSlice value) + unsafe int Broadcast(PinnedSpanByte key, PinnedSpanByte value) { var numSubscribers = 0; @@ -99,7 +98,7 @@ unsafe int Broadcast(ArgSlice key, ArgSlice value) var pattern = entry.pattern; fixed (byte* patternPtr = pattern.ReadOnlySpan) { - var patternSlice = new ArgSlice(patternPtr, pattern.ReadOnlySpan.Length); + var patternSlice = PinnedSpanByte.FromPinnedPointer(patternPtr, pattern.ReadOnlySpan.Length); if (Match(key, patternSlice)) { var sessions = entry.subscriptions; @@ -120,7 +119,7 @@ async Task StartAsync(CancellationToken cancellationToken = default) { try { - using var iterator = log.ScanSingle(log.BeginAddress, long.MaxValue, scanUncommitted: true); + using var iterator = aof.ScanSingle(aof.BeginAddress, long.MaxValue, scanUncommitted: true); var signal = iterator.Signal; using var registration = cts.Token.Register(signal); @@ -154,12 +153,12 @@ public unsafe void Consume(byte* payloadPtr, int payloadLength, long currentAddr } var ptr = payloadPtr; - var key = new ArgSlice(ptr + sizeof(int), *(int*)ptr); - ptr += sizeof(int) + key.length; - var value = new ArgSlice(ptr + sizeof(int), *(int*)ptr); + var key = PinnedSpanByte.FromPinnedPointer(ptr + sizeof(int), *(int*)ptr); + ptr += sizeof(int) + key.Length; + var value = PinnedSpanByte.FromPinnedPointer(ptr + sizeof(int), *(int*)ptr); _ = Broadcast(key, value); - if (nextAddress > log.BeginAddress) - log.TruncateUntil(nextAddress); + if (nextAddress > aof.BeginAddress) + aof.TruncateUntil(nextAddress); previousAddress = nextAddress; } catch (Exception ex) @@ -184,7 +183,7 @@ void Initialize() /// Key to subscribe to /// Server session /// - public unsafe bool Subscribe(ArgSlice key, ServerSessionBase session) + public unsafe bool Subscribe(PinnedSpanByte key, ServerSessionBase session) { if (!initialized && Interlocked.Increment(ref sid) == 1) Initialize(); @@ -204,7 +203,7 @@ public unsafe bool Subscribe(ArgSlice key, ServerSessionBase session) /// Pattern to subscribe to /// Server session /// - public unsafe bool PatternSubscribe(ArgSlice pattern, ServerSessionBase session) + public unsafe bool PatternSubscribe(PinnedSpanByte pattern, ServerSessionBase session) { if (!initialized && Interlocked.Increment(ref sid) == 1) Initialize(); @@ -295,7 +294,7 @@ public unsafe List ListAllPatternSubscriptions(ServerSessionBa /// key that has been updated /// value that has been updated /// Number of subscribers notified - public unsafe int PublishNow(ArgSlice key, ArgSlice value) + public unsafe int PublishNow(PinnedSpanByte key, PinnedSpanByte value) { if (subscriptions == null && patternSubscriptions == null) return 0; return Broadcast(key, value); @@ -306,13 +305,12 @@ public unsafe int PublishNow(ArgSlice key, ArgSlice value) /// /// key that has been updated /// value that has been updated - public unsafe void Publish(ArgSlice key, ArgSlice value) + public unsafe void Publish(PinnedSpanByte key, PinnedSpanByte value) { - if (subscriptions == null && patternSubscriptions == null) return; + if (subscriptions == null && patternSubscriptions == null) + return; - var keySB = key.SpanByte; - var valueSB = value.SpanByte; - log.Enqueue(ref keySB, ref valueSB, out _); + aof.Enqueue(key.ReadOnlySpan, value.ReadOnlySpan, out _); } /// @@ -338,7 +336,7 @@ public List GetChannels() /// /// /// - public unsafe List GetChannels(ArgSlice pattern) + public unsafe List GetChannels(PinnedSpanByte pattern) { if (subscriptions is null || subscriptions.IsEmpty) return []; @@ -350,7 +348,7 @@ public unsafe List GetChannels(ArgSlice pattern) { fixed (byte* keyPtr = entry.Key.ReadOnlySpan) { - if (Match(new ArgSlice(keyPtr, entry.Key.ReadOnlySpan.Length), pattern)) + if (Match(PinnedSpanByte.FromPinnedPointer(keyPtr, entry.Key.ReadOnlySpan.Length), pattern)) channels.Add(entry.Key); } } @@ -382,7 +380,7 @@ public int NumPatternSubscriptions() /// /// /// - public int NumSubscriptions(ArgSlice channel) + public int NumSubscriptions(PinnedSpanByte channel) { if (subscriptions is null) return 0; @@ -398,11 +396,11 @@ public void Dispose() done.WaitOne(); subscriptions?.Clear(); patternSubscriptions?.Clear(); - log.Dispose(); + aof.Dispose(); device.Dispose(); } - unsafe bool Match(ArgSlice key, ArgSlice pattern) - => GlobUtils.Match(pattern.ptr, pattern.length, key.ptr, key.length); + unsafe bool Match(PinnedSpanByte key, PinnedSpanByte pattern) + => GlobUtils.Match(pattern.ToPointer(), pattern.Length, key.ToPointer(), key.Length); } } \ No newline at end of file diff --git a/libs/server/Resp/AdminCommands.cs b/libs/server/Resp/AdminCommands.cs index f2e9b0f47a7..3e93a9e3ea0 100644 --- a/libs/server/Resp/AdminCommands.cs +++ b/libs/server/Resp/AdminCommands.cs @@ -525,43 +525,34 @@ private bool NetworkRegisterCs(CustomCommandManager customCommandManager) private bool NetworkModuleLoad(CustomCommandManager customCommandManager) { if (parseState.Count < 1) // At least module path is required - { return AbortWithWrongNumberOfArguments($"{RespCommand.MODULE}|{Encoding.ASCII.GetString(CmdStrings.LOADCS)}"); - } - if (!CanRunModule()) - { return AbortWithErrorMessage(CmdStrings.GenericErrCommandDisallowedWithOption, RespCommand.MODULE, "enable-module-command"); - } - // Read path to module file + // Read path to module file and module args var modulePath = parseState.GetArgSliceByRef(0).ToString(); - - // Read module args var moduleArgs = new string[parseState.Count - 1]; for (var i = 0; i < moduleArgs.Length; i++) moduleArgs[i] = parseState.GetArgSliceByRef(i + 1).ToString(); - var errorMsg = ReadOnlySpan.Empty; - + // Load only the referenced dependencies from the module directory, not all assemblies. + // This avoids loading hundreds of unrelated DLLs that may be present in binPath. + ReadOnlySpan errorMsg; var binPath = Path.GetDirectoryName(modulePath); - var moduleFileName = Path.GetFileName(modulePath); - - // Load dependencies from the module path - if (Directory.Exists(binPath) && !ModuleUtils.LoadAssemblies([binPath], - storeWrapper.serverOptions.ExtensionBinPaths, - storeWrapper.serverOptions.ExtensionAllowUnsignedAssemblies, out _, out errorMsg, [moduleFileName], - SearchOption.TopDirectoryOnly, true)) + if (Directory.Exists(binPath)) { - if (!errorMsg.IsEmpty) + if (!ModuleUtils.LoadModuleDependencies(modulePath, binPath, + storeWrapper.serverOptions.ExtensionBinPaths, + storeWrapper.serverOptions.ExtensionAllowUnsignedAssemblies, + out _, out errorMsg)) { - WriteError(errorMsg); + if (!errorMsg.IsEmpty) + WriteError(errorMsg); + return true; } - - return true; } - // Load the module path + // Load the module itself if (ModuleUtils.LoadAssemblies([modulePath], storeWrapper.serverOptions.ExtensionBinPaths, storeWrapper.serverOptions.ExtensionAllowUnsignedAssemblies, out var loadedAssemblies, out errorMsg)) { @@ -577,10 +568,7 @@ private bool NetworkModuleLoad(CustomCommandManager customCommandManager) } if (!errorMsg.IsEmpty) - { WriteError(errorMsg); - } - return true; } @@ -803,31 +791,44 @@ private bool NetworkROLE() } else { + var usingShardedLog = storeWrapper.serverOptions.AofPhysicalSublogCount > 1; if (storeWrapper.clusterProvider.IsPrimary()) { var (replication_offset, replicaInfo) = storeWrapper.clusterProvider.GetPrimaryInfo(); - while (!RespWriteUtils.TryWriteArrayLength(3, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteArrayLength(usingShardedLog ? 4 : 3, ref dcurr, dend)) SendAndReset(); while (!RespWriteUtils.TryWriteAsciiBulkString("master", ref dcurr, dend)) SendAndReset(); - while (!RespWriteUtils.TryWriteInt64(replication_offset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteInt64(replication_offset[0], ref dcurr, dend)) SendAndReset(); while (!RespWriteUtils.TryWriteArrayLength(replicaInfo.Count, ref dcurr, dend)) SendAndReset(); - foreach (var replice in replicaInfo) + foreach (var replica in replicaInfo) { - while (!RespWriteUtils.TryWriteArrayLength(3, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteArrayLength(usingShardedLog ? 4 : 3, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(replica.address, ref dcurr, dend)) SendAndReset(); - while (!RespWriteUtils.TryWriteAsciiBulkString(replice.address, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteInt32(replica.port, ref dcurr, dend)) SendAndReset(); - while (!RespWriteUtils.TryWriteInt32(replice.port, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteInt64(replica.replication_offset[0], ref dcurr, dend)) SendAndReset(); - while (!RespWriteUtils.TryWriteInt64(replice.replication_offset, ref dcurr, dend)) + + if (usingShardedLog) + { + while (!RespWriteUtils.TryWriteAsciiBulkString(replica.replication_offset.ToString(), ref dcurr, dend)) + SendAndReset(); + } + } + + if (usingShardedLog) + { + while (!RespWriteUtils.TryWriteAsciiBulkString(replication_offset.ToString(), ref dcurr, dend)) SendAndReset(); } } @@ -835,7 +836,7 @@ private bool NetworkROLE() { var role = storeWrapper.clusterProvider.GetReplicaInfo(); - while (!RespWriteUtils.TryWriteArrayLength(5, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteArrayLength(usingShardedLog ? 6 : 5, ref dcurr, dend)) SendAndReset(); while (!RespWriteUtils.TryWriteAsciiBulkString("slave", ref dcurr, dend)) @@ -850,8 +851,14 @@ private bool NetworkROLE() while (!RespWriteUtils.TryWriteAsciiBulkString(role.replication_state, ref dcurr, dend)) SendAndReset(); - while (!RespWriteUtils.TryWriteInt64(role.replication_offset, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteInt64(role.replication_offset[0], ref dcurr, dend)) SendAndReset(); + + if (usingShardedLog) + { + while (!RespWriteUtils.TryWriteAsciiBulkString(role.replication_offset.ToString(), ref dcurr, dend)) + SendAndReset(); + } } } diff --git a/libs/server/Resp/ArrayCommands.cs b/libs/server/Resp/ArrayCommands.cs index 7ba2d976413..089cfc13478 100644 --- a/libs/server/Resp/ArrayCommands.cs +++ b/libs/server/Resp/ArrayCommands.cs @@ -73,7 +73,7 @@ private bool NetworkMSETNX(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.MSETNX)); } - var input = new RawStringInput(RespCommand.MSETNX, ref parseState); + var input = new StringInput(RespCommand.MSETNX, ref parseState); var status = storageApi.MSET_Conditional(ref input); // For a "set if not exists", NOTFOUND means that the operation succeeded @@ -86,10 +86,11 @@ private bool NetworkDEL(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { int keysDeleted = 0; + for (int c = 0; c < parseState.Count; c++) { - var key = parseState.GetArgSliceByRef(c).SpanByte; - var status = storageApi.DELETE(ref key, StoreType.All); + var key = parseState.GetArgSliceByRef(c); + var status = storageApi.DELETE(key); // This is only an approximate count because the deletion of a key on disk is performed as a blind tombstone append if (status == GarnetStatus.OK) @@ -257,7 +258,7 @@ private bool NetworkSCAN(ref TGarnetApi storageApi) } var pattern = "*"u8; - var patternArgSlice = ArgSlice.FromPinnedSpan(pattern); + var patternArgSlice = PinnedSpanByte.FromPinnedSpan(pattern); var allKeys = true; long countValue = 10; ReadOnlySpan typeParameterValue = default; @@ -330,16 +331,21 @@ private bool NetworkTYPE(ref TGarnetApi storageApi) // TYPE key var keySlice = parseState.GetArgSliceByRef(0); - var status = storageApi.GetKeyType(keySlice, out var typeName); + // Prepare input + var input = new UnifiedInput(RespCommand.TYPE); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); + + var status = storageApi.TYPE(keySlice, ref input, ref output); if (status == GarnetStatus.OK) { - while (!RespWriteUtils.TryWriteSimpleString(typeName, ref dcurr, dend)) - SendAndReset(); + ProcessOutput(output.SpanByteAndMemory); } else { - while (!RespWriteUtils.TryWriteSimpleString("none"u8, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteSimpleString(CmdStrings.none, ref dcurr, dend)) SendAndReset(); } @@ -448,13 +454,10 @@ private bool NetworkLCS(ref TGarnetApi storageApi) return AbortWithErrorMessage(CmdStrings.RESP_ERR_LENGTH_AND_INDEXES); } - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var output = GetStringOutput(); var status = storageApi.LCS(key1, key2, ref output, lenOnly, withIndices, withMatchLen, minMatchLen); - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; + ProcessOutput(output.SpanByteAndMemory); return true; } diff --git a/libs/server/Resp/AsyncProcessor.cs b/libs/server/Resp/AsyncProcessor.cs index a88fe0a534d..9349df6bf6f 100644 --- a/libs/server/Resp/AsyncProcessor.cs +++ b/libs/server/Resp/AsyncProcessor.cs @@ -62,7 +62,7 @@ void NetworkGETPending(ref TGarnetApi storageApi) RunContinuationsAsynchronously = true }; var _storageApi = storageApi; - _ = Task.Run(async () => await AsyncGetProcessorAsync(_storageApi)); + _ = AsyncGetProcessorAsync(_storageApi); } else { @@ -79,6 +79,9 @@ void NetworkGETPending(ref TGarnetApi storageApi) async Task AsyncGetProcessorAsync(TGarnetApi storageApi) where TGarnetApi : IGarnetApi { + // Force async + await Task.Yield(); + while (!asyncWaiterCancel.Token.IsCancellationRequested) { while (asyncCompleted < asyncStarted) @@ -112,9 +115,9 @@ async Task AsyncGetProcessorAsync(TGarnetApi storageApi) SendAndReset(); if (completedOutputs.Current.Status.Found) { - Debug.Assert(!o.IsSpanByte); + Debug.Assert(!o.SpanByteAndMemory.IsSpanByte); sessionMetrics?.incr_total_found(); - SendAndReset(o.Memory, o.Length); + SendAndReset(o.SpanByteAndMemory.Memory, o.SpanByteAndMemory.Length); } else { @@ -138,7 +141,7 @@ async Task AsyncGetProcessorAsync(TGarnetApi storageApi) // Wait for next async operation // We do not need to cancel the wait - it should get garbage collected when the session ends - await asyncWaiter.WaitAsync(); + await asyncWaiter.WaitAsync().ConfigureAwait(false); } } } diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 9d7778e8dcf..f3b7012bc59 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Numerics; using System.Text; using System.Threading.Tasks; using Garnet.common; @@ -29,11 +30,11 @@ bool NetworkGET(ref TGarnetApi storageApi) if (useAsync) return NetworkGETAsync(ref storageApi); - RawStringInput input = new(RespCommand.GET, arg1: -1); + StringInput input = new(RespCommand.GET, arg1: -1); - ref var key = ref parseState.GetArgSliceByRef(0); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.GET(key, ref input, ref o); + var key = parseState.GetArgSliceByRef(0); + var output = GetStringOutput(); + var status = storageApi.GET(key, ref input, ref output); switch (status) { @@ -41,13 +42,10 @@ bool NetworkGET(ref TGarnetApi storageApi) WriteError(CmdStrings.RESP_ERR_WRONG_TYPE); break; case GarnetStatus.OK: - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); break; case GarnetStatus.NOTFOUND: - Debug.Assert(o.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); break; } @@ -62,26 +60,20 @@ bool NetworkGETEX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count < 1 || parseState.Count > 3) - { return AbortWithWrongNumberOfArguments(nameof(RespCommand.GETEX)); - } - var key = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); TimeSpan? tsExpiry = null; if (parseState.Count > 1) { var option = parseState.GetArgSliceByRef(1).ReadOnlySpan; if (option.EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST)) - { tsExpiry = TimeSpan.Zero; - } else { if (parseState.Count < 3 || !parseState.TryGetLong(2, out var expireTime) || expireTime <= 0) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_OUT_OF_RANGE); - } switch (option) { @@ -110,21 +102,18 @@ bool NetworkGETEX(ref TGarnetApi storageApi) } var expiry = (tsExpiry.HasValue && tsExpiry.Value.Ticks > 0) ? DateTimeOffset.UtcNow.Ticks + tsExpiry.Value.Ticks : 0; - var input = new RawStringInput(RespCommand.GETEX, ref parseState, startIdx: 1, arg1: expiry); + var input = new StringInput(RespCommand.GETEX, ref parseState, startIdx: 1, arg1: expiry); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.GETEX(ref key, ref input, ref o); + var output = GetStringOutput(); + var status = storageApi.GETEX(key, ref input, ref output); switch (status) { case GarnetStatus.OK: - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); break; case GarnetStatus.NOTFOUND: - Debug.Assert(o.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); break; } @@ -140,15 +129,15 @@ bool NetworkGETAsync(ref TGarnetApi storageApi) #pragma warning restore VSTHRD200 // Use "Async" suffix for async methods where TGarnetApi : IGarnetApi { - var key = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); // Optimistically ask storage to write output to network buffer - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var output = GetStringOutput(); // Set up input to instruct storage to write output to IMemory rather than // network buffer, if the operation goes pending. - var input = new RawStringInput(RespCommand.ASYNC); + var input = new StringInput(RespCommand.ASYNC); - var status = storageApi.GET_WithPending(ref key, ref input, ref o, asyncStarted, out var pending); + var status = storageApi.GET_WithPending(key, ref input, ref output, asyncStarted, out var pending); if (pending) { @@ -159,13 +148,10 @@ bool NetworkGETAsync(ref TGarnetApi storageApi) switch (status) { case GarnetStatus.OK: - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); break; case GarnetStatus.NOTFOUND: - Debug.Assert(o.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); break; } @@ -179,11 +165,11 @@ bool NetworkGETAsync(ref TGarnetApi storageApi) bool NetworkGET_SG(ref TGarnetApi storageApi) where TGarnetApi : IGarnetAdvancedApi { - var key = parseState.GetArgSliceByRef(0).SpanByte; - RawStringInput input = new(RespCommand.GET, arg1: -1); + var key = parseState.GetArgSliceByRef(0); + StringInput input = new(RespCommand.GET, arg1: -1); var firstPending = -1; - (GarnetStatus, SpanByteAndMemory)[] outputArr = null; - SpanByteAndMemory o = new(dcurr, (int)(dend - dcurr)); + (GarnetStatus, StringOutput)[] outputArr = null; + var output = GetStringOutput(); var c = 0; for (; ; c++) @@ -194,13 +180,13 @@ bool NetworkGET_SG(ref TGarnetApi storageApi) // Store index in context, since completions are not in order long ctx = firstPending == -1 ? 0 : c - firstPending; - var status = storageApi.GET_WithPending(ref key, ref input, ref o, ctx, + var status = storageApi.GET_WithPending(key, ref input, ref output, ctx, out var isPending); if (isPending) { SetResult(c, ref firstPending, ref outputArr, status, default); - o = new SpanByteAndMemory(); + output = new StringOutput(); } else { @@ -209,16 +195,13 @@ bool NetworkGET_SG(ref TGarnetApi storageApi) if (firstPending == -1) { // Found in memory without IO, and no earlier pending, so we can add directly to the output - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; - o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + ProcessOutput(output.SpanByteAndMemory); + output = GetStringOutput(); } else { - SetResult(c, ref firstPending, ref outputArr, status, o); - o = new SpanByteAndMemory(); + SetResult(c, ref firstPending, ref outputArr, status, output); + output = new StringOutput(); } } else @@ -227,12 +210,12 @@ bool NetworkGET_SG(ref TGarnetApi storageApi) { // Realized not-found without IO, and no earlier pending, so we can add directly to the output WriteNull(); - o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + output = GetStringOutput(); } else { - SetResult(c, ref firstPending, ref outputArr, status, o); - o = new SpanByteAndMemory(); + SetResult(c, ref firstPending, ref outputArr, status, output); + output = new StringOutput(); } } } @@ -241,19 +224,16 @@ bool NetworkGET_SG(ref TGarnetApi storageApi) if (firstPending != -1) { // First complete all pending ops - storageApi.GET_CompletePending(outputArr, true); + _ = storageApi.GET_CompletePending(outputArr, true); // Write the outputs to network buffer for (var i = 0; i < c - firstPending; i++) { var status = outputArr[i].Item1; - var output = outputArr[i].Item2; + output = outputArr[i].Item2; if (status == GarnetStatus.OK) { - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; + ProcessOutput(output.SpanByteAndMemory); } else { @@ -286,7 +266,14 @@ private bool NetworkSET(ref TGarnetApi storageApi) var key = parseState.GetArgSliceByRef(0); var value = parseState.GetArgSliceByRef(1); - storageApi.SET(key, value); + var status = storageApi.SET(key, value); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -303,8 +290,8 @@ private bool NetworkGETSET(ref TGarnetApi storageApi) Debug.Assert(parseState.Count == 2); var key = parseState.GetArgSliceByRef(0); - return NetworkSET_Conditional(RespCommand.SET, 0, key, true, - false, false, ref storageApi); + return NetworkSET_Conditional(RespCommand.SET, 0, key, getValue: true, highPrecision: false, + ref storageApi); } /// @@ -326,12 +313,12 @@ private bool NetworkSetRange(ref TGarnetApi storageApi) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE); } - var input = new RawStringInput(RespCommand.SETRANGE, ref parseState, startIdx: 1); + var input = new StringInput(RespCommand.SETRANGE, ref parseState, startIdx: 1); Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length]; - var output = ArgSlice.FromPinnedSpan(outputBuffer); + var output = PinnedSpanByte.FromPinnedSpan(outputBuffer); - storageApi.SETRANGE(key, ref input, ref output); + _ = storageApi.SETRANGE(key, ref input, ref output); while (!RespWriteUtils.TryWriteIntegerFromBytes(outputBuffer.Slice(0, output.Length), ref dcurr, dend)) SendAndReset(); @@ -343,7 +330,6 @@ private bool NetworkGetRange(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var key = parseState.GetArgSliceByRef(0); - var sbKey = key.SpanByte; // Validate range if (!parseState.TryGetInt(1, out _) || !parseState.TryGetInt(2, out _)) @@ -351,24 +337,21 @@ private bool NetworkGetRange(ref TGarnetApi storageApi) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER); } - var input = new RawStringInput(RespCommand.GETRANGE, ref parseState, startIdx: 1); + var input = new StringInput(RespCommand.GETRANGE, ref parseState, startIdx: 1); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var output = GetStringOutput(); - var status = storageApi.GETRANGE(ref sbKey, ref input, ref o); + var status = storageApi.GETRANGE(key, ref input, ref output); if (status == GarnetStatus.OK) { sessionMetrics?.incr_total_found(); - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else { sessionMetrics?.incr_total_notfound(); - Debug.Assert(o.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_EMPTY, ref dcurr, dend)) SendAndReset(); } @@ -400,10 +383,10 @@ private bool NetworkSETEX(bool highPrecision, ref TGarnetApi storage ? TimeSpan.FromMilliseconds(expiry).Ticks : TimeSpan.FromSeconds(expiry).Ticks); - var sbVal = parseState.GetArgSliceByRef(2).SpanByte; + var value = parseState.GetArgSliceByRef(2); - var input = new RawStringInput(RespCommand.SETEX, 0, valMetadata); - _ = storageApi.SET(key, ref input, ref sbVal); + var input = new StringInput(RespCommand.SETEX, 0, valMetadata); + _ = storageApi.SET(key, ref input, value); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); @@ -424,7 +407,7 @@ private bool NetworkSETNX(bool highPrecision, ref TGarnetApi storage var key = parseState.GetArgSliceByRef(0); - var input = new RawStringInput(RespCommand.SETEXNX, ref parseState, startIdx: 1); + var input = new StringInput(RespCommand.SETEXNX, ref parseState, startIdx: 1); var status = storageApi.SET_Conditional(key, ref input); // The status returned for SETNX as NOTFOUND is the expected status in the happy path @@ -436,22 +419,18 @@ private bool NetworkSETNX(bool highPrecision, ref TGarnetApi storage } /// - /// SET EX NX [WITHETAG] + /// SET EX NX /// private bool NetworkSETEXNX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var key = parseState.GetArgSliceByRef(0); - var sbKey = key.SpanByte; - var val = parseState.GetArgSliceByRef(1); - var sbVal = val.SpanByte; var expiry = 0; ReadOnlySpan errorMessage = default; var existOptions = ExistOptions.None; var expOption = ExpirationOption.None; - var etagOption = EtagOption.None; var getValue = false; var tokenIdx = 2; @@ -515,32 +494,8 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) } else if (nextOpt.SequenceEqual(CmdStrings.GET)) { - if (etagOption != EtagOption.None) - { - // cannot do withEtag and getValue since withEtag SET already returns ETag in response - errorMessage = CmdStrings.RESP_ERR_WITHETAG_AND_GETVALUE; - break; - } - getValue = true; } - else if (nextOpt.SequenceEqual(CmdStrings.WITHETAG)) - { - if (etagOption != EtagOption.None) - { - errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; - break; - } - - if (getValue) - { - // cannot do withEtag and getValue since withEtag SET already returns ETag in response - errorMessage = CmdStrings.RESP_ERR_WITHETAG_AND_GETVALUE; - break; - } - - etagOption = EtagOption.WithETag; - } else { if (!optUpperCased) @@ -564,8 +519,6 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) return true; } - bool withEtag = etagOption == EtagOption.WithETag; - bool isHighPrecision = expOption == ExpirationOption.PX; switch (expOption) @@ -576,16 +529,13 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) switch (existOptions) { case ExistOptions.None: - return getValue || withEtag - ? NetworkSET_Conditional(RespCommand.SET, expiry, key, getValue, - isHighPrecision, withEtag, ref storageApi) - : NetworkSET_EX(RespCommand.SET, expOption, expiry, key, ref sbVal, ref storageApi); // Can perform a blind update + return getValue + ? NetworkSET_Conditional(RespCommand.SET, expiry, key, getValue, isHighPrecision, ref storageApi) + : NetworkSET_EX(RespCommand.SET, expOption, expiry, key, val, ref storageApi); // Can perform a blind update case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, key, - getValue, isHighPrecision, withEtag, ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXXX, expiry, key, getValue, isHighPrecision, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, key, - getValue, isHighPrecision, withEtag, ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, key, getValue, isHighPrecision, ref storageApi); } break; case ExpirationOption.KEEPTTL: @@ -594,14 +544,11 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) { case ExistOptions.None: // We can never perform a blind update due to KEEPTTL - return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, key - , getValue, highPrecision: false, withEtag, ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTL, expiry, key, getValue, highPrecision: false, ref storageApi); case ExistOptions.XX: - return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, key, - getValue, highPrecision: false, withEtag, ref storageApi); + return NetworkSET_Conditional(RespCommand.SETKEEPTTLXX, expiry, key, getValue, highPrecision: false, ref storageApi); case ExistOptions.NX: - return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, key, - getValue, highPrecision: false, withEtag, ref storageApi); + return NetworkSET_Conditional(RespCommand.SETEXNX, expiry, key, getValue, highPrecision: false, ref storageApi); } break; } @@ -611,8 +558,7 @@ private bool NetworkSETEXNX(ref TGarnetApi storageApi) return true; } - private unsafe bool NetworkSET_EX(RespCommand cmd, ExpirationOption expOption, int expiry, - ArgSlice key, ref SpanByte val, ref TGarnetApi storageApi) + private unsafe bool NetworkSET_EX(RespCommand cmd, ExpirationOption expOption, int expiry, PinnedSpanByte key, PinnedSpanByte val, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { Debug.Assert(cmd == RespCommand.SET); @@ -623,16 +569,16 @@ private unsafe bool NetworkSET_EX(RespCommand cmd, ExpirationOption ? TimeSpan.FromMilliseconds(expiry).Ticks : TimeSpan.FromSeconds(expiry).Ticks); - var input = new RawStringInput(cmd, 0, valMetadata); + var input = new StringInput(cmd, 0, valMetadata); - storageApi.SET(key, ref input, ref val); + storageApi.SET(key, ref input, val); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); return true; } - private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ArgSlice key, bool getValue, bool highPrecision, bool withEtag, ref TGarnetApi storageApi) + private bool NetworkSET_Conditional(RespCommand cmd, int expiry, PinnedSpanByte key, bool getValue, bool highPrecision, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var inputArg = expiry == 0 @@ -642,13 +588,10 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, Arg ? TimeSpan.FromMilliseconds(expiry).Ticks : TimeSpan.FromSeconds(expiry).Ticks); - var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); + var input = new StringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); - if (!getValue && !withEtag) + if (!getValue) { - // the following debug assertion is the catch any edge case leading to SETIFMATCH, or SETIFGREATER skipping the above block - Debug.Assert(cmd is not (RespCommand.SETIFMATCH or RespCommand.SETIFGREATER), "SETIFMATCH should have gone though pointing to right output variable"); - var status = storageApi.SET_Conditional(key, ref input); // KEEPTTL without flags doesn't care whether it was found or not. @@ -680,25 +623,14 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, Arg } else { - if (withEtag) - input.header.SetWithEtagFlag(); + input.header.SetSetGetFlag(); - if (getValue) - input.header.SetSetGetFlag(); + var output = GetStringOutput(); + GarnetStatus status = storageApi.SET_Conditional(key, ref input, ref output); - // anything with getValue or withEtag always writes to the buffer in the happy path - SpanByteAndMemory outputBuffer = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - GarnetStatus status = storageApi.SET_Conditional(key, ref input, ref outputBuffer); - - // The data will be on the buffer either when we know the response is ok or when the withEtag flag is set. - bool ok = status != GarnetStatus.NOTFOUND || withEtag; - - if (ok) + if (status != GarnetStatus.NOTFOUND) { - if (!outputBuffer.IsSpanByte) - SendAndReset(outputBuffer.Memory, outputBuffer.Length); - else - dcurr += outputBuffer.Length; + ProcessOutput(output.SpanByteAndMemory); } else { @@ -731,27 +663,21 @@ private bool NetworkIncrement(RespCommand cmd, ref TGarnetApi storag } Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length + 1]; - var output = ArgSlice.FromPinnedSpan(outputBuffer); + var output = PinnedSpanByte.FromPinnedSpan(outputBuffer); + StringOutput stringOutput = new(new SpanByteAndMemory(output)); - var input = new RawStringInput(cmd, 0, incrByValue); - storageApi.Increment(key, ref input, ref output); + var input = new StringInput(cmd, 0, incrByValue); + _ = storageApi.Increment(key, ref input, ref stringOutput); + output.Length = stringOutput.SpanByteAndMemory.Length; - var errorFlag = output.Length == NumUtils.MaximumFormatInt64Length + 1 - ? (OperationError)output.Span[0] - : OperationError.SUCCESS; - - switch (errorFlag) + if (!stringOutput.HasError) { - case OperationError.SUCCESS: - while (!RespWriteUtils.TryWriteIntegerFromBytes(outputBuffer.Slice(0, output.Length), ref dcurr, dend)) - SendAndReset(); - break; - case OperationError.NAN_OR_INFINITY: - case OperationError.INVALID_TYPE: - WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER); - break; - default: - throw new GarnetException($"Invalid OperationError {errorFlag}"); + while (!RespWriteUtils.TryWriteIntegerFromBytes(outputBuffer.Slice(0, output.Length), ref dcurr, dend)) + SendAndReset(); + } + else + { + WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER); } return true; @@ -776,22 +702,31 @@ private bool NetworkIncrementByFloat(ref TGarnetApi storageApi) } Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatDoubleLength + 1]; - var output = ArgSlice.FromPinnedSpan(outputBuffer); - var status = storageApi.IncrementByFloat(key, ref output, dbl); + var output = PinnedSpanByte.FromPinnedSpan(outputBuffer); + StringOutput stringOutput = new(new SpanByteAndMemory(output)); - switch (status) + _ = storageApi.IncrementByFloat(key, ref stringOutput, dbl); + output.Length = stringOutput.SpanByteAndMemory.Length; + + if (!stringOutput.HasError) { - case GarnetStatus.OK: - while (!RespWriteUtils.TryWriteBulkString(output.ReadOnlySpan, ref dcurr, dend)) - SendAndReset(); - break; - case GarnetStatus.WRONGTYPE: - default: - if ((OperationError)output.Span[0] == OperationError.NAN_OR_INFINITY) - WriteError(CmdStrings.RESP_ERR_GENERIC_NAN_INFINITY_INCR); - else - WriteError(CmdStrings.RESP_ERR_NOT_VALID_FLOAT); - break; + while (!RespWriteUtils.TryWriteBulkString(output.ReadOnlySpan, ref dcurr, dend)) + SendAndReset(); + } + else + { + if ((stringOutput.OutputFlags & StringOutputFlags.NaNOrInfinityError) != 0) + { + WriteError(CmdStrings.RESP_ERR_GENERIC_NAN_INFINITY_INCR); + } + else if ((stringOutput.OutputFlags & StringOutputFlags.InvalidTypeError) != 0) + { + WriteError(CmdStrings.RESP_ERR_NOT_VALID_FLOAT); + } + else + { + throw new GarnetException($"Unrecognized return output flags value: {stringOutput.OutputFlags}"); + } } return true; @@ -803,16 +738,16 @@ private bool NetworkIncrementByFloat(ref TGarnetApi storageApi) private bool NetworkAppend(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var sbKey = parseState.GetArgSliceByRef(0); - var input = new RawStringInput(RespCommand.APPEND, ref parseState, startIdx: 1); + var input = new StringInput(RespCommand.APPEND, ref parseState, startIdx: 1); Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length]; - var output = SpanByteAndMemory.FromPinnedSpan(outputBuffer); + var output = StringOutput.FromPinnedSpan(outputBuffer); - storageApi.APPEND(ref sbKey, ref input, ref output); + storageApi.APPEND(sbKey, ref input, ref output); - while (!RespWriteUtils.TryWriteIntegerFromBytes(outputBuffer.Slice(0, output.Length), ref dcurr, dend)) + while (!RespWriteUtils.TryWriteIntegerFromBytes(outputBuffer.Slice(0, output.SpanByteAndMemory.Length), ref dcurr, dend)) SendAndReset(); return true; @@ -946,7 +881,7 @@ private bool NetworkSTRLEN(ref TGarnetApi storageApi) //STRLEN key var key = parseState.GetArgSliceByRef(0); - var status = storageApi.GET(key, out var value); + var status = storageApi.GET(key, out PinnedSpanByte value); switch (status) { @@ -968,7 +903,7 @@ private bool NetworkSTRLEN(ref TGarnetApi storageApi) /// private void WriteCOMMANDResponse() { - var spam = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var spam = SpanByteAndMemory.FromPinnedPointer(dcurr, (int)(dend - dcurr)); var writer = new RespMemoryWriter(respProtocolVersion, ref spam); try @@ -1061,7 +996,7 @@ private bool NetworkCOMMAND_DOCS() { var count = parseState.Count; - var spam = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var spam = SpanByteAndMemory.FromPinnedPointer(dcurr, (int)(dend - dcurr)); var writer = new RespMemoryWriter(respProtocolVersion, ref spam); try @@ -1135,7 +1070,7 @@ private bool NetworkCOMMAND_INFO() } else { - var spam = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var spam = SpanByteAndMemory.FromPinnedPointer(dcurr, (int)(dend - dcurr)); var writer = new RespMemoryWriter(respProtocolVersion, ref spam); try @@ -1450,12 +1385,17 @@ private bool NetworkMemoryUsage(ref TGarnetApi storageApi) } } - var status = storageApi.MemoryUsageForKey(key, out var memoryUsage); + // Prepare input + var input = new UnifiedInput(RespCommand.MEMORY_USAGE); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); + + var status = storageApi.MEMORYUSAGE(key, ref input, ref output); if (status == GarnetStatus.OK) { - while (!RespWriteUtils.TryWriteInt32((int)memoryUsage, ref dcurr, dend)) - SendAndReset(); + ProcessOutput(output.SpanByteAndMemory); } else { @@ -1750,7 +1690,7 @@ private static void WriteClientInfo(IClusterProvider provider, StringBuilder int into.Append($" lib-ver={targetSession.clientLibVersion}"); } - bool ParseGETAndKey(ref SpanByte key) + bool ParseGETAndKey(ref PinnedSpanByte key) { var oldEndReadHead = readHead = endReadHead; var cmd = ParseCommand(writeErrorOnFailure: true, out var success); @@ -1760,7 +1700,7 @@ bool ParseGETAndKey(ref SpanByte key) endReadHead = readHead = oldEndReadHead; return false; } - key = parseState.GetArgSliceByRef(0).SpanByte; + key = parseState.GetArgSliceByRef(0); return true; } @@ -1787,13 +1727,13 @@ private bool TryGetSimpleCommandInfo(string cmdName, out SimpleRespCommandInfo s return true; } - static void SetResult(int c, ref int firstPending, ref (GarnetStatus, SpanByteAndMemory)[] outputArr, - GarnetStatus status, SpanByteAndMemory output) + static void SetResult(int c, ref int firstPending, ref (GarnetStatus, StringOutput)[] outputArr, + GarnetStatus status, StringOutput output) { const int initialBatchSize = 8; // number of items in initial batch if (firstPending == -1) { - outputArr = new (GarnetStatus, SpanByteAndMemory)[initialBatchSize]; + outputArr = new (GarnetStatus, StringOutput)[initialBatchSize]; firstPending = c; } @@ -1803,25 +1743,13 @@ static void SetResult(int c, ref int firstPending, ref (GarnetStatus, SpanByteAn if (c - firstPending >= outputArr.Length) { - int newCount = (int)NextPowerOf2(c - firstPending + 1); - var outputArr2 = new (GarnetStatus, SpanByteAndMemory)[newCount]; + int newCount = (int)BitOperations.RoundUpToPowerOf2((uint)(c - firstPending + 1)); + var outputArr2 = new (GarnetStatus, StringOutput)[newCount]; Array.Copy(outputArr, outputArr2, outputArr.Length); outputArr = outputArr2; } outputArr[c - firstPending] = (status, output); } - - static long NextPowerOf2(long v) - { - v--; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - v |= v >> 32; - return v + 1; - } } } \ No newline at end of file diff --git a/libs/server/Resp/BasicEtagCommands.cs b/libs/server/Resp/BasicEtagCommands.cs index 2fee440918d..c7dae8ead9b 100644 --- a/libs/server/Resp/BasicEtagCommands.cs +++ b/libs/server/Resp/BasicEtagCommands.cs @@ -23,21 +23,18 @@ private bool NetworkGETWITHETAG(ref TGarnetApi storageApi) Debug.Assert(parseState.Count == 1); var key = parseState.GetArgSliceByRef(0); - var input = new RawStringInput(RespCommand.GETWITHETAG); - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var input = new StringInput(RespCommand.GETWITHETAG); + var output = GetStringOutput(); var status = storageApi.GET(key, ref input, ref output); switch (status) { case GarnetStatus.NOTFOUND: - Debug.Assert(output.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); break; default: - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; + ProcessOutput(output.SpanByteAndMemory); break; } @@ -54,21 +51,18 @@ private bool NetworkGETIFNOTMATCH(ref TGarnetApi storageApi) Debug.Assert(parseState.Count == 2); var key = parseState.GetArgSliceByRef(0); - var input = new RawStringInput(RespCommand.GETIFNOTMATCH, ref parseState, startIdx: 1); - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var input = new StringInput(RespCommand.GETIFNOTMATCH, ref parseState, startIdx: 1); + var output = GetStringOutput(); var status = storageApi.GET(key, ref input, ref output); switch (status) { case GarnetStatus.NOTFOUND: - Debug.Assert(output.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); break; default: - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; + ProcessOutput(output.SpanByteAndMemory); break; } @@ -87,7 +81,7 @@ private bool NetworkDELIFGREATER(ref TGarnetApi storageApi) if (parseState.Count != 2) return AbortWithWrongNumberOfArguments(nameof(RespCommand.DELIFGREATER)); - SpanByte key = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); if (!parseState.TryGetLong(1, out long givenEtag) || givenEtag < 0) { return AbortWithErrorMessage(CmdStrings.RESP_ERR_INVALID_ETAG); @@ -96,10 +90,9 @@ private bool NetworkDELIFGREATER(ref TGarnetApi storageApi) // Conditional delete is not natively supported for records in the stable region. // To achieve this, we use a conditional DEL command to gain RMW (Read-Modify-Write) access, enabling deletion based on conditions. - RawStringInput input = new RawStringInput(RespCommand.DELIFGREATER, ref parseState, startIdx: 1); - input.header.SetWithEtagFlag(); + StringInput input = new StringInput(RespCommand.DELIFGREATER, ref parseState, startIdx: 1); - GarnetStatus status = storageApi.DEL_Conditional(ref key, ref input); + GarnetStatus status = storageApi.DEL_ETagConditional(key, ref input); int keysDeleted = status == GarnetStatus.OK ? 1 : 0; @@ -134,6 +127,62 @@ private bool NetworkSETIFGREATER(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi => NetworkSetETagConditional(RespCommand.SETIFGREATER, ref storageApi); + /// + /// SETWITHETAG key value [EX seconds | PX milliseconds] + /// Sets a key value pair with an ETag. If the key already exists, the value is overwritten and the ETag is incremented. + /// + private bool NetworkSETWITHETAG(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 2 || parseState.Count > 4) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.SETWITHETAG)); + } + + int expiry = 0; + ReadOnlySpan errorMessage = default; + ExpirationOption expOption = ExpirationOption.None; + + if (parseState.Count > 2) + { + // Parse EX | PX expiry + var tokenIdx = 2; + if (parseState.TryGetExpirationOption(tokenIdx, out expOption)) + { + if (expOption is not ExpirationOption.EX and not ExpirationOption.PX) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + } + else + { + tokenIdx++; + if (tokenIdx >= parseState.Count || !parseState.TryGetInt(tokenIdx, out expiry)) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER; + } + else if (expiry <= 0) + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_INVALIDEXP_IN_SET; + } + } + } + else + { + errorMessage = CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR; + } + } + + if (!errorMessage.IsEmpty) + { + while (!RespWriteUtils.TryWriteError(errorMessage, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var key = parseState.GetArgSliceByRef(0); + return ExecuteETagSetCommand(RespCommand.SETWITHETAG, expiry, expOption == ExpirationOption.PX, key, getValue: false, ref storageApi); + } + private bool NetworkSetETagConditional(RespCommand cmd, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { @@ -214,9 +263,31 @@ private bool NetworkSetETagConditional(RespCommand cmd, ref TGarnetA } var key = parseState.GetArgSliceByRef(0); + return ExecuteETagSetCommand(cmd, expiry, expOption == ExpirationOption.PX, key, getValue: !noGet, ref storageApi); + } + + /// + /// Shared implementation for ETag set commands (SETWITHETAG, SETIFMATCH, SETIFGREATER). + /// Builds input, calls SET_Conditional with output, and writes the response. + /// + private bool ExecuteETagSetCommand(RespCommand cmd, int expiry, bool highPrecision, PinnedSpanByte key, bool getValue, ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + var inputArg = expiry == 0 + ? 0 + : DateTimeOffset.UtcNow.Ticks + + (highPrecision + ? TimeSpan.FromMilliseconds(expiry).Ticks + : TimeSpan.FromSeconds(expiry).Ticks); + + var input = new StringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); - NetworkSET_Conditional(cmd, expiry, key, getValue: !noGet, highPrecision: expOption == ExpirationOption.PX, withEtag: true, ref storageApi); + if (getValue) + input.header.SetSetGetFlag(); + var output = GetStringOutput(); + storageApi.SET_ETagConditional(key, ref input, ref output); + ProcessOutput(output.SpanByteAndMemory); return true; } } diff --git a/libs/server/Resp/Bitmap/BitmapCommands.cs b/libs/server/Resp/Bitmap/BitmapCommands.cs index 20600939332..79658bf369b 100644 --- a/libs/server/Resp/Bitmap/BitmapCommands.cs +++ b/libs/server/Resp/Bitmap/BitmapCommands.cs @@ -9,7 +9,7 @@ namespace Garnet.server { - using SecondaryCommandList = List<(RespCommand, ArgSlice[])>; + using SecondaryCommandList = List<(RespCommand, PinnedSpanByte[])>; /// (1) , (2) , (3) /// overflow check, ptr protection, and status not found implemented for below @@ -135,7 +135,7 @@ private bool NetworkStringSetBit(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.SETBIT)); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); // Validate offset if (!parseState.TryGetLong(1, out var offset) || (offset < 0) || !BitmapManager.IsValidBitOffset(offset)) @@ -150,16 +150,13 @@ private bool NetworkStringSetBit(ref TGarnetApi storageApi) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_BIT_IS_NOT_INTEGER); } - var input = new RawStringInput(RespCommand.SETBIT, ref parseState, startIdx: 1, arg1: offset); + var input = new StringInput(RespCommand.SETBIT, ref parseState, startIdx: 1, arg1: offset); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.StringSetBit( - ref sbKey, - ref input, - ref o); + var output = GetStringOutput(); + var status = storageApi.StringSetBit(key, ref input, ref output); if (status == GarnetStatus.OK) - dcurr += o.Length; + dcurr += output.SpanByteAndMemory.Length; return true; } @@ -175,7 +172,7 @@ private bool NetworkStringGetBit(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.GETBIT)); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); // Validate offset if (!parseState.TryGetLong(1, out var offset) || (offset < 0) || !BitmapManager.IsValidBitOffset(offset)) @@ -183,16 +180,16 @@ private bool NetworkStringGetBit(ref TGarnetApi storageApi) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_BITOFFSET_IS_NOT_INTEGER); } - var input = new RawStringInput(RespCommand.GETBIT, ref parseState, startIdx: 1, arg1: offset); + var input = new StringInput(RespCommand.GETBIT, ref parseState, startIdx: 1, arg1: offset); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.StringGetBit(ref sbKey, ref input, ref o); + var output = GetStringOutput(); + var status = storageApi.StringGetBit(key, ref input, ref output); if (status == GarnetStatus.NOTFOUND) while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_RETURN_VAL_0, ref dcurr, dend)) SendAndReset(); else - dcurr += o.Length; + dcurr += output.SpanByteAndMemory.Length; return true; } @@ -211,7 +208,7 @@ private bool NetworkStringBitCount(ref TGarnetApi storageApi) } // <[Get Key]> - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); // Extract parameters in command order: // start, end, [BIT|BYTE] @@ -234,20 +231,17 @@ private bool NetworkStringBitCount(ref TGarnetApi storageApi) { return AbortWithErrorMessage(CmdStrings.RESP_SYNTAX_ERROR); } - - } } - var input = new RawStringInput(RespCommand.BITCOUNT, ref parseState, startIdx: 1, arg1: useBitIndex ? 1 : 0); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.StringBitCount(ref sbKey, ref input, ref o); + var input = new StringInput(RespCommand.BITCOUNT, ref parseState, startIdx: 1, arg1: useBitIndex ? 1 : 0); + + var output = GetStringOutput(); + + var status = storageApi.StringBitCount(key, ref input, ref output); if (status == GarnetStatus.OK) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else if (status == GarnetStatus.NOTFOUND) { @@ -271,7 +265,7 @@ private bool NetworkStringBitPosition(ref TGarnetApi storageApi) } // <[Get Key]> - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); // Validate value var bSetValSlice = parseState.GetArgSliceByRef(1).ReadOnlySpan; @@ -329,16 +323,15 @@ private bool NetworkStringBitPosition(ref TGarnetApi storageApi) return true; } - var input = new RawStringInput(RespCommand.BITPOS, ref parseState, startIdx: 1); - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.StringBitPosition(ref sbKey, ref input, ref o); + var input = new StringInput(RespCommand.BITPOS, ref parseState, startIdx: 1); + + var output = GetStringOutput(); + + var status = storageApi.StringBitPosition(key, ref input, ref output); if (status == GarnetStatus.OK) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else if (status == GarnetStatus.NOTFOUND) { @@ -372,7 +365,7 @@ private bool NetworkStringBitOperation(BitmapOperation bitOp, ref TG return AbortWithErrorMessage(CmdStrings.RESP_ERR_BITOP_KEY_LIMIT); } - var input = new RawStringInput(RespCommand.BITOP, ref parseState); + var input = new StringInput(RespCommand.BITOP, ref parseState); _ = storageApi.StringBitOperation(ref input, bitOp, out var result); while (!RespWriteUtils.TryWriteInt64(result, ref dcurr, dend)) @@ -394,10 +387,10 @@ private bool StringBitField(ref TGarnetApi storageApi) // BITFIELD key [GET encoding offset] [SET encoding offset value] [INCRBY encoding offset increment] [OVERFLOW WRAP| SAT | FAIL] // Extract Key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); var isOverflowTypeSet = false; - ArgSlice overflowTypeSlice = default; + PinnedSpanByte overflowTypeSlice = default; var secondaryCommandArgs = new SecondaryCommandList(); var currTokenIdx = 1; @@ -476,7 +469,7 @@ private bool StringBitField(ref TGarnetApi storageApi) } } - return StringBitFieldAction(ref storageApi, ref sbKey, RespCommand.BITFIELD, + return StringBitFieldAction(ref storageApi, key, RespCommand.BITFIELD, secondaryCommandArgs, isOverflowTypeSet, overflowTypeSlice); } @@ -493,7 +486,7 @@ private bool StringBitFieldReadOnly(ref TGarnetApi storageApi) // BITFIELD_RO key [GET encoding offset [GET encoding offset] ... ] // Extract Key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); var secondaryCommandArgs = new SecondaryCommandList(); @@ -535,21 +528,21 @@ private bool StringBitFieldReadOnly(ref TGarnetApi storageApi) secondaryCommandArgs.Add((RespCommand.GET, [commandSlice, encodingSlice, offsetSlice])); } - return StringBitFieldAction(ref storageApi, ref sbKey, RespCommand.BITFIELD_RO, secondaryCommandArgs); + return StringBitFieldAction(ref storageApi, key, RespCommand.BITFIELD_RO, secondaryCommandArgs); } private bool StringBitFieldAction(ref TGarnetApi storageApi, - ref SpanByte sbKey, + PinnedSpanByte sbKey, RespCommand cmd, SecondaryCommandList secondaryCommandArgs, bool isOverflowTypeSet = false, - ArgSlice overflowTypeSlice = default) + PinnedSpanByte overflowTypeSlice = default) where TGarnetApi : IGarnetApi { while (!RespWriteUtils.TryWriteArrayLength(secondaryCommandArgs.Count, ref dcurr, dend)) SendAndReset(); - var input = new RawStringInput(cmd); + var input = new StringInput(cmd); for (var i = 0; i < secondaryCommandArgs.Count; i++) { @@ -569,8 +562,8 @@ private bool StringBitFieldAction(ref TGarnetApi storageApi, input.parseState = parseState; - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.StringBitField(ref sbKey, ref input, opCode, + var output = GetStringOutput(); + var status = storageApi.StringBitField(sbKey, ref input, opCode, ref output); if (status == GarnetStatus.NOTFOUND && opCode == RespCommand.GET) @@ -580,10 +573,7 @@ private bool StringBitFieldAction(ref TGarnetApi storageApi, } else { - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; + ProcessOutput(output.SpanByteAndMemory); } } diff --git a/libs/server/Resp/ClientCommands.cs b/libs/server/Resp/ClientCommands.cs index 26a37bb8c0e..82ebcc966cc 100644 --- a/libs/server/Resp/ClientCommands.cs +++ b/libs/server/Resp/ClientCommands.cs @@ -264,13 +264,13 @@ private bool NetworkCLIENTKILL() var filterSpan = filter.Span; var valueIx = argIx + 1; - ref var value = ref parseState.GetArgSliceByRef(valueIx); + var value = parseState.GetArgSliceByRef(valueIx); AsciiUtils.ToUpperInPlace(filterSpan); if (filterSpan.SequenceEqual(CmdStrings.ID)) { - if (!ParseUtils.TryReadLong(ref value, out var idParsed)) + if (!ParseUtils.TryReadLong(value, out var idParsed)) { return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrShouldBeGreaterThanZero, "client-id"))); } @@ -291,7 +291,7 @@ private bool NetworkCLIENTKILL() if (!parseState.TryGetClientType(valueIx, out var typeParsed)) { - var typeStr = ParseUtils.ReadString(ref value); + var typeStr = ParseUtils.ReadString(value); return AbortWithErrorMessage(Encoding.UTF8.GetBytes(string.Format(CmdStrings.GenericUnknownClientType, typeStr))); } @@ -307,7 +307,7 @@ private bool NetworkCLIENTKILL() return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrDuplicateFilter, "USER"))); } - user = ParseUtils.ReadString(ref value); + user = ParseUtils.ReadString(value); } else if (filterSpan.SequenceEqual(CmdStrings.ADDR)) { @@ -316,7 +316,7 @@ private bool NetworkCLIENTKILL() return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrDuplicateFilter, "ADDR"))); } - addr = ParseUtils.ReadString(ref value); + addr = ParseUtils.ReadString(value); } else if (filterSpan.SequenceEqual(CmdStrings.LADDR)) { @@ -325,7 +325,7 @@ private bool NetworkCLIENTKILL() return AbortWithErrorMessage(Encoding.ASCII.GetBytes(string.Format(CmdStrings.GenericErrDuplicateFilter, "LADDR"))); } - lAddr = ParseUtils.ReadString(ref value); + lAddr = ParseUtils.ReadString(value); } else if (filterSpan.SequenceEqual(CmdStrings.SKIPME)) { @@ -351,7 +351,7 @@ private bool NetworkCLIENTKILL() } else if (filterSpan.SequenceEqual(CmdStrings.MAXAGE)) { - if (!ParseUtils.TryReadLong(ref value, out var maxAgeParsed)) + if (!ParseUtils.TryReadLong(value, out var maxAgeParsed)) { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); } diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index e8c5ba5fb9e..5992adacddb 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -31,9 +31,8 @@ static partial class CmdStrings public static ReadOnlySpan REWRITE => "REWRITE"u8; public static ReadOnlySpan rewrite => "rewrite"u8; public static ReadOnlySpan CONFIG => "CONFIG"u8; - public static ReadOnlySpan Memory => "memory"u8; - public static ReadOnlySpan ObjLogMemory => "obj-log-memory"u8; - public static ReadOnlySpan ObjHeapMemory => "obj-heap-memory"u8; + public static ReadOnlySpan MainLogMemory => "memory"u8; + public static ReadOnlySpan ReadCacheMemory => "readcache-memory"u8; public static ReadOnlySpan Index => "index"u8; public static ReadOnlySpan ObjIndex => "obj-index"u8; public static ReadOnlySpan CertFileName => "cert-file-name"u8; @@ -63,6 +62,15 @@ static partial class CmdStrings public static ReadOnlySpan HISTOGRAM => "HISTOGRAM"u8; public static ReadOnlySpan histogram => "histogram"u8; public static ReadOnlySpan REPLICAOF => "REPLICAOF"u8; + public static ReadOnlySpan RICREATE => "RI.CREATE"u8; + public static ReadOnlySpan RISET => "RI.SET"u8; + public static ReadOnlySpan RIGET => "RI.GET"u8; + public static ReadOnlySpan RIDEL => "RI.DEL"u8; + public static ReadOnlySpan RIRANGE => "RI.RANGE"u8; + public static ReadOnlySpan RISCAN => "RI.SCAN"u8; + public static ReadOnlySpan RIEXISTS => "RI.EXISTS"u8; + public static ReadOnlySpan RICONFIG => "RI.CONFIG"u8; + public static ReadOnlySpan RIMETRICS => "RI.METRICS"u8; public static ReadOnlySpan SLAVEOF => "SLAVEOF"u8; public static ReadOnlySpan SECONDARYOF => "SECONDARYOF"u8; public static ReadOnlySpan HELP => "HELP"u8; @@ -117,7 +125,6 @@ static partial class CmdStrings public static ReadOnlySpan NX => "NX"u8; public static ReadOnlySpan XX => "XX"u8; public static ReadOnlySpan CH => "CH"u8; - public static ReadOnlySpan WITHETAG => "WITHETAG"u8; public static ReadOnlySpan UNSAFETRUNCATELOG => "UNSAFETRUNCATELOG"u8; public static ReadOnlySpan SAMPLES => "SAMPLES"u8; public static ReadOnlySpan RANK => "RANK"u8; @@ -158,6 +165,7 @@ static partial class CmdStrings public static ReadOnlySpan GETIFNOTMATCH => "GETIFNOTMATCH"u8; public static ReadOnlySpan SETIFMATCH => "SETIFMATCH"u8; public static ReadOnlySpan SETIFGREATER => "SETIFGREATER"u8; + public static ReadOnlySpan SETWITHETAG => "SETWITHETAG"u8; public static ReadOnlySpan DELIFGREATER => "DELIFGREATER"u8; public static ReadOnlySpan FIELDS => "FIELDS"u8; public static ReadOnlySpan MEMBERS => "MEMBERS"u8; @@ -193,7 +201,6 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_WRONG_TYPE => "WRONGTYPE Operation against a key holding the wrong kind of value."u8; public static ReadOnlySpan RESP_ERR_WRONG_TYPE_HLL => "WRONGTYPE Key is not a valid HyperLogLog string value."u8; public static ReadOnlySpan RESP_ERR_EXEC_ABORT => "EXECABORT Transaction discarded because of previous errors."u8; - public static ReadOnlySpan RESP_ERR_ETAG_ON_CUSTOM_PROC => "WRONGTYPE Key with etag cannot be used for custom procedure."u8; public static ReadOnlySpan RESP_ERR_NOSCRIPT => "ERR This Redis command is not allowed from script"u8; @@ -212,7 +219,6 @@ static partial class CmdStrings public static ReadOnlySpan RESP_ERR_GENERIC_WATCH_IN_MULTI => "ERR WATCH inside MULTI is not allowed"u8; public static ReadOnlySpan RESP_ERR_GENERIC_INVALIDEXP_IN_SET => "ERR invalid expire time in 'set' command"u8; public static ReadOnlySpan RESP_ERR_GENERIC_SYNTAX_ERROR => "ERR syntax error"u8; - public static ReadOnlySpan RESP_ERR_WITHETAG_AND_GETVALUE => "ERR WITHETAG option not allowed with GET inside of SET"u8; public static ReadOnlySpan RESP_ERR_GENERIC_NAN_INFINITY => "ERR value is NaN or Infinity"u8; public static ReadOnlySpan RESP_ERR_GENERIC_NAN_INFINITY_INCR => "ERR increment would produce NaN or Infinity"u8; public static ReadOnlySpan RESP_ERR_GENERIC_SCORE_NAN => "ERR resulting score is not a number (NaN)"u8; @@ -339,7 +345,8 @@ static partial class CmdStrings public const string GenericErrIndexSizeSmallerThanCurrent = "ERR Cannot set dynamic index size smaller than current index size (option: '{0}')"; public const string GenericErrIndexSizeGrowFailed = "ERR failed to grow index size beyond current size (option: '{0}')"; public const string GenericErrMemorySizeGreaterThanBuffer = "ERR Cannot set dynamic memory size greater than configured circular buffer size (option: '{0}')"; - public const string GenericErrHeapMemorySizeTrackerNotRunning = "ERR Cannot adjust object store heap memory size when size tracker is not running (option: '{0}')"; + public const string GenericErrMainLogMemorySizeTrackerNotRunning = "ERR Cannot adjust main log memory size configuration when size tracker is not running (option: '{0}')"; + public const string GenericErrReadCacheMemorySizeTrackerNotRunning = "ERR Cannot adjust readcache memory size configuration when size tracker is not running (option: '{0}')"; /// /// Response errors while scripting @@ -358,6 +365,8 @@ static partial class CmdStrings public static ReadOnlySpan hash => "hash"u8; public static ReadOnlySpan STRING => "STRING"u8; public static ReadOnlySpan stringt => "string"u8; + public static ReadOnlySpan rangeindext => "rangeindex"u8; + public static ReadOnlySpan none => "none"u8; /// /// Register object types @@ -441,7 +450,6 @@ static partial class CmdStrings public static ReadOnlySpan spublish => "SPUBLISH"u8; public static ReadOnlySpan mtasks => "MTASKS"u8; public static ReadOnlySpan reserve => "RESERVE"u8; - public static ReadOnlySpan aofsync => "AOFSYNC"u8; public static ReadOnlySpan appendlog => "APPENDLOG"u8; public static ReadOnlySpan attach_sync => "ATTACH_SYNC"u8; public static ReadOnlySpan banlist => "BANLIST"u8; @@ -452,7 +460,10 @@ static partial class CmdStrings public static ReadOnlySpan initiate_replica_sync => "INITIATE_REPLICA_SYNC"u8; public static ReadOnlySpan send_ckpt_file_segment => "SEND_CKPT_FILE_SEGMENT"u8; public static ReadOnlySpan send_ckpt_metadata => "SEND_CKPT_METADATA"u8; + public static ReadOnlySpan snapshot_data => "SNAPSHOT_DATA"u8; + public static ReadOnlySpan mlog_key_time => "MLOG_KEY_TIME"u8; public static ReadOnlySpan cluster_sync => "SYNC"u8; + public static ReadOnlySpan cluster_advance_time => "ADVANCE_TIME"u8; // Lua scripting strings public static ReadOnlySpan LUA_OK => "OK"u8; diff --git a/libs/server/Resp/GarnetDatabaseSession.cs b/libs/server/Resp/GarnetDatabaseSession.cs index e2c2f917708..e4ea1574d7b 100644 --- a/libs/server/Resp/GarnetDatabaseSession.cs +++ b/libs/server/Resp/GarnetDatabaseSession.cs @@ -2,29 +2,9 @@ // Licensed under the MIT license. using System; -using Tsavorite.core; namespace Garnet.server { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - using LockableGarnetApi = GarnetApi, - SpanByteAllocator>>, - LockableContext>, - GenericAllocator>>>, - LockableContext, - SpanByteAllocator>>>; - /// /// Represents a logical database session in Garnet /// @@ -45,10 +25,20 @@ internal class GarnetDatabaseSession : IDisposable /// public BasicGarnetApi GarnetApi { get; } + /// + /// Consistent Garnet API + /// + public ConsistentReadGarnetApi ConsistentGarnetApi { get; } + /// /// Lockable Garnet API /// - public LockableGarnetApi LockableGarnetApi { get; } + public TransactionalGarnetApi TransactionalGarnetApi { get; } + + /// + /// Lockable Consistent Garnet API + /// + public TransactionalConsistentReadGarnetApi TransactionalConsistentGarnetApi { get; } /// /// Transaction manager @@ -57,13 +47,22 @@ internal class GarnetDatabaseSession : IDisposable bool disposed = false; - public GarnetDatabaseSession(int id, StorageSession storageSession, BasicGarnetApi garnetApi, LockableGarnetApi lockableGarnetApi, TransactionManager txnManager) + public GarnetDatabaseSession( + int id, + StorageSession storageSession, + BasicGarnetApi garnetApi, + TransactionalGarnetApi lockableGarnetApi, + TransactionManager txnManager, + ConsistentReadGarnetApi consistentGarnetApi = default, + TransactionalConsistentReadGarnetApi transactionalConsistentGarnetApi = default) { this.Id = id; this.StorageSession = storageSession; this.GarnetApi = garnetApi; - this.LockableGarnetApi = lockableGarnetApi; + this.TransactionalGarnetApi = lockableGarnetApi; this.TransactionManager = txnManager; + this.ConsistentGarnetApi = consistentGarnetApi; + this.TransactionalConsistentGarnetApi = transactionalConsistentGarnetApi; } public GarnetDatabaseSession(int id, GarnetDatabaseSession srcSession) @@ -71,7 +70,7 @@ public GarnetDatabaseSession(int id, GarnetDatabaseSession srcSession) this.Id = id; this.StorageSession = srcSession.StorageSession; this.GarnetApi = srcSession.GarnetApi; - this.LockableGarnetApi = srcSession.LockableGarnetApi; + this.TransactionalGarnetApi = srcSession.TransactionalGarnetApi; this.TransactionManager = srcSession.TransactionManager; } diff --git a/libs/server/Resp/HyperLogLog/HyperLogLog.cs b/libs/server/Resp/HyperLogLog/HyperLogLog.cs index 6bd81d9f657..ed3626a8b1c 100644 --- a/libs/server/Resp/HyperLogLog/HyperLogLog.cs +++ b/libs/server/Resp/HyperLogLog/HyperLogLog.cs @@ -322,7 +322,7 @@ private bool IsValidSparseStream(byte* ptr) /// /// /// - public void Init(ref RawStringInput input, byte* value, int vlen) + public void Init(ref StringInput input, byte* value, int vlen) { var dense = vlen == this.DenseBytes; @@ -368,7 +368,7 @@ public void InitDense(byte* ptr) /// /// /// - public int SparseInitialLength(ref RawStringInput input) + public int SparseInitialLength(ref StringInput input) { var count = input.parseState.Count; return SparseInitialLength(count); @@ -414,7 +414,7 @@ private int SparseRequiredBytes(int cnt) /// /// Return length of new value /// - public int UpdateGrow(ref RawStringInput input, byte* value) + public int UpdateGrow(ref StringInput input, byte* value) { var count = input.parseState.Count; @@ -428,7 +428,8 @@ public int UpdateGrow(ref RawStringInput input, byte* value) if (IsDense(value)) return this.DenseBytes; - throw new GarnetException("HyperLogLog UpdateGrowV2 invalid data structure type"); + // This is called during GetRMWModifiedFieldInfo so be consistent between this and the actual updaters + throw new GarnetException(CmdStrings.RESP_ERR_WRONG_TYPE_HLL); } /// @@ -484,7 +485,7 @@ public void CopyUpdateMerge(byte* srcHLLPtr, byte* oldDstHLLPtr, byte* newDstHLL /// /// /// - public bool CopyUpdate(ref RawStringInput input, byte* oldValue, byte* newValue, int newValueLen) + public bool CopyUpdate(ref StringInput input, byte* oldValue, byte* newValue, int newValueLen) { var fUpdated = false; @@ -572,7 +573,7 @@ public bool DenseToDense(byte* srcDenseBlob, byte* dstDenseBlob) /// /// /// - public bool Update(ref RawStringInput input, byte* value, int valueLen, ref bool updated) + public bool Update(ref StringInput input, byte* value, int valueLen, ref bool updated) { var count = input.parseState.Count; @@ -644,13 +645,13 @@ private bool UpdateDenseRegister(byte* ptr, ushort idx, byte cntlz) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void SetNonZero(byte* p, byte cnt) => *p = (byte)(cnt - 1); // 0vvv vvvv - private bool IterateUpdate(ref RawStringInput input, byte* value, bool dense) + private bool IterateUpdate(ref StringInput input, byte* value, bool dense) { var updated = false; for (var i = 0; i < input.parseState.Count; i++) { var currElement = input.parseState.GetArgSliceByRef(i); - var hashValue = (long)HashUtils.MurmurHash2x64A(currElement.ptr, currElement.Length); + var hashValue = (long)HashUtils.MurmurHash2x64A(currElement.ToPointer(), currElement.Length); updated |= (dense ? UpdateDense(value, hashValue) : UpdateSparse(value, hashValue)); } return updated; diff --git a/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs b/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs index 3ccff292178..2b1552f00e0 100644 --- a/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs +++ b/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs @@ -4,7 +4,6 @@ //#define HLL_SINGLE_PFADD_ENABLED using Garnet.common; -using Tsavorite.core; namespace Garnet.server { @@ -24,17 +23,17 @@ private bool HyperLogLogAdd(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.PFADD)); } - var input = new RawStringInput(RespCommand.PFADD); + var input = new StringInput(RespCommand.PFADD); var output = stackalloc byte[1]; byte pfaddUpdated = 0; - var key = parseState.GetArgSliceByRef(0).SpanByte; + var key = parseState.GetArgSliceByRef(0); for (var i = 1; i < parseState.Count; i++) { input.parseState = parseState.Slice(i, 1); - var o = new SpanByteAndMemory(output, 1); - storageApi.HyperLogLogAdd(ref key, ref input, ref o); + var o = StringOutput.FromPinnedPointer(output, 1); + storageApi.HyperLogLogAdd(key, ref input, ref o); // Invalid HLL Type if (*output == 0xFF) @@ -76,7 +75,7 @@ private bool HyperLogLogLength(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.PFCOUNT)); } - var input = new RawStringInput(RespCommand.PFCOUNT, ref parseState); + var input = new StringInput(RespCommand.PFCOUNT, ref parseState); storageApi.HyperLogLogLength(ref input, out var cardinality, out var error); if (error) @@ -105,7 +104,7 @@ private bool HyperLogLogMerge(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.PFMERGE)); } - var input = new RawStringInput(RespCommand.PFMERGE, ref parseState); + var input = new StringInput(RespCommand.PFMERGE, ref parseState); var status = storageApi.HyperLogLogMerge(ref input, out var error); diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index dde122b592c..84863d5d7e6 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -98,19 +98,21 @@ bool NetworkRESTORE(ref TGarnetApi storageApi) parseState.InitializeWithArgument(valArgSlice); - RawStringInput input; + StringInput input; if (expiry > 0) { var inputArg = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromSeconds(expiry).Ticks; - input = new RawStringInput(RespCommand.SETEXNX, ref parseState, arg1: inputArg); + input = new StringInput(RespCommand.SETEXNX, ref parseState, arg1: inputArg); } else { - input = new RawStringInput(RespCommand.SETEXNX, ref parseState); + input = new StringInput(RespCommand.SETEXNX, ref parseState); } var status = storageApi.SET_Conditional(key, ref input); + scratchBufferBuilder.RewindScratchBuffer(valArgSlice); + if (status is GarnetStatus.NOTFOUND) { while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) @@ -137,9 +139,9 @@ bool NetworkDUMP(ref TGarnetApi storageApi) var key = parseState.GetArgSliceByRef(0); - var status = storageApi.GET(key, out var value); + var status = storageApi.GET(key, out PinnedSpanByte value); - if (status is GarnetStatus.NOTFOUND) + if (status is GarnetStatus.NOTFOUND or GarnetStatus.WRONGTYPE) { WriteNull(); return true; @@ -224,27 +226,13 @@ bool NetworkDUMP(ref TGarnetApi storageApi) private bool NetworkRENAME(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - // one optional command for with etag - if (parseState.Count < 2 || parseState.Count > 3) - { + if (parseState.Count != 2) return AbortWithWrongNumberOfArguments(nameof(RespCommand.RENAME)); - } var oldKeySlice = parseState.GetArgSliceByRef(0); var newKeySlice = parseState.GetArgSliceByRef(1); - var withEtag = false; - if (parseState.Count == 3) - { - if (!parseState.GetArgSliceByRef(2).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHETAG)) - { - return AbortWithErrorMessage(string.Format(CmdStrings.GenericErrUnsupportedOption, parseState.GetString(2))); - } - - withEtag = true; - } - - var status = storageApi.RENAME(oldKeySlice, newKeySlice, withEtag); + var status = storageApi.RENAME(oldKeySlice, newKeySlice); switch (status) { @@ -266,27 +254,13 @@ private bool NetworkRENAME(ref TGarnetApi storageApi) private bool NetworkRENAMENX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - // one optional command for with etag - if (parseState.Count < 2 || parseState.Count > 3) - { + if (parseState.Count != 2) return AbortWithWrongNumberOfArguments(nameof(RespCommand.RENAMENX)); - } var oldKeySlice = parseState.GetArgSliceByRef(0); var newKeySlice = parseState.GetArgSliceByRef(1); - var withEtag = false; - if (parseState.Count == 3) - { - if (!parseState.GetArgSliceByRef(2).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.WITHETAG)) - { - return AbortWithErrorMessage(string.Format(CmdStrings.GenericErrUnsupportedOption, parseState.GetString(2))); - } - - withEtag = true; - } - - var status = storageApi.RENAMENX(oldKeySlice, newKeySlice, out var result, withEtag); + var status = storageApi.RENAMENX(oldKeySlice, newKeySlice, out var result); if (status == GarnetStatus.OK) { @@ -318,20 +292,17 @@ private bool NetworkGETDEL(ref TGarnetApi garnetApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.GETDEL)); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = garnetApi.GETDEL(ref sbKey, ref o); + var sbKey = parseState.GetArgSliceByRef(0); + var output = GetStringOutput(); + var status = garnetApi.GETDEL(sbKey, ref output); if (status == GarnetStatus.OK) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else { - Debug.Assert(o.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); WriteNull(); } @@ -355,10 +326,15 @@ private bool NetworkEXISTS(ref TGarnetApi storageApi) var exists = 0; + // Prepare input + var input = new UnifiedInput(RespCommand.EXISTS); + + var output = new UnifiedOutput(); + for (var i = 0; i < parseState.Count; i++) { var key = parseState.GetArgSliceByRef(i); - var status = storageApi.EXISTS(key); + var status = storageApi.EXISTS(key, ref input, ref output); if (status == GarnetStatus.OK) exists++; } @@ -451,13 +427,16 @@ private bool NetworkEXPIRE(RespCommand command, ref TGarnetApi stora // Encode expiration time and expiration option and pass them into the input object var expirationWithOption = new ExpirationWithOption(expirationTimeInTicks, expireOption); - var input = new RawStringInput(RespCommand.EXPIRE, arg1: expirationWithOption.Word); - var status = storageApi.EXPIRE(key, ref input, out var timeoutSet); + var input = new UnifiedInput(RespCommand.EXPIRE, arg1: expirationWithOption.Word); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); - if (status == GarnetStatus.OK && timeoutSet) + var status = storageApi.EXPIRE(key, ref input, ref output); + + if (status == GarnetStatus.OK) { - while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_RETURN_VAL_1, ref dcurr, dend)) - SendAndReset(); + ProcessOutput(output.SpanByteAndMemory); } else { @@ -483,12 +462,18 @@ private bool NetworkPERSIST(ref TGarnetApi storageApi) } var key = parseState.GetArgSliceByRef(0); - var status = storageApi.PERSIST(key); + + // Prepare input + var input = new UnifiedInput(RespCommand.PERSIST); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); + + var status = storageApi.PERSIST(key, ref input, ref output); if (status == GarnetStatus.OK) { - while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_RETURN_VAL_1, ref dcurr, dend)) - SendAndReset(); + ProcessOutput(output.SpanByteAndMemory); } else { @@ -510,21 +495,22 @@ private bool NetworkTTL(RespCommand command, ref TGarnetApi storageA { if (parseState.Count != 1) { - return AbortWithWrongNumberOfArguments(nameof(RespCommand.PERSIST)); + return AbortWithWrongNumberOfArguments(command.ToString()); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = command == RespCommand.TTL ? - storageApi.TTL(ref sbKey, StoreType.All, ref o) : - storageApi.PTTL(ref sbKey, StoreType.All, ref o); + var key = parseState.GetArgSliceByRef(0); + + // Prepare input + var input = new UnifiedInput(command); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); + + var status = storageApi.TTL(key, ref input, ref output); if (status == GarnetStatus.OK) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else { @@ -549,18 +535,19 @@ private bool NetworkEXPIRETIME(RespCommand command, ref TGarnetApi s return AbortWithWrongNumberOfArguments(nameof(RespCommand.EXPIRETIME)); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = command == RespCommand.EXPIRETIME ? - storageApi.EXPIRETIME(ref sbKey, StoreType.All, ref o) : - storageApi.PEXPIRETIME(ref sbKey, StoreType.All, ref o); + var key = parseState.GetArgSliceByRef(0); + + // Prepare input + var input = new UnifiedInput(command); + + // Prepare UnifiedOutput output + var output = GetUnifiedOutput(); + + var status = storageApi.EXPIRETIME(key, ref input, ref output); if (status == GarnetStatus.OK) { - if (!o.IsSpanByte) - SendAndReset(o.Memory, o.Length); - else - dcurr += o.Length; + ProcessOutput(output.SpanByteAndMemory); } else { diff --git a/libs/server/Resp/LocalServerSession.cs b/libs/server/Resp/LocalServerSession.cs index 3bf4a4ca1c5..39cbeaf0a27 100644 --- a/libs/server/Resp/LocalServerSession.cs +++ b/libs/server/Resp/LocalServerSession.cs @@ -4,20 +4,9 @@ using System; using System.Diagnostics; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.server { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - /// /// Local server session /// @@ -29,12 +18,18 @@ public class LocalServerSession : IDisposable readonly StoreWrapper storeWrapper; readonly StorageSession storageSession; readonly ScratchBufferBuilder scratchBufferBuilder; + readonly ScratchBufferAllocator scratchBufferAllocator; /// /// Basic Garnet API /// public BasicGarnetApi BasicGarnetApi; + /// + /// Basic Vector Context + /// + public VectorBasicContext VectorBasicContext; + /// /// Create new local server session /// @@ -50,14 +45,15 @@ public LocalServerSession(StoreWrapper storeWrapper) // Initialize session-local scratch buffer of size 64 bytes, used for constructing arguments in GarnetApi this.scratchBufferBuilder = new ScratchBufferBuilder(); + this.scratchBufferAllocator = new ScratchBufferAllocator(); var dbRes = storeWrapper.TryGetOrAddDatabase(0, out var database, out _); Debug.Assert(dbRes, "Should always be able to get DB 0"); // Create storage session and API - this.storageSession = new StorageSession(storeWrapper, scratchBufferBuilder, sessionMetrics, LatencyMetrics, dbId: 0, database.VectorManager, logger); - - this.BasicGarnetApi = new BasicGarnetApi(storageSession, storageSession.basicContext, storageSession.objectStoreBasicContext); + this.storageSession = new StorageSession(storeWrapper, scratchBufferBuilder, scratchBufferAllocator, sessionMetrics, LatencyMetrics, dbId: 0, readSessionState: null, database.VectorManager, logger); + this.BasicGarnetApi = new BasicGarnetApi(storageSession, storageSession.stringBasicContext, storageSession.objectBasicContext, storageSession.unifiedBasicContext); + this.VectorBasicContext = storageSession.vectorBasicContext; } /// diff --git a/libs/server/Resp/MGetReadArgBatch.cs b/libs/server/Resp/MGetReadArgBatch.cs index 77bcfc36006..acc58c6bd62 100644 --- a/libs/server/Resp/MGetReadArgBatch.cs +++ b/libs/server/Resp/MGetReadArgBatch.cs @@ -4,7 +4,9 @@ using System; using System.Buffers; using System.Diagnostics; +#if DEBUG using System.Runtime.CompilerServices; +#endif using System.Runtime.InteropServices; using Garnet.common; using Tsavorite.core; @@ -23,7 +25,7 @@ namespace Garnet.server #if NET9_0_OR_GREATER ref #endif - struct MGetReadArgBatch(ref TGarnetApi storageApi, RespServerSession session) : IReadArgBatch + struct MGetReadArgBatch(ref TGarnetApi storageApi, RespServerSession session) : IReadArgBatch where TGarnetApi : IGarnetAdvancedApi { private Status currentStatus; @@ -42,24 +44,27 @@ private readonly public readonly int Count => session.parseState.Count; + public readonly ReadOnlySpan Parameters + => session.parseState.Parameters; + /// - public readonly void GetInput(int i, out RawStringInput input) + public readonly void GetInput(int i, out StringInput input) => input = new(RespCommand.GET, arg1: -1); /// - public readonly void GetKey(int i, out SpanByte key) - => key = session.parseState.GetArgSliceByRef(i).SpanByte; + public readonly void GetKey(int i, out FixedSpanByteKey key) + => key = (FixedSpanByteKey)session.parseState.GetArgSliceByRef(i); /// - public readonly unsafe void GetOutput(int i, out SpanByteAndMemory output) - => output = SpanByteAndMemory.FromPinnedSpan(MemoryMarshal.CreateSpan(ref Unsafe.AsRef(session.dcurr), (int)(session.dend - session.dcurr))); + public readonly unsafe void GetOutput(int i, out StringOutput output) + => output = StringOutput.FromPinnedPointer(session.dcurr, (int)(session.dend - session.dcurr)); /// public void SetStatus(int i, Status status) => currentStatus = status; /// - public readonly unsafe void SetOutput(int i, SpanByteAndMemory output) + public readonly unsafe void SetOutput(int i, StringOutput output) { var finalStatus = currentStatus; if (finalStatus.IsPending) @@ -89,15 +94,15 @@ public readonly unsafe void SetOutput(int i, SpanByteAndMemory output) // Got a result, write it out - if (output.IsSpanByte) + if (output.SpanByteAndMemory.IsSpanByte) { // Place result directly into buffer, just advance session points - session.dcurr += output.Length; + session.dcurr += output.SpanByteAndMemory.Length; } else { // Didn't fit inline, copy result over - session.SendAndReset(output.Memory, output.Length); + session.SendAndReset(output.SpanByteAndMemory.Memory, output.SpanByteAndMemory.Length); } } else @@ -117,40 +122,43 @@ public readonly unsafe void SetOutput(int i, SpanByteAndMemory output) /// For commands that are served entirely out of memory, writes results directly into the output buffer if possible. /// If operation would complete asynchronously, moves onto the next one and buffers results for later writing. /// - internal struct MGetReadArgBatch_SG(RespServerSession session) : IReadArgBatch + internal struct MGetReadArgBatch_SG(RespServerSession session) : IReadArgBatch { private bool pendingNullWrite; - private Memory<(Status Status, SpanByteAndMemory Output)> runningStatus; + private Memory<(Status Status, StringOutput Output)> runningStatus; /// public readonly int Count => session.parseState.Count; + public readonly ReadOnlySpan Parameters + => session.parseState.Parameters; + private readonly bool HasGoneAsync => !runningStatus.IsEmpty; /// - public readonly void GetInput(int i, out RawStringInput input) + public readonly void GetInput(int i, out StringInput input) { + input = default; + // Save the index so we can order async completions correctly in the response - // - // Use a - so we get "include RESP protocol"-behavior - input = new(RespCommand.GET, arg1: -(i + 1)); + input.arg1 = i; } /// - public readonly void GetKey(int i, out SpanByte key) - => key = session.parseState.GetArgSliceByRef(i).SpanByte; + public readonly void GetKey(int i, out FixedSpanByteKey key) + => key = (FixedSpanByteKey)session.parseState.GetArgSliceByRef(i); /// - public readonly void GetOutput(int i, out SpanByteAndMemory output) + public readonly void GetOutput(int i, out StringOutput output) { if (!HasGoneAsync) { // Attempt to write directly into output buffer unsafe { - output = SpanByteAndMemory.FromPinnedSpan(MemoryMarshal.CreateSpan(ref Unsafe.AsRef(session.dcurr), (int)(session.dend - session.dcurr))); + output = StringOutput.FromPinnedPointer(session.dcurr, (int)(session.dend - session.dcurr)); } } else @@ -161,7 +169,7 @@ public readonly void GetOutput(int i, out SpanByteAndMemory output) } /// - public readonly unsafe void SetOutput(int i, SpanByteAndMemory output) + public readonly unsafe void SetOutput(int i, StringOutput output) { if (!HasGoneAsync) { @@ -172,15 +180,15 @@ public readonly unsafe void SetOutput(int i, SpanByteAndMemory output) } else { - if (output.IsSpanByte) + if (output.SpanByteAndMemory.IsSpanByte) { // We place directly into the output buffer, nothing else needed - session.dcurr += output.Length; + session.dcurr += output.SpanByteAndMemory.Length; } else { // Got it synchronously, but it was too big for the buffer - session.SendAndReset(output.Memory, output.Length); + session.SendAndReset(output.SpanByteAndMemory.Memory, output.SpanByteAndMemory.Length); } } } @@ -204,7 +212,7 @@ public void SetStatus(int i, Status status) { var bufferSize = session.parseState.Count - i; - var arr = ArrayPool<(Status, SpanByteAndMemory)>.Shared.Rent(bufferSize); + var arr = ArrayPool<(Status, StringOutput)>.Shared.Rent(bufferSize); runningStatus = arr.AsMemory()[..bufferSize]; #if DEBUG @@ -277,7 +285,7 @@ public readonly unsafe void CompletePending(ref TGarnetApi storageAp while (iter.Next()) { - var rawIndex = -(int)iter.Current.Input.arg1 - 1; + var rawIndex = (int)iter.Current.Input.arg1; var shiftedIndex = rawIndex - asyncOffset; var asyncStatus = iter.Current.Status; @@ -310,15 +318,15 @@ public readonly unsafe void CompletePending(ref TGarnetApi storageAp { // Found it, either synchronously or async - if (output.IsSpanByte) + if (output.SpanByteAndMemory.IsSpanByte) { // We place directly into the output buffer, nothing else needed - session.dcurr += output.Length; + session.dcurr += output.SpanByteAndMemory.Length; } else { // Got it synchronously, but it was too big for the buffer - session.SendAndReset(output.Memory, output.Length); + session.SendAndReset(output.SpanByteAndMemory.Memory, output.SpanByteAndMemory.Length); } } else @@ -332,9 +340,9 @@ public readonly unsafe void CompletePending(ref TGarnetApi storageAp } finally { - if (MemoryMarshal.TryGetArray<(Status, SpanByteAndMemory)>(runningStatus, out var arrSeg)) + if (MemoryMarshal.TryGetArray<(Status, StringOutput)>(runningStatus, out var arrSeg)) { - ArrayPool<(Status, SpanByteAndMemory)>.Shared.Return(arrSeg.Array); + ArrayPool<(Status, StringOutput)>.Shared.Return(arrSeg.Array); } } } diff --git a/libs/server/Resp/Objects/HashCommands.cs b/libs/server/Resp/Objects/HashCommands.cs index bfcc4986cf5..0e9fd5fa1c9 100644 --- a/libs/server/Resp/Objects/HashCommands.cs +++ b/libs/server/Resp/Objects/HashCommands.cs @@ -31,8 +31,7 @@ private unsafe bool HashSet(RespCommand command, ref TGarnetApi stor return AbortWithWrongNumberOfArguments(command.ToString()); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var hop = command switch @@ -47,8 +46,9 @@ private unsafe bool HashSet(RespCommand command, ref TGarnetApi stor var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = hop }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.HashSet(keyBytes, ref input, out var output); + var status = storageApi.HashSet(key, ref input, ref output); switch (status) { @@ -86,17 +86,16 @@ private bool HashGet(RespCommand command, ref TGarnetApi storageApi) if (parseState.Count != 2) return AbortWithWrongNumberOfArguments(command.ToString()); - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HGET }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.HashGet(keyBytes, ref input, ref output); + var status = storageApi.HashGet(key, ref input, ref output); switch (status) { @@ -129,17 +128,16 @@ private bool HashGetAll(RespCommand command, ref TGarnetApi storageA return AbortWithWrongNumberOfArguments(command.ToString()); // Get the hash key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HGETALL }; var input = new ObjectInput(header, respProtocolVersion); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.HashGetAll(keyBytes, ref input, ref output); + var status = storageApi.HashGetAll(key, ref input, ref output); switch (status) { @@ -172,18 +170,17 @@ private bool HashGetMultiple(RespCommand command, ref TGarnetApi sto if (parseState.Count < 2) return AbortWithWrongNumberOfArguments(command.ToString()); - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HMGET }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.HashGetMultiple(keyBytes, ref input, ref output); + var status = storageApi.HashGetMultiple(key, ref input, ref output); switch (status) { @@ -195,7 +192,7 @@ private bool HashGetMultiple(RespCommand command, ref TGarnetApi sto while (!RespWriteUtils.TryWriteArrayLength(parseState.Count - 1, ref dcurr, dend)) SendAndReset(); - for (var i = 0; i < parseState.Count - 1; ++i) + for (var i = 0; i < parseState.Count - 1; i++) WriteNull(); break; case GarnetStatus.WRONGTYPE: @@ -220,8 +217,7 @@ private bool HashRandomField(RespCommand command, ref TGarnetApi sto if (parseState.Count < 1 || parseState.Count > 3) return AbortWithWrongNumberOfArguments(command.ToString()); - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var paramCount = 1; var withValues = false; @@ -259,17 +255,17 @@ private bool HashRandomField(RespCommand command, ref TGarnetApi sto var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HRANDFIELD }; var input = new ObjectInput(header, countWithMetadata, seed); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); var status = GarnetStatus.NOTFOUND; // This prevents going to the backend if HRANDFIELD is called with a count of 0 if (paramCount != 0) { - // Prepare GarnetObjectStore output - output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); - status = storageApi.HashRandomField(keyBytes, ref input, ref output); + // Prepare output + output = GetObjectOutput(); + status = storageApi.HashRandomField(key, ref input, ref output); } switch (status) @@ -311,15 +307,15 @@ private unsafe bool HashLength(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("HLEN"); } - // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + // Get the key + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HLEN }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = storageApi.HashLength(keyBytes, ref input, out var output); + var status = storageApi.HashLength(key, ref input, ref output); switch (status) { @@ -355,14 +351,14 @@ private unsafe bool HashStrLength(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("HSTRLEN"); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HSTRLEN }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.HashStrLength(keyBytes, ref input, out var output); + var status = storageApi.HashStrLength(key, ref input, ref output); switch (status) { @@ -398,14 +394,14 @@ private unsafe bool HashDelete(ref TGarnetApi storageApi) } // Get the key for Hash - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HDEL }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.HashDelete(keyBytes, ref input, out var output); + var status = storageApi.HashDelete(key, ref input, ref output); switch (status) { @@ -439,14 +435,14 @@ private unsafe bool HashExists(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("HEXISTS"); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HEXISTS }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.HashExists(keyBytes, ref input, out var output); + var status = storageApi.HashExists(key, ref input, ref output); switch (status) { @@ -483,8 +479,7 @@ private unsafe bool HashKeys(RespCommand command, ref TGarnetApi sto } // Get the key for Hash - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var op = command switch @@ -498,12 +493,12 @@ private unsafe bool HashKeys(RespCommand command, ref TGarnetApi sto var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = op }; var input = new ObjectInput(header); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); var status = command == RespCommand.HKEYS - ? storageApi.HashKeys(keyBytes, ref input, ref output) - : storageApi.HashVals(keyBytes, ref input, ref output); + ? storageApi.HashKeys(key, ref input, ref output) + : storageApi.HashVals(key, ref input, ref output); switch (status) { @@ -541,8 +536,7 @@ private unsafe bool HashIncrement(RespCommand command, ref TGarnetAp } // Get the key for Hash - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var op = command switch @@ -556,10 +550,10 @@ private unsafe bool HashIncrement(RespCommand command, ref TGarnetAp var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = op }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.HashIncrement(keyBytes, ref input, ref output); + var status = storageApi.HashIncrement(key, ref input, ref output); switch (status) { @@ -584,9 +578,6 @@ private unsafe bool HashIncrement(RespCommand command, ref TGarnetAp private unsafe bool HashExpire(RespCommand command, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 4) { return AbortWithWrongNumberOfArguments(command.ToString()); @@ -642,7 +633,7 @@ private unsafe bool HashExpire(RespCommand command, ref TGarnetApi s var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HEXPIRE }; var input = new ObjectInput(header, ref parseState, startIdx: currIdx, arg1: expirationWithOption.WordHead, arg2: expirationWithOption.WordTail); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.HashExpire(key, ref input, ref output); @@ -680,9 +671,6 @@ private unsafe bool HashExpire(RespCommand command, ref TGarnetApi s private unsafe bool HashTimeToLive(RespCommand command, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 3) { return AbortWithWrongNumberOfArguments(command.ToString()); @@ -732,7 +720,7 @@ private unsafe bool HashTimeToLive(RespCommand command, ref TGarnetA var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HTTL }; var input = new ObjectInput(header, ref fieldsParseState); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.HashTimeToLive(key, isMilliseconds, isTimestamp, ref input, ref output); @@ -762,9 +750,6 @@ private unsafe bool HashTimeToLive(RespCommand command, ref TGarnetA private unsafe bool HashPersist(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 3) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.HPERSIST)); @@ -794,7 +779,7 @@ private unsafe bool HashPersist(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HPERSIST }; var input = new ObjectInput(header, ref fieldsParseState); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.HashPersist(key, ref input, ref output); diff --git a/libs/server/Resp/Objects/ListCommands.cs b/libs/server/Resp/Objects/ListCommands.cs index 2fac6516ec2..bb37f53fa22 100644 --- a/libs/server/Resp/Objects/ListCommands.cs +++ b/libs/server/Resp/Objects/ListCommands.cs @@ -4,6 +4,7 @@ using System; using System.Text; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -21,12 +22,9 @@ private unsafe bool ListPush(RespCommand command, ref TGarnetApi sto where TGarnetApi : IGarnetApi { if (parseState.Count < 2) - { return AbortWithWrongNumberOfArguments(command.ToString()); - } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var lop = command switch @@ -41,10 +39,11 @@ private unsafe bool ListPush(RespCommand command, ref TGarnetApi sto // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); var status = command == RespCommand.LPUSH || command == RespCommand.LPUSHX - ? storageApi.ListLeftPush(keyBytes, ref input, out var output) - : storageApi.ListRightPush(keyBytes, ref input, out output); + ? storageApi.ListLeftPush(key, ref input, ref output) + : storageApi.ListRightPush(key, ref input, ref output); if (status == GarnetStatus.WRONGTYPE) { @@ -71,13 +70,10 @@ private unsafe bool ListPop(RespCommand command, ref TGarnetApi stor where TGarnetApi : IGarnetApi { if (parseState.Count < 1) - { return AbortWithWrongNumberOfArguments(command.ToString()); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var popCount = 1; @@ -85,9 +81,7 @@ private unsafe bool ListPop(RespCommand command, ref TGarnetApi stor { // Read count if (!parseState.TryGetInt(1, out popCount) || (popCount < 0)) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_OUT_OF_RANGE); - } } var lop = @@ -102,12 +96,12 @@ private unsafe bool ListPop(RespCommand command, ref TGarnetApi stor var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; var input = new ObjectInput(header, popCount); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); var statusOp = command == RespCommand.LPOP - ? storageApi.ListLeftPop(keyBytes, ref input, ref output) - : storageApi.ListRightPop(keyBytes, ref input, ref output); + ? storageApi.ListLeftPop(key, ref input, ref output) + : storageApi.ListRightPop(key, ref input, ref output); switch (statusOp) { @@ -138,22 +132,19 @@ private unsafe bool ListPosition(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count < 2) - { return AbortWithWrongNumberOfArguments(nameof(RespCommand.LPOS)); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LPOS }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var statusOp = storageApi.ListPosition(keyBytes, ref input, ref output); + var statusOp = storageApi.ListPosition(key, ref input, ref output); switch (statusOp) { @@ -161,8 +152,8 @@ private unsafe bool ListPosition(ref TGarnetApi storageApi) ProcessOutput(output.SpanByteAndMemory); break; case GarnetStatus.NOTFOUND: - bool count = false; - for (var i = 2; i < parseState.Count; ++i) + var count = false; + for (var i = 2; i < parseState.Count; i++) { if (parseState.GetArgSliceByRef(i).Span.EqualsUpperCaseSpanIgnoringCase(CmdStrings.COUNT)) { @@ -199,9 +190,7 @@ private unsafe bool ListPopMultiple(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count < 3) - { return AbortWithWrongNumberOfArguments("LMPOP"); - } var currTokenId = 0; @@ -213,23 +202,17 @@ private unsafe bool ListPopMultiple(ref TGarnetApi storageApi) } if (parseState.Count != numKeys + 2 && parseState.Count != numKeys + 4) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } // Get the keys for Lists - var keys = new ArgSlice[numKeys]; + var keys = new PinnedSpanByte[numKeys]; for (var i = 0; i < keys.Length; i++) - { keys[i] = parseState.GetArgSliceByRef(currTokenId++); - } // Get the direction if (!parseState.TryGetOperationDirection(currTokenId++, out var popDirection)) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } var popCount = 1; @@ -239,9 +222,7 @@ private unsafe bool ListPopMultiple(ref TGarnetApi storageApi) var countKeyword = parseState.GetArgSliceByRef(currTokenId++); if (!countKeyword.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.COUNT)) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } // Read count if (!parseState.TryGetInt(currTokenId, out popCount)) @@ -289,24 +270,15 @@ private unsafe bool ListPopMultiple(ref TGarnetApi storageApi) private bool ListBlockingPop(RespCommand command) { if (parseState.Count < 2) - { return AbortWithWrongNumberOfArguments(command.ToString()); - } var keysBytes = new byte[parseState.Count - 1][]; for (var i = 0; i < keysBytes.Length; i++) - { - keysBytes[i] = parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - } + keysBytes[i] = parseState.GetArgSliceByRef(i).ToArray(); if (!parseState.TryGetTimeout(parseState.Count - 1, out var timeout, out var error)) - { return AbortWithErrorMessage(error); - } - - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); // Must block as we're on the network thread var result = AsyncUtils.BlockingWait(storeWrapper.itemBroker.GetCollectionItemAsync(command, keysBytes, this, timeout)); @@ -326,9 +298,7 @@ private bool ListBlockingPop(RespCommand command) } if (!result.Found) - { WriteNullArray(); - } else { while (!RespWriteUtils.TryWriteArrayLength(2, ref dcurr, dend)) @@ -347,23 +317,17 @@ private bool ListBlockingPop(RespCommand command) private unsafe bool ListBlockingMove() { if (parseState.Count != 5) - { return AbortWithWrongNumberOfArguments(nameof(RespCommand.BLMOVE)); - } var srcKey = parseState.GetArgSliceByRef(0); var dstKey = parseState.GetArgSliceByRef(1); if (!parseState.TryGetOperationDirection(2, out var srcDir) || - !parseState.TryGetOperationDirection(3, out var dstDir)) - { + !parseState.TryGetOperationDirection(3, out var dstDir)) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } if (!parseState.TryGetTimeout(4, out var timeout, out var error)) - { return AbortWithErrorMessage(error); - } return ListBlockingMove(srcKey, dstKey, srcDir, dstDir, timeout); } @@ -375,44 +339,35 @@ private unsafe bool ListBlockingMove() private bool ListBlockingPopPush() { if (parseState.Count != 3) - { return AbortWithWrongNumberOfArguments(nameof(RespCommand.BRPOPLPUSH)); - } var srcKey = parseState.GetArgSliceByRef(0); var dstKey = parseState.GetArgSliceByRef(1); if (!parseState.TryGetTimeout(2, out var timeout, out var error)) - { return AbortWithErrorMessage(error); - } return ListBlockingMove(srcKey, dstKey, OperationDirection.Right, OperationDirection.Left, timeout); } - private bool ListBlockingMove(ArgSlice srcKey, ArgSlice dstKey, + private bool ListBlockingMove(PinnedSpanByte srcKey, PinnedSpanByte dstKey, OperationDirection sourceDirection, OperationDirection destinationDirection, double timeout) { - var cmdArgs = new ArgSlice[] { default, default, default }; + var cmdArgs = new PinnedSpanByte[] { default, default, default }; // Read destination key cmdArgs[0] = dstKey; if (sourceDirection == OperationDirection.Unknown || destinationDirection == OperationDirection.Unknown) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } var pSrcDir = (byte*)&sourceDirection; var pDstDir = (byte*)&destinationDirection; - cmdArgs[1] = new ArgSlice(pSrcDir, 1); - cmdArgs[2] = new ArgSlice(pDstDir, 1); - - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); + cmdArgs[1] = PinnedSpanByte.FromPinnedPointer(pSrcDir, 1); + cmdArgs[2] = PinnedSpanByte.FromPinnedPointer(pDstDir, 1); // On the networking thread, no choice but to block var result = @@ -458,18 +413,16 @@ private bool ListLength(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 1) - { return AbortWithWrongNumberOfArguments("LLEN"); - } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LLEN }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = storageApi.ListLength(keyBytes, ref input, out var output); + var status = storageApi.ListLength(key, ref input, ref output); switch (status) { @@ -502,13 +455,10 @@ private bool ListTrim(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 3) - { return AbortWithWrongNumberOfArguments("LTRIM"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Read the parameters(start and stop) from LTRIM if (!parseState.TryGetInt(1, out var start) || @@ -523,7 +473,7 @@ private bool ListTrim(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LTRIM }; var input = new ObjectInput(header, start, stop); - var status = storageApi.ListTrim(keyBytes, ref input); + var status = storageApi.ListTrim(key, ref input); switch (status) { @@ -553,13 +503,10 @@ private bool ListRange(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 3) - { return AbortWithWrongNumberOfArguments("LRANGE"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Read count start and end params for LRANGE if (!parseState.TryGetInt(1, out var start) || @@ -574,10 +521,10 @@ private bool ListRange(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LRANGE }; var input = new ObjectInput(header, start, end); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var statusOp = storageApi.ListRange(keyBytes, ref input, ref output); + var statusOp = storageApi.ListRange(key, ref input, ref output); switch (statusOp) { @@ -608,13 +555,10 @@ private bool ListIndex(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 2) - { return AbortWithWrongNumberOfArguments("LINDEX"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Read index param if (!parseState.TryGetInt(1, out var index)) @@ -628,17 +572,17 @@ private bool ListIndex(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LINDEX }; var input = new ObjectInput(header, index); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var statusOp = storageApi.ListIndex(keyBytes, ref input, ref output); + var statusOp = storageApi.ListIndex(key, ref input, ref output); switch (statusOp) { case GarnetStatus.OK: //process output ProcessOutput(output.SpanByteAndMemory); - if (output.Header.result1 == -1) + if (output.result1 == -1) WriteNull(); break; case GarnetStatus.NOTFOUND: @@ -664,19 +608,17 @@ private bool ListInsert(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 4) - { return AbortWithWrongNumberOfArguments("LINSERT"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LINSERT }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var statusOp = storageApi.ListInsert(keyBytes, ref input, out var output); + var statusOp = storageApi.ListInsert(key, ref input, ref output); switch (statusOp) { @@ -712,13 +654,10 @@ private bool ListRemove(ref TGarnetApi storageApi) { // if params are missing return error if (parseState.Count != 3) - { return AbortWithWrongNumberOfArguments("LREM"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Get count parameter if (!parseState.TryGetInt(1, out var nCount)) @@ -731,8 +670,9 @@ private bool ListRemove(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LREM }; var input = new ObjectInput(header, ref parseState, startIdx: 2, arg1: nCount); + var output = new ObjectOutput(); - var statusOp = storageApi.ListRemove(keyBytes, ref input, out var output); + var statusOp = storageApi.ListRemove(key, ref input, ref output); switch (statusOp) { @@ -767,21 +707,16 @@ private bool ListMove(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 4) - { return AbortWithWrongNumberOfArguments("LMOVE"); - } var srcKey = parseState.GetArgSliceByRef(0); var dstKey = parseState.GetArgSliceByRef(1); if (!parseState.TryGetOperationDirection(2, out var sourceDirection) || - !parseState.TryGetOperationDirection(3, out var destinationDirection)) - { + !parseState.TryGetOperationDirection(3, out var destinationDirection)) return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } - if (!ListMove(srcKey, dstKey, sourceDirection, destinationDirection, out var node, - ref storageApi, out var garnetStatus)) + if (!ListMove(srcKey, dstKey, sourceDirection, destinationDirection, out var node, ref storageApi, out var garnetStatus)) return false; switch (garnetStatus) @@ -816,15 +751,12 @@ private bool ListRightPopLeftPush(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 2) - { return AbortWithWrongNumberOfArguments("RPOPLPUSH"); - } var srcKey = parseState.GetArgSliceByRef(0); var dstKey = parseState.GetArgSliceByRef(1); - if (!ListMove(srcKey, dstKey, OperationDirection.Right, OperationDirection.Left, - out var node, ref storageApi, out var garnetStatus)) + if (!ListMove(srcKey, dstKey, OperationDirection.Right, OperationDirection.Left, out var node, ref storageApi, out var garnetStatus)) return false; switch (garnetStatus) @@ -862,16 +794,12 @@ private bool ListRightPopLeftPush(ref TGarnetApi storageApi) /// /// /// - private bool ListMove(ArgSlice sourceKey, ArgSlice destinationKey, + private static bool ListMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] node, ref TGarnetApi storageApi, out GarnetStatus garnetStatus) where TGarnetApi : IGarnetApi { - garnetStatus = GarnetStatus.OK; - node = null; - - garnetStatus = - storageApi.ListMove(sourceKey, destinationKey, sourceDirection, destinationDirection, out node); + garnetStatus = storageApi.ListMove(sourceKey, destinationKey, sourceDirection, destinationDirection, out node); return true; } @@ -886,22 +814,19 @@ public bool ListSet(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count != 3) - { return AbortWithWrongNumberOfArguments("LSET"); - } // Get the key for List - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LSET }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var statusOp = storageApi.ListSet(keyBytes, ref input, ref output); + var statusOp = storageApi.ListSet(key, ref input, ref output); switch (statusOp) { @@ -929,17 +854,13 @@ public bool ListSet(ref TGarnetApi storageApi) private unsafe bool ListBlockingPopMultiple() { if (parseState.Count < 4) - { return AbortWithWrongNumberOfArguments(nameof(RespCommand.BLMPOP)); - } var currTokenId = 0; // Read timeout if (!parseState.TryGetTimeout(currTokenId++, out var timeout, out var error)) - { return AbortWithErrorMessage(error); - } // Read count of keys if (!parseState.TryGetInt(currTokenId++, out var numKeys)) @@ -956,18 +877,15 @@ private unsafe bool ListBlockingPopMultiple() // Get the keys for Lists var keysBytes = new byte[numKeys][]; for (var i = 0; i < keysBytes.Length; i++) - { - keysBytes[i] = parseState.GetArgSliceByRef(currTokenId++).SpanByte.ToByteArray(); - } + keysBytes[i] = parseState.GetArgSliceByRef(currTokenId++).ToArray(); - var cmdArgs = new ArgSlice[2]; + var cmdArgs = new PinnedSpanByte[2]; // Get the direction if (!parseState.TryGetOperationDirection(currTokenId++, out var popDirection)) - { return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); - } - cmdArgs[0] = new ArgSlice((byte*)&popDirection, 1); + + cmdArgs[0] = PinnedSpanByte.FromPinnedPointer((byte*)&popDirection, 1); var popCount = 1; @@ -989,10 +907,7 @@ private unsafe bool ListBlockingPopMultiple() } } - cmdArgs[1] = new ArgSlice((byte*)&popCount, sizeof(int)); - - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); + cmdArgs[1] = PinnedSpanByte.FromPinnedPointer((byte*)&popCount, sizeof(int)); // Must block, we're on the networking thread var result = AsyncUtils.BlockingWait(storeWrapper.itemBroker.GetCollectionItemAsync(RespCommand.BLMPOP, keysBytes, this, timeout, cmdArgs)); diff --git a/libs/server/Resp/Objects/SetCommands.cs b/libs/server/Resp/Objects/SetCommands.cs index 7d7438ec465..8a363d1975f 100644 --- a/libs/server/Resp/Objects/SetCommands.cs +++ b/libs/server/Resp/Objects/SetCommands.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Text; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -30,14 +31,14 @@ private unsafe bool SetAdd(ref TGarnetApi storageApi) } // Get the key for the Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SADD }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.SetAdd(keyBytes, ref input, out var output); + var status = storageApi.SetAdd(key, ref input, ref output); switch (status) { @@ -71,7 +72,7 @@ private bool SetIntersect(ref TGarnetApi storageApi) } // Read all keys - var keys = new ArgSlice[parseState.Count]; + var keys = new PinnedSpanByte[parseState.Count]; for (var i = 0; i < keys.Length; i++) { keys[i] = parseState.GetArgSliceByRef(i); @@ -125,15 +126,15 @@ private bool SetIntersectStore(ref TGarnetApi storageApi) } // Get the key - var keyBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); - var keys = new ArgSlice[parseState.Count - 1]; + var keys = new PinnedSpanByte[parseState.Count - 1]; for (var i = 1; i < parseState.Count; i++) { keys[i - 1] = parseState.GetArgSliceByRef(i); } - var status = storageApi.SetIntersectStore(keyBytes, keys, out var output); + var status = storageApi.SetIntersectStore(key, keys, out var output); switch (status) { case GarnetStatus.OK: @@ -231,7 +232,7 @@ private bool SetUnion(ref TGarnetApi storageApi) } // Read all the keys - var keys = new ArgSlice[parseState.Count]; + var keys = new PinnedSpanByte[parseState.Count]; for (var i = 0; i < keys.Length; i++) { @@ -276,15 +277,15 @@ private bool SetUnionStore(ref TGarnetApi storageApi) } // Get the key - var keyBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); - var keys = new ArgSlice[parseState.Count - 1]; + var keys = new PinnedSpanByte[parseState.Count - 1]; for (var i = 1; i < parseState.Count; i++) { keys[i - 1] = parseState.GetArgSliceByRef(i); } - var status = storageApi.SetUnionStore(keyBytes, keys, out var output); + var status = storageApi.SetUnionStore(key, keys, out var output); switch (status) { case GarnetStatus.OK: @@ -316,15 +317,15 @@ private unsafe bool SetRemove(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("SREM"); } - // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + // Get the key + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SREM }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.SetRemove(keyBytes, ref input, out var output); + var status = storageApi.SetRemove(key, ref input, ref output); switch (status) { @@ -361,14 +362,14 @@ private unsafe bool SetLength(ref TGarnetApi storageApi) } // Get the key for the Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SCARD }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = storageApi.SetLength(keyBytes, ref input, out var output); + var status = storageApi.SetLength(key, ref input, ref output); switch (status) { @@ -405,17 +406,16 @@ private unsafe bool SetMembers(ref TGarnetApi storageApi) } // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SMEMBERS }; var input = new ObjectInput(header); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SetMembers(keyBytes, ref input, ref output); + var status = storageApi.SetMembers(key, ref input, ref output); switch (status) { @@ -457,17 +457,16 @@ private unsafe bool SetIsMember(RespCommand cmd, ref TGarnetApi stor } // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = isSingle ? SetOperation.SISMEMBER : SetOperation.SMISMEMBER }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SetIsMember(keyBytes, ref input, ref output); + var status = storageApi.SetIsMember(key, ref input, ref output); switch (status) { @@ -518,9 +517,7 @@ private unsafe bool SetPop(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("SPOP"); } - // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var countParameter = int.MinValue; if (parseState.Count == 2) @@ -544,10 +541,10 @@ private unsafe bool SetPop(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SPOP }; var input = new ObjectInput(header, countParameter); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SetPop(keyBytes, ref input, ref output); + var status = storageApi.SetPop(key, ref input, ref output); switch (status) { @@ -630,9 +627,7 @@ private unsafe bool SetRandomMember(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("SRANDMEMBER"); } - // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var countParameter = int.MinValue; if (parseState.Count == 2) @@ -659,10 +654,10 @@ private unsafe bool SetRandomMember(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SRANDMEMBER }; var input = new ObjectInput(header, countParameter, seed); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SetRandomMember(keyBytes, ref input, ref output); + var status = storageApi.SetRandomMember(key, ref input, ref output); switch (status) { @@ -703,7 +698,7 @@ private bool SetDiff(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("SDIFF"); } - var keys = new ArgSlice[parseState.Count]; + var keys = new PinnedSpanByte[parseState.Count]; for (var i = 0; i < parseState.Count; i++) { keys[i] = parseState.GetArgSliceByRef(i); @@ -746,15 +741,15 @@ private bool SetDiffStore(ref TGarnetApi storageApi) } // Get the key - var keyBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); - var keys = new ArgSlice[parseState.Count - 1]; + var keys = new PinnedSpanByte[parseState.Count - 1]; for (var i = 1; i < parseState.Count; i++) { keys[i - 1] = parseState.GetArgSliceByRef(i); } - var status = storageApi.SetDiffStore(keyBytes, keys, out var output); + var status = storageApi.SetDiffStore(key, keys, out var output); switch (status) { case GarnetStatus.OK: diff --git a/libs/server/Resp/Objects/SharedObjectCommands.cs b/libs/server/Resp/Objects/SharedObjectCommands.cs index 965f8d3645c..693e025391c 100644 --- a/libs/server/Resp/Objects/SharedObjectCommands.cs +++ b/libs/server/Resp/Objects/SharedObjectCommands.cs @@ -34,8 +34,7 @@ private unsafe bool ObjectScan(GarnetObjectType objectType, ref TGar } // Read key for the scan - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Get cursor value if (!parseState.TryGetLong(1, out var cursorValue) || cursorValue < 0) @@ -63,9 +62,9 @@ private unsafe bool ObjectScan(GarnetObjectType objectType, ref TGar break; } - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); - var status = storageApi.ObjectScan(keyBytes, ref input, ref output); + // Prepare output + var output = GetObjectOutput(); + var status = storageApi.ObjectScan(key, ref input, ref output); switch (status) { @@ -73,7 +72,7 @@ private unsafe bool ObjectScan(GarnetObjectType objectType, ref TGar // Process output ProcessOutput(output.SpanByteAndMemory); // Validation for partial input reading or error - if (output.Header.result1 == int.MinValue) + if (output.result1 == int.MinValue) return false; break; case GarnetStatus.NOTFOUND: diff --git a/libs/server/Resp/Objects/SortedSetCommands.cs b/libs/server/Resp/Objects/SortedSetCommands.cs index 37cc6715a99..a4cc6e78697 100644 --- a/libs/server/Resp/Objects/SortedSetCommands.cs +++ b/libs/server/Resp/Objects/SortedSetCommands.cs @@ -23,20 +23,17 @@ private unsafe bool SortedSetAdd(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { if (parseState.Count < 3) - { return AbortWithWrongNumberOfArguments("ZADD"); - } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZADD }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); - var status = storageApi.SortedSetAdd(keyBytes, ref input, ref output); + var status = storageApi.SortedSetAdd(key, ref input, ref output); switch (status) { @@ -68,18 +65,18 @@ private unsafe bool SortedSetRemove(ref TGarnetApi storageApi) } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREM }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); - var status = storageApi.SortedSetRemove(keyBytes, ref input, out var rmwOutput); + var status = storageApi.SortedSetRemove(key, ref input, ref output); switch (status) { case GarnetStatus.OK: - while (!RespWriteUtils.TryWriteInt32(rmwOutput.result1, ref dcurr, dend)) + while (!RespWriteUtils.TryWriteInt32(output.result1, ref dcurr, dend)) SendAndReset(); break; case GarnetStatus.NOTFOUND: @@ -109,13 +106,13 @@ private unsafe bool SortedSetLength(ref TGarnetApi storageApi) } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCARD }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = storageApi.SortedSetLength(keyBytes, ref input, out var output); + var status = storageApi.SortedSetLength(key, ref input, ref output); switch (status) { @@ -155,8 +152,7 @@ private unsafe bool SortedSetRange(RespCommand command, ref TGarnetA return AbortWithWrongNumberOfArguments(nameof(RespCommand.ZRANGE)); } - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var rangeOpts = SortedSetRangeOpts.None; @@ -187,9 +183,9 @@ private unsafe bool SortedSetRange(RespCommand command, ref TGarnetA var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZRANGE }; var input = new ObjectInput(header, ref parseState, startIdx: 1, arg1: respProtocolVersion, arg2: (int)rangeOpts); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); - var status = storageApi.SortedSetRange(keyBytes, ref input, ref output); + var status = storageApi.SortedSetRange(key, ref input, ref output); switch (status) { @@ -258,17 +254,16 @@ private unsafe bool SortedSetScore(ref TGarnetApi storageApi) } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZSCORE }; var input = new ObjectInput(header, ref parseState, startIdx: 1, arg1: respProtocolVersion); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SortedSetScore(keyBytes, ref input, ref output); + var status = storageApi.SortedSetScore(key, ref input, ref output); switch (status) { @@ -304,17 +299,16 @@ private unsafe bool SortedSetScores(ref TGarnetApi storageApi) } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZMSCORE }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SortedSetScores(keyBytes, ref input, ref output); + var status = storageApi.SortedSetScores(key, ref input, ref output); switch (status) { @@ -326,7 +320,7 @@ private unsafe bool SortedSetScores(ref TGarnetApi storageApi) while (!RespWriteUtils.TryWriteArrayLength(parseState.Count - 1, ref dcurr, dend)) SendAndReset(); - for (var i = 0; i < parseState.Count - 1; ++i) + for (var i = 0; i < parseState.Count - 1; i++) WriteNull(); break; case GarnetStatus.WRONGTYPE: @@ -355,8 +349,7 @@ private unsafe bool SortedSetPop(RespCommand command, ref TGarnetApi } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var popCount = -1; @@ -382,9 +375,9 @@ private unsafe bool SortedSetPop(RespCommand command, ref TGarnetApi var input = new ObjectInput(header, popCount); // Prepare output - var output = new GarnetObjectStoreOutput(new SpanByteAndMemory(SpanByte.FromPinnedPointer(dcurr, (int)(dend - dcurr)))); + var output = GetObjectOutput(); - var status = storageApi.SortedSetPop(keyBytes, ref input, ref output); + var status = storageApi.SortedSetPop(key, ref input, ref output); switch (status) { @@ -542,17 +535,16 @@ private unsafe bool SortedSetCount(ref TGarnetApi storageApi) } // Get the key for the Sorted Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOUNT }; var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare output - var output = new GarnetObjectStoreOutput(new SpanByteAndMemory(SpanByte.FromPinnedPointer(dcurr, (int)(dend - dcurr)))); + var output = GetObjectOutput(); - var status = storageApi.SortedSetCount(keyBytes, ref input, ref output); + var status = storageApi.SortedSetCount(key, ref input, ref output); switch (status) { @@ -592,8 +584,7 @@ private unsafe bool SortedSetLengthByValue(RespCommand command, ref } // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var op = command switch @@ -606,10 +597,11 @@ private unsafe bool SortedSetLengthByValue(RespCommand command, ref // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; var input = new ObjectInput(header, ref parseState, startIdx: 1); + var output = new ObjectOutput(); var status = op == SortedSetOperation.ZREMRANGEBYLEX ? - storageApi.SortedSetRemoveRangeByLex(keyBytes, ref input, out var output) : - storageApi.SortedSetLengthByValue(keyBytes, ref input, out output); + storageApi.SortedSetRemoveRangeByLex(key, ref input, ref output) : + storageApi.SortedSetLengthByValue(key, ref input, ref output); switch (status) { @@ -657,17 +649,16 @@ private unsafe bool SortedSetIncrement(ref TGarnetApi storageApi) } // Get the key for the Sorted Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZINCRBY }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SortedSetIncrement(keyBytes, ref input, ref output); + var status = storageApi.SortedSetIncrement(key, ref input, ref output); switch (status) { @@ -701,8 +692,7 @@ private unsafe bool SortedSetRank(RespCommand command, ref TGarnetAp } // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var includeWithScore = false; // Read WITHSCORE @@ -730,10 +720,10 @@ private unsafe bool SortedSetRank(RespCommand command, ref TGarnetAp var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; var input = new ObjectInput(header, ref parseState, startIdx: 1, arg1: includeWithScore ? 1 : 0); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SortedSetRank(keyBytes, ref input, ref output); + var status = storageApi.SortedSetRank(key, ref input, ref output); switch (status) { @@ -770,8 +760,7 @@ private unsafe bool SortedSetRemoveRange(RespCommand command, ref TG } // Get the key - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var op = command switch @@ -785,10 +774,10 @@ private unsafe bool SortedSetRemoveRange(RespCommand command, ref TG var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; var input = new ObjectInput(header, ref parseState, startIdx: 1); - // Prepare GarnetObjectStore output - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + // Prepare output + var output = GetObjectOutput(); - var status = storageApi.SortedSetRemoveRange(keyBytes, ref input, ref output); + var status = storageApi.SortedSetRemoveRange(key, ref input, ref output); switch (status) { @@ -822,8 +811,7 @@ private unsafe bool SortedSetRandomMember(ref TGarnetApi storageApi) } // Get the key for the Sorted Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); var paramCount = 1; var includeWithScores = false; var includedCount = false; @@ -864,14 +852,14 @@ private unsafe bool SortedSetRandomMember(ref TGarnetApi storageApi) var input = new ObjectInput(header, inputArg, seed); var status = GarnetStatus.NOTFOUND; - GarnetObjectStoreOutput output = default; + ObjectOutput output = default; // This prevents going to the backend if ZRANDMEMBER is called with a count of 0 if (paramCount != 0) { - // Prepare GarnetObjectStore output - output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); - status = storageApi.SortedSetRandomMember(keyBytes, ref input, ref output); + // Prepare output + output = GetObjectOutput(); + status = storageApi.SortedSetRandomMember(key, ref input, ref output); } switch (status) @@ -926,7 +914,7 @@ private unsafe bool SortedSetDifference(ref TGarnetApi storageApi) } var includeWithScores = false; - var keys = new ArgSlice[nKeys]; + var keys = new PinnedSpanByte[nKeys]; for (var i = 1; i < nKeys + 1; i++) { @@ -1554,9 +1542,6 @@ private unsafe bool SortedSetUnionStore(ref TGarnetApi storageApi) /// private unsafe bool SortedSetBlockingPop(RespCommand command) { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count < 2) { return AbortWithWrongNumberOfArguments(command.ToString()); @@ -1570,9 +1555,7 @@ private unsafe bool SortedSetBlockingPop(RespCommand command) var keysBytes = new byte[parseState.Count - 1][]; for (var i = 0; i < keysBytes.Length; i++) - { - keysBytes[i] = parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); - } + keysBytes[i] = parseState.GetArgSliceByRef(i).ToArray(); // Must block, we're on the networking thread var result = AsyncUtils.BlockingWait(storeWrapper.itemBroker.GetCollectionItemAsync(command, keysBytes, this, timeout)); @@ -1617,9 +1600,6 @@ private unsafe bool SortedSetBlockingPop(RespCommand command) /// private unsafe bool SortedSetBlockingMPop() { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count < 4) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.BZMPOP)); @@ -1647,11 +1627,9 @@ private unsafe bool SortedSetBlockingMPop() var keysBytes = new byte[numKeys][]; for (var i = 0; i < keysBytes.Length; i++) - { - keysBytes[i] = parseState.GetArgSliceByRef(currTokenId++).SpanByte.ToByteArray(); - } + keysBytes[i] = parseState.GetArgSliceByRef(currTokenId++).ToArray(); - var cmdArgs = new ArgSlice[2]; + var cmdArgs = new PinnedSpanByte[2]; var orderArg = parseState.GetArgSliceByRef(currTokenId++); var orderSpan = orderArg.ReadOnlySpan; @@ -1666,7 +1644,7 @@ private unsafe bool SortedSetBlockingMPop() return AbortWithErrorMessage(CmdStrings.RESP_ERR_GENERIC_SYNTAX_ERROR); } - cmdArgs[0] = new ArgSlice((byte*)&lowScoresFirst, 1); + cmdArgs[0] = PinnedSpanByte.FromPinnedPointer((byte*)&lowScoresFirst, 1); var popCount = 1; @@ -1685,7 +1663,7 @@ private unsafe bool SortedSetBlockingMPop() } } - cmdArgs[1] = new ArgSlice((byte*)&popCount, sizeof(int)); + cmdArgs[1] = PinnedSpanByte.FromPinnedPointer((byte*)&popCount, sizeof(int)); // We're on the networking thread, so must block var result = AsyncUtils.BlockingWait(storeWrapper.itemBroker.GetCollectionItemAsync(RespCommand.BZMPOP, keysBytes, this, timeout, cmdArgs)); @@ -1720,7 +1698,7 @@ private unsafe bool SortedSetBlockingMPop() while (!RespWriteUtils.TryWriteArrayLength(result.Items.Length, ref dcurr, dend)) SendAndReset(); - for (var i = 0; i < result.Items.Length; ++i) + for (var i = 0; i < result.Items.Length; i++) { while (!RespWriteUtils.TryWriteArrayLength(2, ref dcurr, dend)) SendAndReset(); @@ -1746,9 +1724,6 @@ private unsafe bool SortedSetBlockingMPop() private unsafe bool SortedSetExpire(RespCommand command, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 4) { return AbortWithWrongNumberOfArguments(command.ToString()); @@ -1803,7 +1778,7 @@ private unsafe bool SortedSetExpire(RespCommand command, ref TGarnet var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZEXPIRE }; var input = new ObjectInput(header, ref parseState, startIdx: currIdx, expirationWithOption.WordHead, expirationWithOption.WordTail); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.SortedSetExpire(key, ref input, ref output); @@ -1845,9 +1820,6 @@ private unsafe bool SortedSetExpire(RespCommand command, ref TGarnet private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 3) { return AbortWithWrongNumberOfArguments(command.ToString()); @@ -1896,7 +1868,7 @@ private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGa var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZTTL }; var input = new ObjectInput(header, ref membersParseState, arg1: isMilliseconds ? 1 : 0, arg2: isTimestamp ? 1 : 0); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.SortedSetTimeToLive(key, ref input, ref output); @@ -1934,9 +1906,6 @@ private unsafe bool SortedSetTimeToLive(RespCommand command, ref TGa private unsafe bool SortedSetPersist(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { - if (storeWrapper.objectStore == null) - throw new GarnetException("Object store is disabled"); - if (parseState.Count <= 3) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.ZPERSIST)); @@ -1965,7 +1934,7 @@ private unsafe bool SortedSetPersist(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZPERSIST }; var input = new ObjectInput(header, ref membersParseState); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); var status = storageApi.SortedSetPersist(key, ref input, ref output); diff --git a/libs/server/Resp/Objects/SortedSetGeoCommands.cs b/libs/server/Resp/Objects/SortedSetGeoCommands.cs index 3c641ff9ac5..2f33ead5bd5 100644 --- a/libs/server/Resp/Objects/SortedSetGeoCommands.cs +++ b/libs/server/Resp/Objects/SortedSetGeoCommands.cs @@ -29,7 +29,7 @@ private unsafe bool GeoAdd(ref TGarnetApi storageApi) var currTokenIdx = 0; // Get the key for SortedSet - var sbKey = parseState.GetArgSliceByRef(currTokenIdx++).SpanByte; + var key = parseState.GetArgSliceByRef(currTokenIdx++); while (currTokenIdx < parseState.Count) { @@ -85,9 +85,9 @@ private unsafe bool GeoAdd(ref TGarnetApi storageApi) var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.GEOADD }; var input = new ObjectInput(header, ref parseState, startIdx: memberStart, arg1: (int)addOption); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); - var status = storageApi.GeoAdd(sbKey.ToByteArray(), ref input, ref output); + var status = storageApi.GeoAdd(key, ref input, ref output); switch (status) { @@ -137,8 +137,7 @@ private unsafe bool GeoCommands(RespCommand command, ref TGarnetApi } // Get the key for the Sorted Set - var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var keyBytes = sbKey.ToByteArray(); + var key = parseState.GetArgSliceByRef(0); SortedSetOperation op; @@ -166,9 +165,9 @@ private unsafe bool GeoCommands(RespCommand command, ref TGarnetApi var input = new ObjectInput(header, ref parseState, startIdx: 1); - var output = new GarnetObjectStoreOutput(new(dcurr, (int)(dend - dcurr))); + var output = GetObjectOutput(); - var status = storageApi.GeoCommands(keyBytes, ref input, ref output); + var status = storageApi.GeoCommands(key, ref input, ref output); switch (status) { @@ -258,7 +257,7 @@ private unsafe bool GeoSearchCommands(RespCommand command, ref TGarn { SortedSetOp = SortedSetOperation.GEOSEARCH }, ref parseState, startIdx: sourceIdx + 1, arg1: (int)command); - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var output = SpanByteAndMemory.FromPinnedPointer(dcurr, (int)(dend - dcurr)); if (!input.parseState.TryGetGeoSearchOptions(command, out var searchOpts, out var destIdx, out var errorMessage)) { diff --git a/libs/server/Resp/Parser/ParseUtils.cs b/libs/server/Resp/Parser/ParseUtils.cs index 02e9a2c41ca..8dd2c2e81bf 100644 --- a/libs/server/Resp/Parser/ParseUtils.cs +++ b/libs/server/Resp/Parser/ParseUtils.cs @@ -6,6 +6,7 @@ using System.Text; using Garnet.common; using Garnet.common.Parsing; +using Tsavorite.core; namespace Garnet.server { @@ -21,7 +22,7 @@ public static unsafe class ParseUtils /// Parsed integer /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int ReadInt(ref ArgSlice slice) + public static int ReadInt(PinnedSpanByte slice) { int number = default; var ptr = slice.ptr; @@ -43,7 +44,7 @@ public static int ReadInt(ref ArgSlice slice) /// True if integer read successfully /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryReadInt(ref ArgSlice slice, out int number) + public static bool TryReadInt(PinnedSpanByte slice, out int number) { number = default; var ptr = slice.ptr; @@ -60,7 +61,7 @@ public static bool TryReadInt(ref ArgSlice slice, out int number) /// Parsed long /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static long ReadLong(ref ArgSlice slice) + public static long ReadLong(PinnedSpanByte slice) { long number = default; var ptr = slice.ptr; @@ -82,7 +83,7 @@ public static long ReadLong(ref ArgSlice slice) /// True if long parsed successfully /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryReadLong(ref ArgSlice slice, out long number) + public static bool TryReadLong(PinnedSpanByte slice, out long number) { number = default; var ptr = slice.ptr; @@ -92,6 +93,23 @@ public static bool TryReadLong(ref ArgSlice slice, out long number) (int)bytesRead == slice.length; } + /// + /// Try to read a signed 64-bit long from a given ArgSlice. + /// + /// + /// True if long parsed successfully + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryReadLong(PinnedSpanByte slice, bool allowLeadingZeros, out long number) + { + number = default; + var ptr = slice.ptr; + return slice.length != 0 && + RespReadUtils.TryReadInt64Safe(ref ptr, slice.ptr + slice.length, out number, out var bytesRead, + out _, out _, allowLeadingZeros: allowLeadingZeros) && + (int)bytesRead == slice.length; + } + /// /// Read a signed 64-bit double from a given ArgSlice. /// @@ -101,9 +119,9 @@ public static bool TryReadLong(ref ArgSlice slice, out long number) /// Parsed double /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static double ReadDouble(ref ArgSlice slice, bool canBeInfinite) + public static double ReadDouble(PinnedSpanByte slice, bool canBeInfinite) { - if (!TryReadDouble(ref slice, out var number, canBeInfinite)) + if (!TryReadDouble(slice, out var number, canBeInfinite)) { RespParsingException.ThrowNotANumber(slice.ptr, slice.length); } @@ -120,7 +138,7 @@ public static double ReadDouble(ref ArgSlice slice, bool canBeInfinite) /// True if double parsed successfully /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryReadDouble(ref ArgSlice slice, out double number, bool canBeInfinite) + public static bool TryReadDouble(PinnedSpanByte slice, out double number, bool canBeInfinite) { var sbNumber = slice.ReadOnlySpan; if (Utf8Parser.TryParse(sbNumber, out number, out var bytesConsumed) && @@ -139,9 +157,9 @@ public static bool TryReadDouble(ref ArgSlice slice, out double number, bool can /// Parsed double /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static float ReadFloat(ref ArgSlice slice, bool canBeInfinite) + public static float ReadFloat(PinnedSpanByte slice, bool canBeInfinite) { - if (!TryReadFloat(ref slice, out var number, canBeInfinite)) + if (!TryReadFloat(slice, out var number, canBeInfinite)) { RespParsingException.ThrowNotANumber(slice.ptr, slice.length); } @@ -158,7 +176,7 @@ public static float ReadFloat(ref ArgSlice slice, bool canBeInfinite) /// True if float parsed successfully /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryReadFloat(ref ArgSlice slice, out float number, bool canBeInfinite) + public static bool TryReadFloat(PinnedSpanByte slice, out float number, bool canBeInfinite) { var sbNumber = slice.ReadOnlySpan; if (Utf8Parser.TryParse(sbNumber, out number, out var bytesConsumed) && @@ -175,7 +193,7 @@ public static bool TryReadFloat(ref ArgSlice slice, out float number, bool canBe /// Parsed string /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string ReadString(ref ArgSlice slice) + public static string ReadString(PinnedSpanByte slice) { return Encoding.ASCII.GetString(slice.ReadOnlySpan); } @@ -187,9 +205,9 @@ public static string ReadString(ref ArgSlice slice) /// Parsed integer /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool ReadBool(ref ArgSlice slice) + public static bool ReadBool(PinnedSpanByte slice) { - if (!TryReadBool(ref slice, out var value)) + if (!TryReadBool(slice, out var value)) { RespParsingException.ThrowNotANumber(slice.ptr, slice.length); } @@ -203,7 +221,7 @@ public static bool ReadBool(ref ArgSlice slice) /// True if integer read successfully /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryReadBool(ref ArgSlice slice, out bool value) + public static bool TryReadBool(PinnedSpanByte slice, out bool value) { value = false; diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 3ed6ce1554a..3dd90e749ec 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Text; using Garnet.common; +using Garnet.common.Parsing; using Microsoft.Extensions.Logging; namespace Garnet.server @@ -117,6 +118,14 @@ public enum RespCommand : ushort ZSCORE, // Note: Last read command should immediately precede FirstWriteCommand ZUNION, + // Read-only RangeIndex commands + RICONFIG, + RIEXISTS, + RIGET, + RIMETRICS, + RIRANGE, + RISCAN, + // Write commands APPEND, // Note: Update FirstWriteCommand if adding new write commands before this BITFIELD, @@ -178,6 +187,11 @@ public enum RespCommand : ushort PFMERGE, PSETEX, RENAME, + RICREATE, + RIDEL, + RIPROMOTE, + RIRESTORE, + RISET, RESTORE, RENAMENX, RPOP, @@ -194,6 +208,7 @@ public enum RespCommand : ushort SETNX, SETIFMATCH, SETIFGREATER, + SETWITHETAG, SETKEEPTTL, SETKEEPTTLXX, SETRANGE, @@ -353,7 +368,7 @@ public enum RespCommand : ushort CLUSTER, CLUSTER_ADDSLOTS, // Note: Update IsClusterSubCommand if adding new cluster subcommands before this CLUSTER_ADDSLOTSRANGE, - CLUSTER_AOFSYNC, + CLUSTER_ADVANCE_TIME, CLUSTER_APPENDLOG, CLUSTER_ATTACH_SYNC, CLUSTER_BANLIST, @@ -378,6 +393,7 @@ public enum RespCommand : ushort CLUSTER_KEYSLOT, CLUSTER_MEET, CLUSTER_MIGRATE, + CLUSTER_MLOG_KEY_TIME, CLUSTER_MTASKS, CLUSTER_MYID, CLUSTER_MYPARENTID, @@ -396,6 +412,7 @@ public enum RespCommand : ushort CLUSTER_SHARDS, CLUSTER_SLOTS, CLUSTER_SLOTSTATE, + CLUSTER_SNAPSHOT_DATA, CLUSTER_SYNC, // Note: Update IsClusterSubCommand if adding new cluster subcommands after this // Don't require AUTH (if auth is enabled) @@ -568,6 +585,36 @@ public static ReadOnlySpan ExpandForACLs(this RespCommand cmd) public static bool IsReadOnly(this RespCommand cmd) => cmd <= LastReadCommand; + /// + /// Returns true if this command can legally operate on a RangeIndex key. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsLegalOnRangeIndex(this RespCommand cmd) + => cmd is RespCommand.DEL or RespCommand.UNLINK or RespCommand.TYPE + or RespCommand.DEBUG or RespCommand.RENAME or RespCommand.RENAMENX + or RespCommand.RICREATE or RespCommand.RIPROMOTE or RespCommand.RIRESTORE or RespCommand.RISET or RespCommand.RIGET or RespCommand.RIDEL + or RespCommand.RISCAN or RespCommand.RIRANGE + or RespCommand.RIEXISTS or RespCommand.RICONFIG or RespCommand.RIMETRICS; + + /// + /// Returns true if this command is a RangeIndex-specific command (not a generic command that happens to be legal on RI keys). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsRangeIndexCommand(this RespCommand cmd) + => cmd is RespCommand.RICREATE or RespCommand.RISET or RespCommand.RIGET or RespCommand.RIDEL + or RespCommand.RISCAN or RespCommand.RIRANGE + or RespCommand.RIPROMOTE or RespCommand.RIRESTORE + or RespCommand.RIEXISTS or RespCommand.RICONFIG or RespCommand.RIMETRICS; + + /// + /// Returns true if this command is a VectorSet-specific command (not a generic command that happens to be legal on Vector keys). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsVectorSetCommand(this RespCommand cmd) + => cmd is RespCommand.VADD or RespCommand.VCARD or RespCommand.VDIM or RespCommand.VEMB + or RespCommand.VGETATTR or RespCommand.VINFO or RespCommand.VISMEMBER or RespCommand.VLINKS + or RespCommand.VRANDMEMBER or RespCommand.VREM or RespCommand.VSETATTR or RespCommand.VSIM; + public static bool IsDataCommand(this RespCommand cmd) { return cmd switch @@ -645,7 +692,7 @@ public static bool IsClusterSubCommand(this RespCommand cmd) /// Returns true if this command can operate on a Vector Set. /// public static bool IsLegalOnVectorSet(this RespCommand cmd) - => cmd is RespCommand.DEL or server.RespCommand.UNLINK or RespCommand.TYPE or RespCommand.DEBUG or RespCommand.RENAME or RespCommand.RENAMENX or RespCommand.VADD or RespCommand.VCARD or RespCommand.VDIM or RespCommand.VEMB or RespCommand.VGETATTR or RespCommand.VINFO or server.RespCommand.VISMEMBER or RespCommand.VLINKS or RespCommand.VRANDMEMBER or RespCommand.VREM or RespCommand.VSETATTR or RespCommand.VSIM; + => cmd is RespCommand.DEL or RespCommand.UNLINK or RespCommand.TYPE or RespCommand.DEBUG or RespCommand.RENAME or RespCommand.RENAMENX or RespCommand.VADD or RespCommand.VCARD or RespCommand.VDIM or RespCommand.VEMB or RespCommand.VGETATTR or RespCommand.VINFO or RespCommand.VISMEMBER or RespCommand.VLINKS or RespCommand.VRANDMEMBER or RespCommand.VREM or RespCommand.VSETATTR or RespCommand.VSIM; } /// @@ -661,6 +708,12 @@ enum RespCommandOption : byte /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { + /// + /// Maximum number of arguments (excluding the command name) allowed in a single RESP command. + /// Prevents pre-auth memory exhaustion from oversized RESP array headers. + /// + const int MaxRespArrayLength = 1 << 20; // 1,048,576 + /// /// Fast-parses command type for inline RESP commands, starting at the current read head in the receive buffer /// and advances read head. @@ -820,6 +873,7 @@ static RespCommand MatchedNone(RespServerSession session, int oldReadHead) return RespCommand.NONE; } + /// /// Fast parsing function for common command names. /// Parses the receive buffer starting from the current read head and advances it to the end of @@ -1316,7 +1370,15 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan break; case 'R': - if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPUSHX\r\n"u8)) + if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RI.GET\r\n"u8)) + { + return RespCommand.RIGET; + } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RI.SET\r\n"u8)) + { + return RespCommand.RISET; + } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RPUSHX\r\n"u8)) { return RespCommand.RPUSHX; } @@ -1488,6 +1550,12 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan return RespCommand.PFMERGE; } break; + case 'R': + if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("RI.SCAN\r"u8) && *(byte*)(ptr + 12) == '\n') + { + return RespCommand.RISCAN; + } + break; case 'W': if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("WATCHMS\r"u8) && *(byte*)(ptr + 12) == '\n') { @@ -2285,10 +2353,6 @@ private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, { return RespCommand.CLUSTER_MTASKS; } - else if (subCommand.SequenceEqual(CmdStrings.aofsync)) - { - return RespCommand.CLUSTER_AOFSYNC; - } else if (subCommand.SequenceEqual(CmdStrings.appendlog)) { return RespCommand.CLUSTER_APPENDLOG; @@ -2337,10 +2401,22 @@ private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, { return RespCommand.CLUSTER_SEND_CKPT_METADATA; } + else if (subCommand.SequenceEqual(CmdStrings.snapshot_data)) + { + return RespCommand.CLUSTER_SNAPSHOT_DATA; + } + else if (subCommand.SequenceEqual(CmdStrings.mlog_key_time)) + { + return RespCommand.CLUSTER_MLOG_KEY_TIME; + } else if (subCommand.SequenceEqual(CmdStrings.cluster_sync)) { return RespCommand.CLUSTER_SYNC; } + else if (subCommand.SequenceEqual(CmdStrings.cluster_advance_time)) + { + return RespCommand.CLUSTER_ADVANCE_TIME; + } string errMsg = string.Format(CmdStrings.GenericErrUnknownSubCommand, Encoding.UTF8.GetString(subCommand), @@ -2666,6 +2742,10 @@ private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, { return RespCommand.SETIFGREATER; } + else if (command.SequenceEqual(CmdStrings.SETWITHETAG)) + { + return RespCommand.SETWITHETAG; + } else if (command.SequenceEqual(CmdStrings.GETWITHETAG)) { return RespCommand.GETWITHETAG; @@ -2678,6 +2758,42 @@ private RespCommand SlowParseCommand(ReadOnlySpan command, ref int count, { return RespCommand.DELIFGREATER; } + else if (command.SequenceEqual(CmdStrings.RICREATE)) + { + return RespCommand.RICREATE; + } + else if (command.SequenceEqual(CmdStrings.RISET)) + { + return RespCommand.RISET; + } + else if (command.SequenceEqual(CmdStrings.RIGET)) + { + return RespCommand.RIGET; + } + else if (command.SequenceEqual(CmdStrings.RIDEL)) + { + return RespCommand.RIDEL; + } + else if (command.SequenceEqual(CmdStrings.RISCAN)) + { + return RespCommand.RISCAN; + } + else if (command.SequenceEqual(CmdStrings.RIRANGE)) + { + return RespCommand.RIRANGE; + } + else if (command.SequenceEqual(CmdStrings.RIEXISTS)) + { + return RespCommand.RIEXISTS; + } + else if (command.SequenceEqual(CmdStrings.RICONFIG)) + { + return RespCommand.RICONFIG; + } + else if (command.SequenceEqual(CmdStrings.RIMETRICS)) + { + return RespCommand.RIMETRICS; + } // If this command name was not known to the slow pass, we are out of options and the command is unknown. return RespCommand.INVALID; @@ -2822,6 +2938,11 @@ private RespCommand ParseCommand(bool writeErrorOnFailure, out bool success) if (!success) return cmd; } + if (count > MaxRespArrayLength) + { + RespParsingException.ThrowExcessiveArgumentCount(count, MaxRespArrayLength); + } + // Set up parse state parseState.Initialize(count); var ptr = recvBufferPtr + readHead; diff --git a/libs/server/Resp/Parser/SessionParseState.cs b/libs/server/Resp/Parser/SessionParseState.cs index c25fb1cc8dc..3bc61c6fe8e 100644 --- a/libs/server/Resp/Parser/SessionParseState.cs +++ b/libs/server/Resp/Parser/SessionParseState.cs @@ -27,31 +27,32 @@ public unsafe struct SessionParseState public int Count; /// - /// Pointer to accessible buffer + /// Get a Span of the parsed parameters in the form an PinnedSpanByte /// - ArgSlice* bufferPtr; + public readonly ReadOnlySpan Parameters => new(bufferPtr, Count); /// - /// Count of arguments in the original buffer + /// Pointer to the slice of (which is always pinned) that is accessible within the range of this instance's arguments. /// - int rootCount; + PinnedSpanByte* bufferPtr; /// - /// Arguments original buffer + /// Count of arguments in the original buffer /// - ArgSlice[] rootBuffer; + int rootCount; /// - /// Get a Span of the parsed parameters in the form an ArgSlice + /// Arguments original buffer (always pinned) /// - public ReadOnlySpan Parameters => new(bufferPtr, Count); + PinnedSpanByte[] rootBuffer; - private SessionParseState(ref ArgSlice[] rootBuffer, int rootCount, ref ArgSlice* bufferPtr, int count) : this() + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private SessionParseState(ref PinnedSpanByte[] rootBuffer, int rootCount, PinnedSpanByte* bufferPtr, int count) { this.rootBuffer = rootBuffer; this.rootCount = rootCount; this.bufferPtr = bufferPtr; - this.Count = count; + Count = count; } /// @@ -61,8 +62,8 @@ public void Initialize() { Count = 0; rootCount = 0; - rootBuffer = GC.AllocateArray(MinParams, true); - bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref rootBuffer[0]); + rootBuffer = GC.AllocateArray(MinParams, true); + bufferPtr = (PinnedSpanByte*)Unsafe.AsPointer(ref rootBuffer[0]); } /// @@ -78,8 +79,8 @@ public void Initialize(int count) if (rootBuffer != null && (count <= MinParams || count <= rootBuffer.Length)) return; - rootBuffer = GC.AllocateArray(count <= MinParams ? MinParams : count, true); - bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref rootBuffer[0]); + rootBuffer = GC.AllocateArray(count <= MinParams ? MinParams : count, true); + bufferPtr = (PinnedSpanByte*)Unsafe.AsPointer(ref rootBuffer[0]); } /// @@ -87,7 +88,7 @@ public void Initialize(int count) /// /// Argument to initialize buffer with [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArgument(ArgSlice arg) + public void InitializeWithArgument(PinnedSpanByte arg) { Initialize(1); @@ -100,7 +101,7 @@ public void InitializeWithArgument(ArgSlice arg) /// First argument /// Second argument [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2) + public void InitializeWithArguments(PinnedSpanByte arg1, PinnedSpanByte arg2) { Initialize(2); @@ -115,7 +116,7 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2) /// Second argument /// Third argument [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3) + public void InitializeWithArguments(PinnedSpanByte arg1, PinnedSpanByte arg2, PinnedSpanByte arg3) { Initialize(3); @@ -132,7 +133,7 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3) /// Third argument /// Fourth argument [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, ArgSlice arg4) + public void InitializeWithArguments(PinnedSpanByte arg1, PinnedSpanByte arg2, PinnedSpanByte arg3, PinnedSpanByte arg4) { Initialize(4); @@ -151,7 +152,7 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, /// Fourth argument /// Fifth argument [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, ArgSlice arg4, ArgSlice arg5) + public void InitializeWithArguments(PinnedSpanByte arg1, PinnedSpanByte arg2, PinnedSpanByte arg3, PinnedSpanByte arg4, PinnedSpanByte arg5) { Initialize(5); @@ -163,32 +164,62 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, } /// - /// Expand (if necessary) capacity of , preserving contents. + /// Initialize the parse state with a given set of arguments /// - public void EnsureCapacity(int count) + /// Set of arguments to initialize buffer with + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void InitializeWithArguments(PinnedSpanByte[] args) { - if (count <= Count) - { - return; - } + Initialize(args.Length); + + for (var i = 0; i < args.Length; i++) + *(bufferPtr + i) = args[i]; + } + /// + /// Ensure the argument buffer can hold at least entries + /// from the current slice offset, preserving existing contents. No-op if already large enough. + /// + public void EnsureCapacity(int capacity) + { var oldBuffer = rootBuffer; - Initialize(count); - oldBuffer?.AsSpan().CopyTo(rootBuffer); + // Compute slice offset (bufferPtr may point into the middle of rootBuffer) + var sliceOffset = oldBuffer != null + ? (int)(bufferPtr - (PinnedSpanByte*)Unsafe.AsPointer(ref oldBuffer[0])) + : 0; + + // Total buffer size needed = slice offset + requested capacity + var requiredLength = sliceOffset + capacity; + + if (oldBuffer != null && requiredLength <= oldBuffer.Length) + return; + + var oldCount = Count; + Initialize(requiredLength); + + if (oldBuffer != null) + { + // Copy all data up to the end of the current slice + var copyLength = sliceOffset + oldCount; + if (copyLength > 0) + oldBuffer.AsSpan(0, copyLength).CopyTo(rootBuffer); + } + + // Restore slice offset and count + bufferPtr = (PinnedSpanByte*)Unsafe.AsPointer(ref rootBuffer[0]) + sliceOffset; + Count = oldCount; } /// /// Limit access to the argument buffer to start at a specified index. /// /// Offset value to the underlying buffer + [MethodImpl(MethodImplOptions.AggressiveInlining)] public SessionParseState Slice(int idxOffset) { Debug.Assert(idxOffset - 1 < rootCount); - - var count = rootCount - idxOffset; - var offsetBuffer = bufferPtr + idxOffset; - return new SessionParseState(ref rootBuffer, rootCount, ref offsetBuffer, count); + return new SessionParseState(ref rootBuffer, rootCount, bufferPtr: bufferPtr + idxOffset, count: rootCount - idxOffset); } /// @@ -197,12 +228,11 @@ public SessionParseState Slice(int idxOffset) /// /// Offset value to the underlying buffer /// Argument count + [MethodImpl(MethodImplOptions.AggressiveInlining)] public SessionParseState Slice(int idxOffset, int count) { Debug.Assert(idxOffset + count - 1 < rootCount); - - var offsetBuffer = bufferPtr + idxOffset; - return new SessionParseState(ref rootBuffer, rootCount, ref offsetBuffer, count); + return new SessionParseState(ref rootBuffer, rootCount, bufferPtr: bufferPtr + idxOffset, count); } /// @@ -210,14 +240,12 @@ public SessionParseState Slice(int idxOffset, int count) /// /// Set of arguments to initialize buffer with [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(ReadOnlySpan args) + public void InitializeWithArguments(ReadOnlySpan args) { Initialize(args.Length); for (var i = 0; i < args.Length; i++) - { *(bufferPtr + i) = args[i]; - } } /// @@ -225,24 +253,13 @@ public void InitializeWithArguments(ReadOnlySpan args) /// /// Index of buffer at which to set argument /// Argument to set - public void SetArgument(int i, ArgSlice arg) + public void SetArgument(int i, PinnedSpanByte arg) { - Debug.Assert(i < Count); + Debug.Assert(i < rootBuffer.Length); *(bufferPtr + i) = arg; - } - /// - /// Set arguments starting at a specific index - /// - /// Index of buffer at which to start setting arguments - /// Arguments to set - public void SetArguments(int i, params ArgSlice[] args) - { - Debug.Assert(i + args.Length - 1 < Count); - for (var j = 0; j < args.Length; j++) - { - *(bufferPtr + i + j) = args[j]; - } + if (i >= Count) + Count = i + 1; } /// @@ -250,28 +267,23 @@ public void SetArguments(int i, params ArgSlice[] args) /// /// Index of buffer at which to start setting arguments /// Arguments to set - public void SetArguments(int i, params ReadOnlySpan args) + public readonly void SetArguments(int i, params ReadOnlySpan args) { Debug.Assert(i + args.Length - 1 < Count); for (var j = 0; j < args.Length; j++) - { *(bufferPtr + i + j) = args[j]; - } } /// /// Get serialized length of parse state /// /// The serialized length - public int GetSerializedLength() + public readonly int GetSerializedLength() { var serializedLength = sizeof(int); for (var i = 0; i < Count; i++) - { - serializedLength += (*(bufferPtr + i)).SpanByte.TotalSize; - } - + serializedLength += (*(bufferPtr + i)).TotalSize; return serializedLength; } @@ -282,7 +294,7 @@ public int GetSerializedLength() /// The memory buffer to serialize into (of size at least SerializedLength(firstIdx) bytes) /// Length of buffer to serialize into. /// Total serialized bytes - public int CopyTo(byte* dest, int length) + public readonly int SerializeTo(byte* dest, int length) { var curr = dest; @@ -293,9 +305,9 @@ public int CopyTo(byte* dest, int length) // Serialize arguments for (var i = 0; i < Count; i++) { - var sbParam = (*(bufferPtr + i)).SpanByte; - sbParam.CopyTo(curr); - curr += sbParam.TotalSize; + var argument = *(bufferPtr + i); + argument.SerializeTo(curr); + curr += argument.TotalSize; } return (int)(curr - dest); @@ -306,7 +318,7 @@ public int CopyTo(byte* dest, int length) /// /// Memory buffer to deserialize from /// Number of deserialized bytes - public unsafe int DeserializeFrom(byte* src) + public int DeserializeFrom(byte* src) { var curr = src; @@ -317,9 +329,9 @@ public unsafe int DeserializeFrom(byte* src) for (var i = 0; i < argCount; i++) { - ref var sbArgument = ref Unsafe.AsRef(curr); - *(bufferPtr + i) = new ArgSlice(ref sbArgument); - curr += sbArgument.TotalSize; + var argument = PinnedSpanByte.FromLengthPrefixedPinnedPointer(curr); + *(bufferPtr + i) = argument; + curr += argument.TotalSize; } return (int)(curr - src); @@ -329,30 +341,23 @@ public unsafe int DeserializeFrom(byte* src) /// Read the next argument from the input buffer /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Read(int i, ref byte* ptr, byte* end) + public readonly bool Read(int i, ref byte* ptr, byte* end) { Debug.Assert(i < Count); - ref var slice = ref Unsafe.AsRef(bufferPtr + i); + ref var slice = ref Unsafe.AsRef(bufferPtr + i); // Parse RESP string header - if (!RespReadUtils.TryReadUnsignedLengthHeader(out slice.length, ref ptr, end)) - { + if (!RespReadUtils.TryReadUnsignedLengthHeader(out var length, ref ptr, end)) return false; - } - - slice.ptr = ptr; + slice.Set(ptr, length); // Parse content: ensure that input contains key + '\r\n' - ptr += slice.length + 2; + ptr += slice.Length + 2; if (ptr > end) - { return false; - } if (*(ushort*)(ptr - 2) != MemoryMarshal.Read("\r\n"u8)) - { RespParsingException.ThrowUnexpectedToken(*(ptr - 2)); - } return true; } @@ -361,10 +366,10 @@ public bool Read(int i, ref byte* ptr, byte* end) /// Get the argument at the given index /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref ArgSlice GetArgSliceByRef(int i) + public readonly ref PinnedSpanByte GetArgSliceByRef(int i) { Debug.Assert(i < Count); - return ref Unsafe.AsRef(bufferPtr + i); + return ref Unsafe.AsRef(bufferPtr + i); } /// @@ -372,10 +377,10 @@ public ref ArgSlice GetArgSliceByRef(int i) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetInt(int i) + public readonly int GetInt(int i) { Debug.Assert(i < Count); - return ParseUtils.ReadInt(ref Unsafe.AsRef(bufferPtr + i)); + return ParseUtils.ReadInt(*(bufferPtr + i)); } /// @@ -383,10 +388,10 @@ public int GetInt(int i) /// /// True if integer parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetInt(int i, out int value) + public readonly bool TryGetInt(int i, out int value) { Debug.Assert(i < Count); - return ParseUtils.TryReadInt(ref Unsafe.AsRef(bufferPtr + i), out value); + return ParseUtils.TryReadInt(*(bufferPtr + i), out value); } /// @@ -394,10 +399,21 @@ public bool TryGetInt(int i, out int value) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetLong(int i) + public readonly long GetLong(int i) + { + Debug.Assert(i < Count); + return ParseUtils.ReadLong(*(bufferPtr + i)); + } + + /// + /// Try to get long argument at the given index + /// + /// True if long parsed successfully + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TryGetLong(int i, out long value) { Debug.Assert(i < Count); - return ParseUtils.ReadLong(ref Unsafe.AsRef(bufferPtr + i)); + return ParseUtils.TryReadLong(*(bufferPtr + i), out value); } /// @@ -405,10 +421,10 @@ public long GetLong(int i) /// /// True if long parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetLong(int i, out long value) + public readonly bool TryGetLong(int i, bool allowLeadingZeros, out long value) { Debug.Assert(i < Count); - return ParseUtils.TryReadLong(ref Unsafe.AsRef(bufferPtr + i), out value); + return ParseUtils.TryReadLong(*(bufferPtr + i), allowLeadingZeros, out value); } /// @@ -416,10 +432,10 @@ public bool TryGetLong(int i, out long value) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public double GetDouble(int i, bool canBeInfinite = true) + public readonly double GetDouble(int i, bool canBeInfinite = true) { Debug.Assert(i < Count); - return ParseUtils.ReadDouble(ref Unsafe.AsRef(bufferPtr + i), canBeInfinite); + return ParseUtils.ReadDouble(Unsafe.AsRef(bufferPtr + i), canBeInfinite); } /// @@ -427,10 +443,10 @@ public double GetDouble(int i, bool canBeInfinite = true) /// /// True if double parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetDouble(int i, out double value, bool canBeInfinite = true) + public readonly bool TryGetDouble(int i, out double value, bool canBeInfinite = true) { Debug.Assert(i < Count); - return ParseUtils.TryReadDouble(ref Unsafe.AsRef(bufferPtr + i), out value, canBeInfinite); + return ParseUtils.TryReadDouble(Unsafe.AsRef(bufferPtr + i), out value, canBeInfinite); } /// @@ -438,10 +454,10 @@ public bool TryGetDouble(int i, out double value, bool canBeInfinite = true) /// /// True if double parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public float GetFloat(int i, bool canBeInfinite = true) + public readonly float GetFloat(int i, bool canBeInfinite = true) { Debug.Assert(i < Count); - return ParseUtils.ReadFloat(ref Unsafe.AsRef(bufferPtr + i), canBeInfinite); + return ParseUtils.ReadFloat(Unsafe.AsRef(bufferPtr + i), canBeInfinite); } /// @@ -449,10 +465,10 @@ public float GetFloat(int i, bool canBeInfinite = true) /// /// True if double parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetFloat(int i, out float value, bool canBeInfinite = true) + public readonly bool TryGetFloat(int i, out float value, bool canBeInfinite = true) { Debug.Assert(i < Count); - return ParseUtils.TryReadFloat(ref Unsafe.AsRef(bufferPtr + i), out value, canBeInfinite); + return ParseUtils.TryReadFloat(Unsafe.AsRef(bufferPtr + i), out value, canBeInfinite); } /// @@ -460,10 +476,10 @@ public bool TryGetFloat(int i, out float value, bool canBeInfinite = true) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public string GetString(int i) + public readonly string GetString(int i) { Debug.Assert(i < Count); - return ParseUtils.ReadString(ref Unsafe.AsRef(bufferPtr + i)); + return ParseUtils.ReadString(*(bufferPtr + i)); } /// @@ -471,10 +487,10 @@ public string GetString(int i) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool GetBool(int i) + public readonly bool GetBool(int i) { Debug.Assert(i < Count); - return ParseUtils.ReadBool(ref Unsafe.AsRef(bufferPtr + i)); + return ParseUtils.ReadBool(*(bufferPtr + i)); } /// @@ -482,10 +498,10 @@ public bool GetBool(int i) /// /// True if boolean parsed successfully [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryGetBool(int i, out bool value) + public readonly bool TryGetBool(int i, out bool value) { Debug.Assert(i < Count); - return ParseUtils.TryReadBool(ref Unsafe.AsRef(bufferPtr + i), out value); + return ParseUtils.TryReadBool(*(bufferPtr + i), out value); } } } \ No newline at end of file diff --git a/libs/server/Resp/PubSubCommands.cs b/libs/server/Resp/PubSubCommands.cs index d353b7b5a7a..9b05c1667e8 100644 --- a/libs/server/Resp/PubSubCommands.cs +++ b/libs/server/Resp/PubSubCommands.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -18,7 +19,7 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase int numActiveChannels = 0; /// - public override unsafe void Publish(ArgSlice key, ArgSlice value) + public override unsafe void Publish(PinnedSpanByte key, PinnedSpanByte value) { try { @@ -48,7 +49,7 @@ public override unsafe void Publish(ArgSlice key, ArgSlice value) } /// - public override unsafe void PatternPublish(ArgSlice pattern, ArgSlice key, ArgSlice value) + public override unsafe void PatternPublish(PinnedSpanByte pattern, PinnedSpanByte key, PinnedSpanByte value) { try { diff --git a/libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs b/libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs new file mode 100644 index 00000000000..22e87c95b58 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs @@ -0,0 +1,397 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Garnet.common; +using Garnet.server.BfTreeInterop; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Stub struct, serialization, and flag management methods for RangeIndex records. + /// The stub is the fixed-size value stored inline in Tsavorite's log for each RI key. + /// + public sealed partial class RangeIndexManager + { + /// + /// Fixed-size struct stored as a raw-byte value in Tsavorite's unified store. + /// Contains BfTree configuration metadata and a native pointer to the live instance. + /// + /// Layout: The struct uses explicit field offsets for deterministic binary layout + /// (no padding, no reordering). This ensures the stub can be safely reinterpreted + /// via Unsafe.As from raw store value spans. + /// + /// Total size: (35 bytes). + /// + /// Flags byte (offset 33) bit layout (LSB = bit 0 = first allocated): + /// + /// [Unused7][Unused6][Unused5][Pinned*][Modified*][Transferred][Recovered][Flushed] + /// + /// * = reserved for future use; bit position allocated, no property exposed yet. + /// + /// Flushed — stub has been flushed; needs promotion to tail on next access. + /// Recovered — stub was loaded from a checkpoint snapshot file. + /// Transferred — ownership transferred to a newer record (compaction or RIPROMOTE). + /// Modified* — [reserved] for future write-tracking. + /// Pinned* — [reserved] for future eviction-pinning. + /// Unused5..7 — available for future expansion. + /// + /// + [StructLayout(LayoutKind.Explicit, Size = Size)] + internal struct RangeIndexStub + { + internal const int Size = 35; + + // Flag bit masks (Flags byte at offset 33). + private const byte kFlushedBitMask = 1 << 0; + private const byte kRecoveredBitMask = 1 << 1; + private const byte kTransferredBitMask = 1 << 2; + // Reserved for future use (bit positions allocated; properties deferred until semantics exist): + // bit 3 = Modified + // bit 4 = Pinned + // bits 5..7 currently unused + + /// Pointer to the live BfTreeService instance (managed object handle). + [FieldOffset(0)] + public nint TreeHandle; + + /// BfTree circular buffer size in bytes. + [FieldOffset(8)] + public ulong CacheSize; + + /// BfTree minimum record size. + [FieldOffset(16)] + public uint MinRecordSize; + + /// BfTree maximum record size. + [FieldOffset(20)] + public uint MaxRecordSize; + + /// BfTree maximum key length. + [FieldOffset(24)] + public uint MaxKeyLen; + + /// BfTree leaf page size. + [FieldOffset(28)] + public uint LeafPageSize; + + /// Storage backend: 0=Disk, 1=Memory. + [FieldOffset(32)] + public byte StorageBackend; + + /// Flags byte. Private so all writes must go through typed properties / , + /// which centralizes invariant control as additional bits gain semantics. + [FieldOffset(33)] + private byte flags; + + /// Serialization phase for checkpoint coordination. + [FieldOffset(34)] + public byte SerializationPhase; + + /// Whether the stub has been flushed and needs promotion to tail. + internal bool IsFlushed + { + readonly get => (flags & kFlushedBitMask) != 0; + set => flags = value ? (byte)(flags | kFlushedBitMask) : (byte)(flags & ~kFlushedBitMask); + } + + /// Whether the stub was recovered from a checkpoint snapshot file. + internal bool IsRecovered + { + readonly get => (flags & kRecoveredBitMask) != 0; + set => flags = value ? (byte)(flags | kRecoveredBitMask) : (byte)(flags & ~kRecoveredBitMask); + } + + /// Whether ownership has been transferred to a newer record at the tail + /// (compaction or RIPROMOTE PostCopyUpdater + /// set this on the source record so a later on the + /// stale source does NOT remove the liveIndexes entry that now belongs to the newer + /// destination, and so a later on the stale source + /// does NOT snapshot a stale view of data.bftree). + internal bool IsTransferred + { + readonly get => (flags & kTransferredBitMask) != 0; + set => flags = value ? (byte)(flags | kTransferredBitMask) : (byte)(flags & ~kTransferredBitMask); + } + + /// Reset all flag bits to 0. Used by on a freshly initialized + /// stub and by AOF replay () when reinitializing an + /// in-memory stub for an RI.CREATE record. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ResetFlags() => flags = 0; + } + + /// + /// Write a new stub into the value span of a LogRecord. + /// Called from InitialUpdater in RMWMethods.cs during RI.CREATE. + /// + /// BfTree circular buffer size in bytes. + /// BfTree minimum record size. + /// BfTree maximum record size. + /// BfTree maximum key length. + /// BfTree leaf page size. + /// Storage backend type (0=Disk, 1=Memory). + /// Native pointer to the live BfTreeService instance. + /// The store value span to write the stub into (must be ≥ ). + internal void CreateIndex( + ulong cacheSize, + uint minRecordSize, + uint maxRecordSize, + uint maxKeyLen, + uint leafPageSize, + byte storageBackend, + nint treeHandle, + Span valueSpan) + { + Debug.Assert(Unsafe.SizeOf() == RangeIndexStub.Size, "Constant stub size is incorrect"); + Debug.Assert(valueSpan.Length >= RangeIndexStub.Size, $"Value span too small: {valueSpan.Length} < {RangeIndexStub.Size}"); + + ref var stub = ref Unsafe.As(ref MemoryMarshal.GetReference(valueSpan)); + stub.TreeHandle = treeHandle; + stub.CacheSize = cacheSize; + stub.MinRecordSize = minRecordSize; + stub.MaxRecordSize = maxRecordSize; + stub.MaxKeyLen = maxKeyLen; + stub.LeafPageSize = leafPageSize; + stub.StorageBackend = storageBackend; + stub.ResetFlags(); + stub.SerializationPhase = 0; + } + + /// + /// Get a readonly reference to the stub from a store value span (zero-copy). + /// Uses Unsafe.As to reinterpret the raw bytes — the caller must ensure the + /// span is at least long. + /// + /// Raw value bytes from the store record. + /// A readonly reference to the reinterpreted stub. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ref readonly RangeIndexStub ReadIndex(ReadOnlySpan value) + => ref Unsafe.As(ref MemoryMarshal.GetReference(value)); + + /// + /// Update TreeHandle after lazy restore from a flush or checkpoint snapshot file. + /// Called by after recovering the native BfTree. + /// + /// The native pointer of the newly restored BfTree. + /// The store value span containing the stub. + internal static void RecreateIndex(nint newTreeHandle, Span valueSpan) + { + ref var stub = ref Unsafe.As(ref MemoryMarshal.GetReference(valueSpan)); + stub.TreeHandle = newTreeHandle; + + // Clear IsRecovered so that future eviction cycles use the flush snapshot + // (which reflects post-recovery writes) instead of the stale checkpoint snapshot. + stub.IsRecovered = false; + } + + /// + /// Zero the TreeHandle in a stub span. Used by CopyUpdater (RIPROMOTE) to prevent + /// the old record's eviction callback from freeing the BfTree that the new + /// (promoted) record now owns. + /// + /// The store value span containing the stub. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void ClearTreeHandle(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.TreeHandle = nint.Zero; + } + + /// + /// Set the flag on a stub span. Called by + /// PostCopyToTail (compaction) and RIPROMOTE PostCopyUpdater on the SOURCE record after + /// ownership is transferred to a new tail destination. This guarantees a later + /// on the stale source no-ops (does not remove + /// the liveIndexes entry that now belongs to the destination), and a later + /// on the stale source no-ops (does not + /// snapshot a now-stale view of data.bftree). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetTransferredFlag(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.IsTransferred = true; + } + + /// + /// Dispose the BfTree under an exclusive lock, preventing concurrent RI data + /// operations and checkpoint snapshots from accessing the tree during disposal. + /// Used by (delete) and + /// (page eviction). + /// + /// The raw key bytes (used to compute the key hash for lock acquisition). + /// The store value span containing the stub. + /// When true (DEL/UNLINK), delete the working + /// <hash>.data.bftree file and the CPR scratch <hash>.scratch.cpr + /// in the riLogRoot. Per-flush snapshot files (<hash>.<addr>.flush.bftree) + /// are deliberately preserved — they are LOG-tied (lifetime tracks BeginAddress) and may + /// still be needed to recover an OLDER checkpoint that pre-dates the DEL. They are + /// reclaimed by only once Tsavorite's BeginAddress passes + /// their address. When false (eviction), files are preserved for lazy restore. + internal void DisposeTreeUnderLock(ReadOnlySpan key, ReadOnlySpan valueSpan, bool deleteFiles) + { + ref readonly var stub = ref ReadIndex(valueSpan); + + // Stale-source eviction (OnEvict) on a record whose ownership was transferred to a + // newer tail record: no-op. The liveIndexes entry now belongs to the destination + // record; removing it here would lose the newer record's tree (live transfer) or + // pending entry (cold transfer). IsTransferred is set by PostCopyToTail and RIPROMOTE + // PostCopyUpdater on the source after a successful CAS. + // For DEL/UNLINK (deleteFiles=true), we still proceed — the user explicitly asked to + // delete this key, and the tombstone path supersedes any newer record as well. + if (stub.IsTransferred && !deleteFiles) + return; + + // All liveIndexes mutations (cold-pending eviction below, plus the activated path + // further down) MUST hold the per-key exclusive lock to serialize against + // SetCheckpointBarrier / SnapshotAllTreesForCheckpoint / RegisterIndex / + // PreStageAndRegisterPending which all touch this entry under the same lock. + // Without it, a cold eviction concurrent with checkpoint can silently drop a + // pending entry mid-snapshot iteration, narrowing checkpoint coverage for the key. + var keyHash = GarnetKeyComparer.StaticGetHashCode64((FixedSpanByteKey)PinnedSpanByte.FromPinnedSpan(key)); + BfTreeService disposedTree = null; + string hashPrefix = null; + rangeIndexLocks.AcquireExclusiveLock(keyHash, out var lockToken); + try + { + if (stub.TreeHandle == nint.Zero && !deleteFiles) + { + // Pending entry cleanup on eviction: remove from liveIndexes (no native tree to dispose). + _ = liveIndexes.TryRemove(KeyId(key), out _); + return; + } + + // Remove entry from liveIndexes synchronously so concurrent restorers see no + // tree for this key. Stash the BfTree wrapper (if any) for deferred disposal. + if (liveIndexes.TryRemove(KeyId(key), out var entry)) + disposedTree = entry?.Tree; + + if (deleteFiles) + hashPrefix = HashKeyToPrefix(key); + } + finally + { + rangeIndexLocks.ReleaseExclusiveLock(lockToken); + } + + // Defer the heavyweight cleanup (native dispose + file delete) via the store epoch. + // The deferred action runs only after all current epoch holders have moved past, so + // any reader that observed a non-zero TreeHandle from this entry's stub completes + // its native call before the bfTree is freed. + if (disposedTree == null && hashPrefix == null) + return; + if (storeEpoch != null) + { + var capturedTree = disposedTree; + var capturedPrefix = hashPrefix; + storeEpoch.BumpCurrentEpoch(() => DisposeAndDeleteFilesDeferred(capturedTree, capturedPrefix)); + } + else + { + // No epoch available (unit-test scenario without a Tsavorite store). Synchronous + // dispose is OK here because no concurrent readers exist in such tests. + DisposeAndDeleteFilesDeferred(disposedTree, hashPrefix); + } + } + + /// + /// Deferred BfTree disposal + file deletion. Invoked from inside + /// storeEpoch.BumpCurrentEpoch action so it runs only after all readers that + /// could have observed the tree's TreeHandle have moved past. + /// + /// The BfTree wrapper to dispose, or null. + /// Hash prefix for file deletion, or null to skip file deletion. + private void DisposeAndDeleteFilesDeferred(BfTreeService tree, string hashPrefix) + { + // Order matters on Windows: dispose first so the native side closes any open + // file handles, then File.Delete (otherwise the unlink races with the close). + if (tree != null) + { + try { tree.Dispose(); } + catch (Exception ex) { logger?.LogWarning(ex, "Deferred dispose failed for BfTree"); } + } + if (hashPrefix != null && !string.IsNullOrEmpty(riLogRoot) && Directory.Exists(riLogRoot)) + { + TryDelete(LogDataPath(hashPrefix)); + TryDelete(LogScratchPath(hashPrefix)); + } + + void TryDelete(string p) + { + try + { + if (File.Exists(p)) File.Delete(p); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Deferred file delete failed: {Path}", p); + } + } + } + + /// + /// Set the Flushed flag on the in-memory stub. Called from + /// on the original (not copied) record, so the next data operation detects the flag + /// and promotes the stub to the mutable tail region via . + /// + /// The store value span containing the stub (writable even though declared ReadOnlySpan, via Unsafe.As). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void SetFlushedFlag(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.IsFlushed = true; + } + + /// + /// Clear the Flushed flag on a stub span. Called by CopyUpdater after promoting + /// the stub to the mutable tail region, so subsequent operations no longer trigger promotion. + /// + /// The store value span containing the promoted stub. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void ClearFlushedFlag(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.IsFlushed = false; + } + + /// + /// Clear TreeHandle on a stub span. Called by + /// when a record is loaded from disk — the native pointer from the original process is stale, + /// so we zero it to signal that the BfTree must be lazily restored on next access. + /// + /// The store value span containing the stub loaded from disk. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void InvalidateStub(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.TreeHandle = nint.Zero; + } + + /// + /// Set the Recovered flag and zero TreeHandle on a stub loaded from a checkpoint snapshot. + /// Used by OnRecoverySnapshotRead to mark stubs whose state is the snapshot + /// rather than any post-recovery flush file. + /// + /// The store value span containing the stub from the checkpoint snapshot. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void MarkRecoveredFromCheckpoint(Span valueSpan) + { + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.TreeHandle = nint.Zero; + stub.IsRecovered = true; + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs b/libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs new file mode 100644 index 00000000000..01c5bd14bf8 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading; +using Garnet.common; +using Garnet.server.BfTreeInterop; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Locking methods for RangeIndex operations. + /// + /// Locking protocol: + /// + /// Shared locks are acquired for data operations (RI.SET, RI.GET, RI.DEL field) + /// via . Multiple sessions can operate on the same BfTree + /// concurrently — the native BfTree is internally thread-safe for point operations. + /// Exclusive locks are acquired for lifecycle operations (DEL key, eviction, + /// checkpoint snapshot, lazy restore) via . + /// These prevent concurrent data operations from accessing a BfTree that is being + /// freed, snapshotted, or restored. + /// + /// + /// Locks are striped by key hash via for scalability. + /// + public sealed partial class RangeIndexManager + { + /// + /// RAII holder for a shared lock on a RangeIndex key. + /// Disposing releases the shared lock. + /// + internal readonly ref struct ReadRangeIndexLock : IDisposable + { + private readonly ref readonly ReadOptimizedLock lockRef; + private readonly int lockToken; + + internal ReadRangeIndexLock(ref readonly ReadOptimizedLock lockRef, int lockToken) + { + this.lockToken = lockToken; + this.lockRef = ref lockRef; + } + + /// + public void Dispose() + { + if (Unsafe.IsNullRef(in lockRef)) + return; + + lockRef.ReleaseSharedLock(lockToken); + } + } + + /// + /// RAII holder for an exclusive lock on a RangeIndex key. + /// Disposing releases the exclusive lock. + /// + internal readonly ref struct ExclusiveRangeIndexLock : IDisposable + { + private readonly ref readonly ReadOptimizedLock lockRef; + private readonly int lockToken; + + internal ExclusiveRangeIndexLock(ref readonly ReadOptimizedLock lockRef, int lockToken) + { + this.lockToken = lockToken; + this.lockRef = ref lockRef; + } + + /// + public void Dispose() + { + if (Unsafe.IsNullRef(in lockRef)) + return; + + lockRef.ReleaseExclusiveLock(lockToken); + } + } + + private readonly ReadOptimizedLock rangeIndexLocks; + + /// + /// Read the RangeIndex stub under a shared lock. + /// Used by RI.SET, RI.GET, RI.DEL (field-level operations). + /// Returns an RAII lock holder; caller operates on the BfTree while the lock is held. + /// + /// + /// This method handles several edge cases transparently: + /// + /// Checkpoint in progress: If the tree is being snapshotted, releases + /// the lock, waits for snapshot completion, and retries. + /// Flushed stub: If the stub has been flushed to the read-only region + /// (IsFlushed set), promotes the stub to the tail via RMW and retries. + /// Null TreeHandle: If the tree was evicted to disk (TreeHandle == 0), + /// triggers lazy restore from the flush/checkpoint snapshot file and retries. + /// + /// + /// The storage session for store reads. + /// The pinned Garnet key. + /// The StringInput for the read operation (used for WRONGTYPE checks). + /// Caller-provided span to receive the stub bytes (must be ≥ ). + /// On return, the read status (OK, NOTFOUND, or WRONGTYPE). + /// An RAII lock holder. Dispose to release the shared lock. + internal ReadRangeIndexLock ReadRangeIndex( + StorageSession session, + PinnedSpanByte key, + ref StringInput input, + scoped Span indexSpan, + out GarnetStatus status) + { + Debug.Assert(indexSpan.Length >= IndexSizeBytes, "Insufficient space for index"); + + var keyHash = session.stringBasicContext.GetKeyHash((FixedSpanByteKey)key); + + Retry: + var output = StringOutput.FromPinnedSpan(indexSpan); + rangeIndexLocks.AcquireSharedLock(keyHash, out var sharedLockToken); + + GarnetStatus readRes; + try + { + readRes = session.Read_RangeIndex(key.ReadOnlySpan, ref input, ref output, ref session.stringBasicContext); + } + catch + { + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + throw; + } + + if (readRes != GarnetStatus.OK) + { + status = readRes; + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + return default; + } + + var outputSpan = output.SpanByteAndMemory.IsSpanByte + ? output.SpanByteAndMemory.SpanByte.ReadOnlySpan + : output.SpanByteAndMemory.MemorySpan; + + if (outputSpan.Length != IndexSizeBytes) + { + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + throw new GarnetException($"Unexpected stub size {outputSpan.Length} for RangeIndex read, expected {IndexSizeBytes}"); + } + + ref readonly var stub = ref ReadIndex(outputSpan); + + // Per-tree checkpoint barrier: one volatile read on hot path (no checkpoint = skipped). + if (checkpointInProgress + && WaitForTreeCheckpoint(key.ReadOnlySpan, ref output, indexSpan, ref sharedLockToken)) + { + goto Retry; + } + + if (stub.IsFlushed) + { + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + PromoteToTail(session, key); + goto Retry; + } + + if (stub.TreeHandle == nint.Zero) + { + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + + // Restore under exclusive lock to prevent concurrent restores. + // Pre-staging of data.bftree always happened earlier (PostCopyToTail-cold, + // RIPROMOTE PostCopyUpdater-cold, or OnRecoverySnapshotRead) so RestoreTree + // just opens data.bftree directly. + if (!RestoreTree(session, key, keyHash, ref input, indexSpan)) + { + status = GarnetStatus.NOTFOUND; + return default; + } + goto Retry; + } + + status = GarnetStatus.OK; + return new(in rangeIndexLocks, sharedLockToken); + } + + /// + /// Acquire an exclusive lock for the given key hash. + /// Used by TryDeleteRangeIndex to prevent concurrent data operations + /// while a BfTree is being freed. + /// + /// The key hash for lock striping. + /// An RAII lock holder. Dispose to release the exclusive lock. + internal ExclusiveRangeIndexLock AcquireExclusiveForDelete(long keyHash) + { + rangeIndexLocks.AcquireExclusiveLock(keyHash, out var exclusiveLockToken); + return new(in rangeIndexLocks, exclusiveLockToken); + } + + /// + /// Issue an RMW with RIPROMOTE to copy the stub from read-only to the mutable region. + /// CopyUpdater copies the stub bytes and clears the flushed flag. + /// Multiple threads may race here — only one wins the CAS; others see the new record. + /// + private static void PromoteToTail(StorageSession session, PinnedSpanByte key) + { + session.PromoteRangeIndexToTail(key); + } + + /// + /// Cold path: checks if this key's tree (activated or pending) is currently being snapshotted + /// for a checkpoint. If so, releases the shared lock, spin-waits for the snapshot to complete, + /// and signals the caller to retry the entire read operation. + /// + /// Lookup is by (XxHash128 → Guid) so it works uniformly for activated + /// (TreeHandle != 0) and pending (TreeHandle == 0) entries. Hot-path Guid derivation is gated + /// by the short-circuit so steady-state cost is one volatile + /// bool read. + /// + /// The raw key bytes. + /// The StringOutput from the read (passed for lifetime management). + /// The stub span (passed for lifetime management). + /// The shared lock token; released before waiting. + /// true if caller should retry the read; false if no wait was needed. + [MethodImpl(MethodImplOptions.NoInlining)] + private bool WaitForTreeCheckpoint(ReadOnlySpan key, ref StringOutput output, + Span indexSpan, ref int sharedLockToken) + { + var keyId = KeyId(key); + if (!liveIndexes.TryGetValue(keyId, out var treeEntry) + || Volatile.Read(ref treeEntry.SnapshotPending) == 0) + return false; + + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + while (Volatile.Read(ref treeEntry.SnapshotPending) != 0) + Thread.Yield(); + return true; + } + + /// + /// Restore a BfTree from its pre-staged CPR snapshot (data.bftree) and publish + /// the resulting native pointer into the stub via a RIRESTORE RMW. + /// + /// + /// Split design (essential for deadlock safety): the per-key X-lock is + /// held ONLY for the recovery + register step, then RELEASED before issuing the RMW. + /// Holding an RI X-lock across a Tsavorite RMW would risk firing a deferred OnFlush on + /// this thread (e.g., from the RMW's allocator drain), which would attempt the per-key + /// shared lock for the cold case and self-deadlock with the X-lock we still hold. + /// + /// Race between releasing X and issuing RIRESTORE RMW: a concurrent DEL could + /// fire, taking the X-lock, removing our entry from , and + /// queuing the bfTree for deferred disposal via storeEpoch.BumpCurrentEpoch. The + /// deferred dispose CANNOT execute until our thread's epoch advances — i.e., until the + /// RMW's Tsavorite session-suspend point. By that time the RMW has either: + /// + /// Succeeded — our nativePtr is now in the stub bytes; subsequent readers see + /// it and route through ReadRangeIndex's hot path which acquires the shared RI lock + /// before calling the native handle. The DEL's deferred dispose fires only after all + /// such readers complete (epoch barrier). + /// Returned NOTFOUND on a tombstoned key (RIRESTORE.NeedInitialUpdate=false). + /// Caller treats this as "key was deleted concurrently" → no re-restore loop. + /// + /// + /// Pre-staging of data.bftree always happened earlier: + /// + /// PostCopyToTail-cold (compaction/CopyReadsToTail with disk source). + /// RIPROMOTE PostCopyUpdater-cold (post-eviction or post-recovery first promote). + /// OnRecoverySnapshotRead (above-FUA-at-checkpoint stubs DURING recovery, since + /// the checkpoint snapshot file may be deleted post-recovery). + /// + /// + private bool RestoreTree( + StorageSession session, + PinnedSpanByte key, + long keyHash, + ref StringInput input, + Span indexSpan) + { + nint nativePtr; + rangeIndexLocks.AcquireExclusiveLock(keyHash, out var exclusiveLockToken); + try + { + // Re-read stub under exclusive lock to check if another thread already restored. + var output = StringOutput.FromPinnedSpan(indexSpan); + var readRes = session.Read_RangeIndex(key.ReadOnlySpan, ref input, ref output, ref session.stringBasicContext); + if (readRes != GarnetStatus.OK) + return false; + + var outputSpan = output.SpanByteAndMemory.IsSpanByte + ? output.SpanByteAndMemory.SpanByte.ReadOnlySpan + : output.SpanByteAndMemory.MemorySpan; + if (outputSpan.Length < IndexSizeBytes) + return false; + + ref readonly var stub = ref ReadIndex(outputSpan); + if (stub.TreeHandle != nint.Zero) + return true; // Another thread already restored + + var keySpan = key.ReadOnlySpan; + var keyId = KeyId(keySpan); + + // Race-resolved path: another stub for this key may already have a live tree + // (RestoreTree ran via a different addr). Reuse it instead of opening another. + if (liveIndexes.TryGetValue(keyId, out var existing) && existing?.Tree != null) + { + nativePtr = existing.Tree.NativePtr; + } + else + { + var hashPrefix = HashKeyToPrefix(keySpan); + var workingPath = LogDataPath(hashPrefix); + var scratchPath = LogScratchPath(hashPrefix); + + if (!File.Exists(workingPath)) + { + // Should not happen in normal flow — pre-staging guarantees data.bftree + // exists for any stub above FUA whose TreeHandle is 0. Assert in Debug; + // LogWarning + return false in Release so the affected key surfaces + // NOTFOUND rather than crashing the process. + Debug.Assert(false, $"RestoreTree: data.bftree missing for {hashPrefix} — pre-stage invariant violated"); + logger?.LogWarning("RestoreTree: data.bftree missing for {Hash} — pre-stage invariant violated", hashPrefix); + return false; + } + + var bfTree = BfTreeService.RecoverFromCprSnapshot( + workingPath, + scratchPath, + (StorageBackendType)stub.StorageBackend); + + RegisterIndex(bfTree, keyHash, keySpan); + + // Re-look-up to use the WINNER's pointer (handles concurrent restorer race). + if (!liveIndexes.TryGetValue(keyId, out var winner) || winner?.Tree == null) + { + // We disposed our duplicate via RegisterIndex's loser-disposal path AND + // someone removed the winner before we observed it — extremely rare race + // (concurrent DEL between RegisterIndex and TryGetValue). Bail out; the + // next reader will retry. + return false; + } + nativePtr = winner.Tree.NativePtr; + } + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed to restore BfTree from data.bftree"); + return false; + } + finally + { + rangeIndexLocks.ReleaseExclusiveLock(exclusiveLockToken); + } + + // RIRESTORE RMW WITHOUT the X-lock. Safe because OnFlush no longer takes any RI + // X-lock from any code path that may fire as a deferred epoch action. + // RIRESTORE.NeedInitialUpdate=false → if the key was concurrently DEL'd (tombstone), + // the RMW returns NOTFOUND and the caller does not loop. + try + { + session.RestoreRangeIndexStub(key, nativePtr); + return true; + } + catch (Exception ex) + { + logger?.LogError(ex, "RestoreRangeIndexStub RMW failed"); + return false; + } + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/RangeIndex/RangeIndexManager.Replication.cs b/libs/server/Resp/RangeIndex/RangeIndexManager.Replication.cs new file mode 100644 index 00000000000..43b0fea2984 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RangeIndexManager.Replication.cs @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Garnet.common; +using Garnet.server.BfTreeInterop; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Provides methods for managing range index operations, including replication and handling of AOF (Append-Only + /// File) entries for create, set, and delete operations. + /// + public sealed partial class RangeIndexManager + { + /// + /// Log RI.SET to AOF via direct enqueue (no synthetic RMW). + /// Skipped when is true (stored procedure logs as a unit). + /// + internal void ReplicateRangeIndexSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + GarnetAppendOnlyFile appendOnlyFile, long version, int sessionId, bool storedProcMode) + { + if (appendOnlyFile == null || storedProcMode) return; + + var replicateParseState = new SessionParseState(); + replicateParseState.InitializeWithArguments(field, value); + var input = new StringInput(RespCommand.RISET, ref replicateParseState); + input.header.flags |= RespInputFlags.Deterministic; + + appendOnlyFile.Log.Enqueue( + AofEntryType.StoreRMW, + version, + sessionId, + key.ReadOnlySpan, + ref input, + out _); + } + + /// + /// Log RI.DEL to AOF via direct enqueue (no synthetic RMW). + /// Skipped when is true (stored procedure logs as a unit). + /// + internal void ReplicateRangeIndexDel(PinnedSpanByte key, PinnedSpanByte field, + GarnetAppendOnlyFile appendOnlyFile, long version, int sessionId, bool storedProcMode) + { + if (appendOnlyFile == null || storedProcMode) return; + + var replicateParseState = new SessionParseState(); + replicateParseState.InitializeWithArgument(field); + var input = new StringInput(RespCommand.RIDEL, ref replicateParseState); + input.header.flags |= RespInputFlags.Deterministic; + + appendOnlyFile.Log.Enqueue( + AofEntryType.StoreRMW, + version, + sessionId, + key.ReadOnlySpan, + ref input, + out _); + } + + /// + /// Handle RI.CREATE replay from AOF. + /// + /// + /// The AOF entry contains the serialized stub bytes (including a stale TreeHandle + /// from the original process). This method: + /// + /// Extracts BfTree configuration from the stale stub. + /// Creates a fresh BfTree instance with a new native pointer. + /// Replaces the stale TreeHandle in the stub bytes with the new pointer. + /// Lets the normal RMW path (InitialUpdater) create the store record. + /// + /// If the key already exists (e.g., AOF replay of a duplicate RI.CREATE after + /// checkpoint recovery), the RMW returns InPlaceUpdated and the fresh + /// BfTree is disposed. + /// + /// The storage session for issuing the RMW. + /// The Garnet key being created. + /// The RMW input containing the stub bytes in parseState. + internal unsafe void HandleRangeIndexCreateReplay(StorageSession session, ReadOnlySpan key, ref StringInput input) + { + var stubSpan = input.parseState.GetArgSliceByRef(0).Span; + if (stubSpan.Length != IndexSizeBytes) + throw new GarnetException($"Corrupt RI.CREATE AOF entry: stub size {stubSpan.Length}, expected {IndexSizeBytes}"); + + ref var stub = ref Unsafe.As(ref MemoryMarshal.GetReference(stubSpan)); + + // Create a fresh BfTree from the config in the stub + BfTreeService bfTree; + try + { + bfTree = CreateBfTree( + (StorageBackendType)stub.StorageBackend, key, + stub.CacheSize, stub.MinRecordSize, stub.MaxRecordSize, + stub.MaxKeyLen, stub.LeafPageSize); + } + catch (Exception ex) + { + logger?.LogError(ex, "Failed to recreate BfTree during AOF replay"); + return; + } + + // Replace stale handle with fresh one in the stub bytes + stub.TreeHandle = bfTree.NativePtr; + stub.ResetFlags(); + + // Let the normal RMW path create the record from the updated stub bytes + var output = new StringOutput(); + var pinnedKey = PinnedSpanByte.FromPinnedSpan(key); + var status = session.stringBasicContext.RMW((FixedSpanByteKey)pinnedKey, ref input, ref output); + if (status.IsPending) + StorageSession.CompletePendingForSession(ref status, ref output, ref session.stringBasicContext); + + if (status.Record.Created) + { + var keyHash = session.stringBasicContext.GetKeyHash((FixedSpanByteKey)pinnedKey); + RegisterIndex(bfTree, keyHash, key); + } + else + { + bfTree.Dispose(); + } + } + + /// + /// Handle RI.SET replay from AOF. Acquires a shared lock, reads the stub to get + /// the live BfTree pointer, then performs the native insert. + /// + /// The storage session for reading the stub. + /// The Garnet key of the RangeIndex. + /// The RMW input containing field and value in parseState. + internal void HandleRangeIndexSetReplay(StorageSession session, ReadOnlySpan key, ref StringInput input) + { + var field = input.parseState.GetArgSliceByRef(0); + var value = input.parseState.GetArgSliceByRef(1); + + var pinnedKey = PinnedSpanByte.FromPinnedSpan(key); + var inputCopy = input; + inputCopy.arg1 = default; + Span stubSpan = stackalloc byte[IndexSizeBytes]; + + using (ReadRangeIndex(session, pinnedKey, ref inputCopy, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) return; + var treePtr = ReadIndex(stubSpan).TreeHandle; + if (treePtr == nint.Zero) return; + BfTreeService.InsertByPtr(treePtr, field, value); + } + } + + /// + /// Handle RI.DEL replay from AOF. Acquires a shared lock, reads the stub to get + /// the live BfTree pointer, then performs the native delete. + /// + /// The storage session for reading the stub. + /// The Garnet key of the RangeIndex. + /// The RMW input containing the field in parseState. + internal void HandleRangeIndexDelReplay(StorageSession session, ReadOnlySpan key, ref StringInput input) + { + var field = input.parseState.GetArgSliceByRef(0); + + var pinnedKey = PinnedSpanByte.FromPinnedSpan(key); + var inputCopy = input; + inputCopy.arg1 = default; + Span stubSpan = stackalloc byte[IndexSizeBytes]; + + using (ReadRangeIndex(session, pinnedKey, ref inputCopy, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) return; + var treePtr = ReadIndex(stubSpan).TreeHandle; + if (treePtr == nint.Zero) return; + BfTreeService.DeleteByPtr(treePtr, field); + } + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/RangeIndex/RangeIndexManager.cs b/libs/server/Resp/RangeIndex/RangeIndexManager.cs new file mode 100644 index 00000000000..ca25e8c6628 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RangeIndexManager.cs @@ -0,0 +1,895 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Concurrent; +using System.Globalization; +using System.IO; +using System.IO.Hashing; +using System.Runtime.CompilerServices; +using System.Threading; +using Garnet.common; +using Garnet.server.BfTreeInterop; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Manages the lifecycle of RangeIndex (BfTree) instances stored as fixed-size stubs + /// in Garnet's main store. + /// + /// Architecture: Each RangeIndex key in the store holds a + /// containing BfTree configuration metadata and a native pointer to the live BfTree instance. + /// The manager tracks all live BfTree instances (keyed by Guid = XxHash128 of the key bytes — + /// same scheme as the file-name prefix), coordinates checkpoint/flush/eviction snapshots, + /// and handles lazy restore from disk. + /// + /// File layout (flat under two roots): + /// + /// Log root ({LogDir ?? CheckpointDir ?? cwd}/Store/rangeindex/) — working + /// files (<hash>.data.bftree) and immutable per-flush snapshots + /// (<hash>.<addr:x16>.flush.bftree). + /// Checkpoint root ({CheckpointDir}/Store/checkpoints[_dbId]/cpr-checkpoints/<token>/rangeindex/) — + /// per-checkpoint snapshots (<hash>.bftree); deleted automatically when + /// Tsavorite removes the parent token directory. + /// + /// + /// Concurrency: Data operations (RI.SET, RI.GET, RI.DEL) acquire a shared lock + /// via ; lifecycle operations (DEL key, eviction, checkpoint) + /// acquire an exclusive lock. See RangeIndexManager.Locking.cs for details. + /// + /// liveIndexes access discipline: is consulted on the hot + /// path ONLY for checkpoint coordination (). All other + /// hot-path code uses the stub's TreeHandle directly. + /// + public sealed partial class RangeIndexManager : IDisposable + { + /// + /// RecordType discriminator for RangeIndex records in the unified store. + /// Stored in RecordDataHeader.RecordType to distinguish RI stubs + /// from normal strings (0) and VectorSet stubs. + /// + internal const byte RangeIndexRecordType = 2; + + /// Size of the RangeIndex stub in bytes. + internal const int IndexSizeBytes = RangeIndexStub.Size; + + /// Gets the number of live (registered) BfTree indexes (activated + pending). + internal int LiveIndexCount => liveIndexes.Count; + + /// + /// Tracks BfTree entries (activated + pending), keyed by = a Guid + /// derived from the key bytes via XxHash128. Same hash scheme used to derive the + /// on-disk filename prefix; collision risk is the same as for filename collisions + /// (cryptographically negligible). + /// + private readonly ConcurrentDictionary liveIndexes = new(); + + private readonly ILogger logger; + + /// + /// Tsavorite store epoch, used for deferred BfTree disposal via + /// storeEpoch.BumpCurrentEpoch(...). Ensures concurrent readers using a + /// TreeHandle are not affected by a concurrent DEL — the actual dispose runs + /// only after all current epoch holders move past. + /// + private readonly LightEpoch storeEpoch; + + /// + /// Log-tied root directory (without trailing separator). Holds the working file + /// (<hash>.data.bftree), CPR scratch files (<hash>.scratch.cpr), + /// and immutable per-flush snapshots (<hash>.<addr:x16>.flush.bftree). + /// + private readonly string riLogRoot; + + /// + /// Checkpoint-tied parent directory (the Tsavorite cpr-checkpoints/ directory). + /// Per-checkpoint snapshots live under {cprDir}/<token>/rangeindex/<hash>.bftree. + /// May be null if checkpointing is not enabled. + /// + private readonly string cprDir; + + /// + /// Global checkpoint barrier. When non-zero, a checkpoint is snapshotting trees. + /// RI operations check this first (one volatile read on hot path); if set, they + /// look up by and check the per-tree + /// flag. + /// + private volatile bool checkpointInProgress; + + /// + /// Checkpoint token from the last recovery. Used by + /// to locate the correct checkpoint + /// snapshot file for above-FUA stubs recovered from snapshot. + /// + private Guid recoveredCheckpointToken; + + /// + /// Per-tree entry in . Class (not struct) so the + /// field can be updated in-place via Volatile.Write. + /// + internal sealed class TreeEntry + { + /// The managed BfTree wrapper owning the native tree pointer. + /// null for a "pending" entry — data.bftree on disk has correct content + /// but no native BfTree has been opened yet (awaiting RestoreTree activation). + public BfTreeService Tree; + + /// Hash of the Garnet key, used for lock striping. + public readonly long KeyHash; + + /// 32-character lowercase-hex prefix derived from the key (XxHash128 → Guid("N")). + /// Used to construct file paths under both roots. + public readonly string HashPrefix; + + /// + /// 1 while this tree is being snapshotted for checkpoint, 0 otherwise. + /// Set at time, cleared after snapshot completes. + /// RI data operations spin-wait on this flag when is set. + /// + public int SnapshotPending; + + /// + /// Per-tree snapshot serialization atomic. 0 = idle; 1 = a snapshot is in flight. + /// Both and + /// claim this before calling + /// cpr_snapshot so the two callers do not race for bftree's internal + /// snapshot_in_progress flag (which would no-op one of them silently). + /// + public int SnapshotInProgress; + + /// Try to claim the per-tree snapshot atomic. Returns true if claimed. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryClaimSnapshot() + => Interlocked.CompareExchange(ref SnapshotInProgress, 1, 0) == 0; + + /// Release the per-tree snapshot atomic. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReleaseSnapshot() + => Volatile.Write(ref SnapshotInProgress, 0); + + /// Spin-wait until the per-tree snapshot atomic is released. + public void WaitForSnapshot() + { + while (Volatile.Read(ref SnapshotInProgress) != 0) + Thread.Yield(); + } + + public TreeEntry(BfTreeService tree, long keyHash, string hashPrefix) + { + Tree = tree; + KeyHash = keyHash; + HashPrefix = hashPrefix; + } + } + + /// + /// Creates a new . Constructed only when range index + /// is enabled in server options; GarnetServer passes null in place of + /// a manager when the feature is disabled, so this constructor never runs in the + /// disabled case. + /// + /// Log-tied root directory for working/flush files (e.g. + /// {LogDir ?? CheckpointDir ?? cwd}/Store/rangeindex). MUST be a non-empty path + /// and the directory MUST be creatable — the constructor throws otherwise so + /// misconfiguration (missing permissions, bad path, etc.) surfaces at server startup + /// rather than at first use. + /// Tsavorite cpr-checkpoints/ directory; per-checkpoint snapshots + /// live under {cprDir}/<token>/rangeindex/. May be null if no checkpointing. + /// The store's ; used to defer native + /// BfTree.Dispose + file deletion past any in-flight reader observing the + /// stub's TreeHandle. May be null in unit-test scenarios with no concurrent + /// readers; in that case disposal is performed synchronously. + /// Optional logger. + /// Thrown when is + /// null or empty. + /// Thrown when the riLogRoot directory cannot be + /// created (e.g., insufficient permissions). Wraps the underlying exception. + public RangeIndexManager(string riLogRoot, string cprDir = null, + LightEpoch storeEpoch = null, ILogger logger = null) + { + if (string.IsNullOrEmpty(riLogRoot)) + throw new ArgumentException( + "RangeIndexManager: riLogRoot is required.", + nameof(riLogRoot)); + + this.riLogRoot = riLogRoot; + this.cprDir = cprDir; + this.storeEpoch = storeEpoch; + this.logger = logger; + rangeIndexLocks = new ReadOptimizedLock(Environment.ProcessorCount); + + try + { + Directory.CreateDirectory(riLogRoot); + } + catch (Exception ex) + { + logger?.LogError(ex, "RangeIndexManager: failed to create riLogRoot {Path}", riLogRoot); + throw new IOException( + $"RangeIndexManager: failed to create riLogRoot '{riLogRoot}'. " + + "Check that the parent directory exists and the process has write permissions.", + ex); + } + } + + /// + /// Compute the unambiguous identity of a RangeIndex key as a 128-bit Guid. + /// Same scheme used to derive the on-disk filename prefix + /// (). The cryptographically negligible collision risk + /// is the same risk we accept for filename collisions. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Guid KeyId(ReadOnlySpan keyBytes) => new(XxHash128.Hash(keyBytes)); + + /// + /// Hash key bytes to a 32-character lowercase-hex filename prefix using XxHash128. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string HashKeyToPrefix(ReadOnlySpan keyBytes) + => new Guid(XxHash128.Hash(keyBytes)).ToString("N"); + + // -- Path helpers -- + + /// {logRoot}/<hash>.data.bftree + internal string LogDataPath(string hashPrefix) + => Path.Combine(riLogRoot ?? string.Empty, hashPrefix + ".data.bftree"); + + /// {logRoot}/<hash>.scratch.cpr — bftree CPR snapshot scratch file (overwritten each cpr_snapshot). + internal string LogScratchPath(string hashPrefix) + => Path.Combine(riLogRoot ?? string.Empty, hashPrefix + ".scratch.cpr"); + + /// {logRoot}/<hash>.<addr:x16>.flush.bftree + internal string LogFlushPath(string hashPrefix, long logicalAddress) + => Path.Combine(riLogRoot ?? string.Empty, $"{hashPrefix}.{logicalAddress:x16}.flush.bftree"); + + /// {cprDir}/<token>/rangeindex/<hash>.bftree + internal string CheckpointSnapshotPath(string hashPrefix, Guid checkpointToken) + => Path.Combine(cprDir ?? string.Empty, checkpointToken.ToString(), "rangeindex", hashPrefix + ".bftree"); + + /// The directory holding per-checkpoint RI snapshots for a given token. + internal string CheckpointSnapshotDir(Guid checkpointToken) + => Path.Combine(cprDir ?? string.Empty, checkpointToken.ToString(), "rangeindex"); + + // -- Convenience helpers used outside this class (RangeIndexOps via raw key) -- + internal string LogDataPathFor(ReadOnlySpan keyBytes) => LogDataPath(HashKeyToPrefix(keyBytes)); + internal string LogScratchPathFor(ReadOnlySpan keyBytes) => LogScratchPath(HashKeyToPrefix(keyBytes)); + + /// + /// Creates a new BfTree instance via the native interop layer. + /// For disk-backed trees, derives the file path deterministically from the key bytes. + /// + internal BfTreeService CreateBfTree( + StorageBackendType storageBackend, + ReadOnlySpan keyBytes, + ulong cacheSize, + uint minRecordSize, + uint maxRecordSize, + uint maxKeyLen, + uint leafPageSize) + { + var hashPrefix = HashKeyToPrefix(keyBytes); + string filePath = null; + string snapshotFilePath = null; + + if (storageBackend == StorageBackendType.Disk) + { + filePath = LogDataPath(hashPrefix); + } + + // Configure the bftree's CPR snapshot scratch path. cpr_snapshot writes here; + // OnFlush / SnapshotAllTreesForCheckpoint File.Move scratch -> final destination. + // Required for both backends; leave null only if riLogRoot is unset (test scenarios). + // riLogRoot itself was already created in the constructor. + if (!string.IsNullOrEmpty(riLogRoot)) + { + snapshotFilePath = LogScratchPath(hashPrefix); + } + + return new BfTreeService( + storageBackend: storageBackend, + filePath: filePath, + snapshotFilePath: snapshotFilePath, + cbSizeByte: cacheSize, + cbMinRecordSize: minRecordSize, + cbMaxRecordSize: maxRecordSize, + cbMaxKeyLen: maxKeyLen, + leafPageSize: leafPageSize); + } + + /// + /// Compute the leaf page size from the max record size when not explicitly specified. + /// + /// The configured maximum record size in bytes. + /// + /// A power-of-two page size: + /// + /// For ≤ 2 KB → 4 KB + /// For larger values → 2.5× record size rounded to next power of 2, capped at 32 KB + /// + /// + internal static uint ComputeLeafPageSize(uint maxRecordSize) + { + if (maxRecordSize <= 2048) + return 4096; + + // 2.5x, capped at 32KB + var target = (uint)(maxRecordSize * 2.5); + if (target > 32768) + target = 32768; + + // Round up to next power of 2 + return RoundUpToPowerOf2(target); + } + + /// + /// Rounds up to the next power of 2 using the standard bit-manipulation algorithm. + /// + private static uint RoundUpToPowerOf2(uint v) + { + v--; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v++; + return v; + } + + /// + /// Register a BfTreeService in the live index dictionary after successful creation or restore. + /// Cold path — called once per RI.CREATE or activation (RestoreTree). + /// + internal void RegisterIndex(BfTreeService bfTree, long keyHash, ReadOnlySpan keyBytes) + { + var hashPrefix = HashKeyToPrefix(keyBytes); + var keyId = KeyId(keyBytes); + var newEntry = new TreeEntry(bfTree, keyHash, hashPrefix); + if (liveIndexes.TryAdd(keyId, newEntry)) + return; // First registration for this key — done. + + // A prior entry exists. Per caller invariants: + // - RICREATE: invoked only when the underlying RMW reports Record.Created=true, + // so a prior entry cannot exist for the same key on the create path. + // - RestoreTree: holds the per-key rangeIndexLocks X-lock for the duration of + // RegisterIndex, so concurrent RegisterIndex on the same key is impossible. + // The only legitimate prior entry is a pending one (Tree==null) registered + // earlier by PreStageAndRegisterPending; we activate it in place. + // CompareExchange makes the activation atomic so even a future caller-invariant + // violation cannot result in two threads both believing they activated the entry + // (one would observe a non-null prior in CompareExchange and dispose its duplicate). + if (liveIndexes.TryGetValue(keyId, out var existing)) + { + var prior = Interlocked.CompareExchange(ref existing.Tree, bfTree, null); + if (prior is null) + return; // We activated the pending entry. + } + + logger?.LogError( + "RegisterIndex: liveIndexes entry for {Hash} is unexpectedly already activated. " + + "Caller invariant violated (RICREATE should fire only on Record.Created=true; " + + "RestoreTree must hold the per-key rangeIndexLocks X-lock). Disposing duplicate.", + hashPrefix); + DisposeBfTreeDeferred(bfTree, "duplicate-register"); + } + + /// + /// Defer-dispose a BfTree past any reader that may still be using its TreeHandle. + /// Falls back to synchronous dispose when no is wired + /// (unit-test scenarios with no concurrent readers). + /// + private void DisposeBfTreeDeferred(BfTreeService bfTree, string reason) + { + if (storeEpoch != null) + { + var loser = bfTree; + storeEpoch.BumpCurrentEpoch(() => + { + try { loser.Dispose(); } + catch (Exception ex) { logger?.LogWarning(ex, "Deferred dispose failed: {Reason}", reason); } + }); + } + else + { + try { bfTree.Dispose(); } + catch (Exception ex) { logger?.LogWarning(ex, "Synchronous dispose failed: {Reason}", reason); } + } + } + + /// + /// Register a "pending" entry: data.bftree on disk has correct content (just pre-staged) but + /// no native BfTree has been opened yet. Activated by a later call + /// from RestoreTree. + /// + internal void RegisterPending(ReadOnlySpan keyBytes, long keyHash) + { + var hashPrefix = HashKeyToPrefix(keyBytes); + var keyId = KeyId(keyBytes); + var pending = new TreeEntry(tree: null, keyHash, hashPrefix); + // Add only if no entry exists; if one exists (activated or pending), leave it alone. + _ = liveIndexes.TryAdd(keyId, pending); + } + + /// + /// Unregister the entry for a key. Disposes the native tree if present and the entry + /// is owned by this manager. The caller must already hold the exclusive lock for this key. + /// + /// true if an entry was found and removed. + internal bool UnregisterIndex(ReadOnlySpan keyBytes) + { + var keyId = KeyId(keyBytes); + if (liveIndexes.TryRemove(keyId, out var entry)) + { + try { entry.Tree?.Dispose(); } + catch (Exception ex) { logger?.LogWarning(ex, "Failed to dispose BfTree on unregister"); } + return true; + } + return false; + } + + /// + public void Dispose() + { + foreach (var kvp in liveIndexes) + { + try { kvp.Value.Tree?.Dispose(); } + catch (Exception ex) { logger?.LogWarning(ex, "Failed to dispose BfTree {Hash}", kvp.Value.HashPrefix); } + } + liveIndexes.Clear(); + } + + /// + /// Atomically pre-stage data.bftree from <srcAddr:x16>.flush.bftree + /// and register a pending entry in so a subsequent checkpoint + /// will capture it via . + /// + /// Called from RIPROMOTE PostCopyUpdater (cold case: src.TreeHandle == 0) and + /// from PostCopyToTail (compaction with disk source). + /// + /// Concurrency: takes the per-key EXCLUSIVE rangeIndex lock for the duration of the + /// file copy. This is required because CASRecordIntoChain unseals dst immediately + /// on CAS-success (in Helpers.cs.CASRecordIntoChain), so by the time this trigger + /// fires, concurrent readers can already observe dst with TreeHandle == 0 and + /// invoke RestoreTree, which opens data.bftree under its own per-key + /// exclusive lock. Holding the exclusive lock here blocks RestoreTree until the + /// file is fully written, preventing it from observing a partial data.bftree. + /// + /// A direct File.Copy(overwrite: true) is sufficient under this lock — the + /// exclusive lock serializes against any reader that would open data.bftree, and + /// against other concurrent PreStageAndRegisterPending calls for the same key. + /// A crash mid-copy is self-healing: post-recovery either OnRecoverySnapshotRead + /// (above-FUA stub) or the next RIPROMOTE-cold (IsFlushed=true stub) re-pre-stages and + /// overwrites any partial file before RestoreTree can observe it. + /// + internal void PreStageAndRegisterPending(ReadOnlySpan keyBytes, long srcFlushAddress) + { + if (string.IsNullOrEmpty(riLogRoot)) + return; + + var hashPrefix = HashKeyToPrefix(keyBytes); + var snapshotPath = LogFlushPath(hashPrefix, srcFlushAddress); + if (!File.Exists(snapshotPath)) + { + // Per Invariant 2 (Per-flush snapshot invariant), the per-flush file must exist + // for any IsFlushed=true stub at addr >= BeginAddress. If it's missing here, the + // invariant has been violated (likely a race with a concurrent OnTruncate, or + // external file deletion). Log loudly as ERROR — recovering from any other source + // file would risk restoring the wrong tree version. Leave the destination record + // with TreeHandle=0 and no pending entry; the next RestoreTree will return NOTFOUND + // for the affected key, surfacing the data loss explicitly rather than silently + // restoring incorrect data. + logger?.LogError("PreStageAndRegisterPending: invariant violation — source flush file missing: {Path}. " + + "The destination record at the tail will have no pending entry; subsequent RestoreTree " + + "will return NOTFOUND for the affected key. NOT falling back to any other flush file " + + "to avoid restoring an incorrect tree version.", snapshotPath); + return; + } + + var dataPath = LogDataPath(hashPrefix); + var keyHash = GarnetKeyComparer.StaticGetHashCode64((FixedSpanByteKey)PinnedSpanByte.FromPinnedSpan(keyBytes)); + + // Acquire the per-key exclusive lock for the duration of the file copy AND the + // pending-entry registration. This blocks any concurrent RestoreTree (which also + // takes the exclusive lock) from observing a partial data.bftree. + // riLogRoot was created in the constructor. + rangeIndexLocks.AcquireExclusiveLock(keyHash, out var lockToken); + try + { + File.Copy(snapshotPath, dataPath, overwrite: true); + + var keyId = KeyId(keyBytes); + _ = liveIndexes.TryAdd(keyId, new TreeEntry(tree: null, keyHash, hashPrefix)); + } + catch (Exception ex) + { + logger?.LogError(ex, "PreStageAndRegisterPending: copy/register failed for {Hash}; " + + "destination record will have no pending entry; subsequent RestoreTree will return NOTFOUND", + hashPrefix); + } + finally + { + rangeIndexLocks.ReleaseExclusiveLock(lockToken); + } + } + + /// + /// Pre-stage data.bftree from cpr-checkpoints/<recoveredCheckpointToken>/rangeindex/<hash>.bftree + /// during recovery. Called from OnRecoverySnapshotRead for above-FUA-at-checkpoint stubs + /// (snapshot file may be deleted post-recovery, so this MUST run during recovery). + /// Registers a pending entry so any first checkpoint after recovery captures the key correctly. + /// + /// Recovery is single-threaded with no concurrent readers, and a crash mid-copy is + /// self-healing — the next recovery attempt re-fires OnRecoverySnapshotRead for + /// the same stub and fully overwrites any partial file before any RestoreTree + /// can observe it. + /// + /// Thrown when this method is called without + /// the recovery state required to locate checkpoint snapshots + /// ( is empty or is Guid.Empty). + /// This indicates that OnRecoverySnapshotRead fired without + /// OnRecovery(token) having captured the recovered checkpoint token first — + /// a wiring bug that would otherwise silently lose the recovered tree. + internal void RebuildFromSnapshotIfPending(ReadOnlySpan keyBytes) + { + if (string.IsNullOrEmpty(cprDir) || recoveredCheckpointToken == Guid.Empty) + throw new InvalidOperationException( + "RebuildFromSnapshotIfPending: recovery state missing " + + $"(cprDir empty: {string.IsNullOrEmpty(cprDir)}, recoveredCheckpointToken empty: {recoveredCheckpointToken == Guid.Empty}). " + + "This indicates OnRecoverySnapshotRead fired without OnRecovery(token) " + + "having captured the recovered checkpoint token. The recovered RangeIndex tree would " + + "otherwise be silently lost."); + + var hashPrefix = HashKeyToPrefix(keyBytes); + var snapshotPath = CheckpointSnapshotPath(hashPrefix, recoveredCheckpointToken); + if (!File.Exists(snapshotPath)) + { + // Below-FUA stubs have no checkpoint snapshot; RIPROMOTE PostCopyUpdater handles + // them lazily via the per-flush snapshot file on first access. NOT an error. + logger?.LogDebug("OnRecoverySnapshotRead: snapshot absent for {Hash} — RIPROMOTE will handle lazily", hashPrefix); + return; + } + + var dataPath = LogDataPath(hashPrefix); + + try + { + File.Copy(snapshotPath, dataPath, overwrite: true); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "RebuildFromSnapshotIfPending: copy failed for {Hash}", hashPrefix); + return; + } + + var keyHash = GarnetKeyComparer.StaticGetHashCode64((FixedSpanByteKey)PinnedSpanByte.FromPinnedSpan(keyBytes)); + var keyId = KeyId(keyBytes); + _ = liveIndexes.TryAdd(keyId, new TreeEntry(tree: null, keyHash, hashPrefix)); + } + + /// + /// Store the recovered checkpoint token for use by + /// . + /// + internal void SetRecoveredCheckpointToken(Guid token) => recoveredCheckpointToken = token; + + /// + /// Log an OnFlush invariant violation: was zero + /// (no live tree) but the working file data.bftree was missing. Per Invariant 5 + /// (Pending entry invariant), every above-FUA stub with TreeHandle=0 must have a pending + /// entry in and a pre-staged data.bftree on disk; if + /// data.bftree is missing, something has corrupted that invariant (e.g. a prior + /// pre-stage failure, or external file deletion). The caller () + /// must NOT set in this case — that would put the + /// record into an unrestorable state where RIPROMOTE-cold would also fail and the key + /// would be permanently broken. + /// + internal void LogOnFlushInvariantViolation(string hashPrefix, long logicalAddress) + { + logger?.LogError("OnFlush: invariant violation — TreeHandle=0 but data.bftree missing for {Hash} at addr 0x{Addr:x16}; skipping IsFlushed (record will route through RestoreTree which will return NOTFOUND for the affected key)", + hashPrefix, logicalAddress); + } + + /// + /// Snapshot a BfTree's current contents to its per-flush snapshot file. Called from + /// when a page transitions to read-only. + /// + /// Live case (stub.TreeHandle != 0): take a CPR snapshot via the + /// native handle. CPR is concurrent-safe with workers (no per-key X-lock needed). + /// Per-tree atomic serializes against + /// concurrent for the same tree (otherwise + /// bftree's internal snapshot_in_progress would no-op one of them). + /// + /// Cold case (stub.TreeHandle == 0): the stub was just CAS'd at the + /// tail by PostCopyToTail-cold or RIPROMOTE-PostCopyUpdater-cold; PreStage already + /// copied <srcAddr>.flush.bftree → data.bftree but RestoreTree hasn't + /// activated a live tree yet. ANOTHER stub for the same key may have a live tree in + /// (RestoreTree ran against a different addr's stub) — workers + /// using that tree would write to data.bftree concurrently, making + /// File.Copy(data.bftree) unsafe. So: + /// + /// Acquire per-key SHARED RI lock — blocks RestoreTree's X-lock from registering + /// a new tree during our copy. Deadlock-free: S-vs-S compatible with hot path; no path + /// holds an RI X-lock across a Tsavorite op that fires deferred OnFlush. + /// If has a live tree under another stub → use + /// CPR snapshot (concurrent-safe with workers). + /// Else → safe to File.Copy(data.bftree → flushPath). + /// + /// + /// Sets on the in-memory stub on success so + /// the next data operation routes through + /// or RIPROMOTE PostCopyUpdater to re-anchor the tree at the tail. + /// + /// The raw key bytes (used for hash prefix + lock acquisition). + /// The store value span containing the stub. + /// The logical address of the record being flushed. + internal void SnapshotTreeForFlush(ReadOnlySpan key, Span valueSpan, long logicalAddress) + { + ref readonly var stub = ref ReadIndex(valueSpan); + + // Stale source whose ownership was transferred to a newer record at the tail: no-op. + if (stub.IsTransferred) + return; + + // Need riLogRoot for any disk artifact (both backends use it as staging directory). + if (string.IsNullOrEmpty(riLogRoot)) + return; + + var hashPrefix = HashKeyToPrefix(key); + var dataPath = LogDataPath(hashPrefix); + var scratchPath = LogScratchPath(hashPrefix); + var flushPath = LogFlushPath(hashPrefix, logicalAddress); + var keyId = KeyId(key); + + if (stub.TreeHandle != nint.Zero) + { + // Live case: stub directly references a live tree. CPR snapshot via the handle. + if (!liveIndexes.TryGetValue(keyId, out var entry) || entry?.Tree == null) + { + // Edge case: stub.TreeHandle points at a tree no longer in liveIndexes + // (DEL deferred-disposed it but stub bytes weren't updated). Treat as cold. + SnapshotForFlushCold(key, hashPrefix, dataPath, flushPath, valueSpan, logicalAddress); + return; + } + SnapshotForFlushViaCpr(entry, scratchPath, flushPath); + SetFlushedFlag(valueSpan); + } + else + { + SnapshotForFlushCold(key, hashPrefix, dataPath, flushPath, valueSpan, logicalAddress); + } + } + + /// + /// Take a CPR snapshot via the live tree's native handle and copy the produced scratch + /// file to the addr-tagged per-flush destination. We copy rather than move because + /// bftree's internal VFS keeps a file descriptor for the configured snapshot path; a move + /// would not invalidate the descriptor and the next cpr_snapshot would write through the + /// stale FD into the moved-away file (overwriting our finalized flush snapshot). + /// Per-tree atomic serializes against concurrent + /// on the same tree. + /// + private void SnapshotForFlushViaCpr(TreeEntry entry, string scratchPath, string flushPath) + { + if (!entry.TryClaimSnapshot()) + { + // Concurrent SnapshotAllTreesForCheckpoint owns the snapshot. Wait for it, + // then copy the produced scratch file to our addr-tagged location. + entry.WaitForSnapshot(); + File.Copy(scratchPath, flushPath, overwrite: false); + return; + } + try + { + BfTreeService.CprSnapshotByPtr(entry.Tree.NativePtr); + File.Copy(scratchPath, flushPath, overwrite: false); + } + finally + { + entry.ReleaseSnapshot(); + } + } + + /// + /// OnFlush cold-case: stub.TreeHandle == 0. The pre-staged data.bftree is the + /// only candidate source for capturing this flush. We must serialize against any + /// concurrent RestoreTree (X-lock) that could activate a tree mid-copy and start + /// writing to data.bftree from a worker thread. Use SHARED RI lock — deadlock-free + /// because no firing-thread holds an X-lock across a Tsavorite op that fires deferred + /// OnFlush (RestoreTree releases X before its RMW; PreStage / DisposeTreeUnderLock + /// don't issue Tsavorite ops while holding X; OnFlush itself doesn't take X). + /// + private void SnapshotForFlushCold(ReadOnlySpan key, string hashPrefix, + string dataPath, string flushPath, Span valueSpan, long logicalAddress) + { + var keyHash = GarnetKeyComparer.StaticGetHashCode64((FixedSpanByteKey)PinnedSpanByte.FromPinnedSpan(key)); + rangeIndexLocks.AcquireSharedLock(keyHash, out var sharedLockToken); + try + { + // Re-check: a tree may have become live under a different stub for this key + // (RestoreTree completed before we acquired the shared lock). + if (liveIndexes.TryGetValue(KeyId(key), out var entry) && entry?.Tree != null) + { + var scratchPath = LogScratchPath(hashPrefix); + SnapshotForFlushViaCpr(entry, scratchPath, flushPath); + SetFlushedFlag(valueSpan); + return; + } + + // No live tree exists for this key (S-lock blocks RestoreTree from activating + // one mid-copy). data.bftree is stable — no concurrent writer. + if (!File.Exists(dataPath)) + { + LogOnFlushInvariantViolation(hashPrefix, logicalAddress); + return; // do NOT set IsFlushed + } + File.Copy(dataPath, flushPath, overwrite: false); + SetFlushedFlag(valueSpan); + } + finally + { + rangeIndexLocks.ReleaseSharedLock(sharedLockToken); + } + } + + /// + /// Set the checkpoint barrier. Called at version shift (PREPARE → IN_PROGRESS). + /// Marks all entries (activated + pending) as snapshot-pending, then sets the global flag. + /// + internal void SetCheckpointBarrier(Guid checkpointToken) + { + foreach (var kvp in liveIndexes) + Volatile.Write(ref kvp.Value.SnapshotPending, 1); + checkpointInProgress = true; + } + + /// + /// Clear the checkpoint barrier. Called after snapshot completes at WAIT_FLUSH. + /// + internal void ClearCheckpointBarrier() + { + checkpointInProgress = false; + foreach (var kvp in liveIndexes) + Volatile.Write(ref kvp.Value.SnapshotPending, 0); + } + + /// + /// Snapshot all live BfTrees for a checkpoint. Called at FlushBegin. + /// + /// For each entry with set: + /// + /// Activated entries (Tree != null) → take CPR snapshot via the tree handle and + /// File.Move the produced scratch file to the checkpoint destination. + /// Pending entries (Tree == null) → File.Copy(data.bftree) (no live tree + /// to snapshot from; data.bftree was pre-staged by PreStage). + /// + /// + /// Uses the per-tree atomic to serialize + /// against concurrent for the same tree. Per-key + /// X-lock is NOT taken here — that lock would deadlock if any deferred OnFlush fired + /// on the checkpoint thread while it held S-locks on hot-path readers' shards. + /// + /// Memory-backed trees are also captured via CPR snapshot (bftree 0.5.0 supports + /// CPR for memory-backed trees uniformly with disk-backed). + /// + /// Failure is fatal — the exception propagates to the state machine driver. + /// + internal void SnapshotAllTreesForCheckpoint(Guid checkpointToken) + { + try + { + if (string.IsNullOrEmpty(cprDir)) + return; + + var snapshotDir = CheckpointSnapshotDir(checkpointToken); + + foreach (var kvp in liveIndexes) + { + var entry = kvp.Value; + + if (Volatile.Read(ref entry.SnapshotPending) == 0) + continue; + + try + { + Directory.CreateDirectory(snapshotDir); + var checkpointPath = CheckpointSnapshotPath(entry.HashPrefix, checkpointToken); + var scratchPath = LogScratchPath(entry.HashPrefix); + + if (entry.Tree is not null) + { + // Per-tree atomic serializes against concurrent OnFlush on the same + // tree (which would also call cpr_snapshot via the same handle — + // bftree's internal snapshot_in_progress would otherwise no-op one). + while (!entry.TryClaimSnapshot()) Thread.Yield(); + try + { + BfTreeService.CprSnapshotByPtr(entry.Tree.NativePtr); + // Copy scratch -> token-tagged checkpoint destination. Copy rather + // than move because bftree's internal VFS keeps a file descriptor + // for the scratch path (see SnapshotForFlushViaCpr docs). + File.Copy(scratchPath, checkpointPath, overwrite: true); + } + finally + { + entry.ReleaseSnapshot(); + } + } + else + { + // Pending entry: data.bftree was pre-staged from the source flush + // file by PreStageAndRegisterPending. data.bftree is the only + // correct source. No live tree means no concurrent writer. + var dataPath = LogDataPath(entry.HashPrefix); + if (File.Exists(dataPath)) + File.Copy(dataPath, checkpointPath, overwrite: true); + else + logger?.LogWarning("SnapshotAllTreesForCheckpoint: data.bftree missing for pending {Hash}", entry.HashPrefix); + } + } + finally + { + Volatile.Write(ref entry.SnapshotPending, 0); + } + } + } + finally + { + ClearCheckpointBarrier(); + } + } + + /// + /// On log truncation, delete per-flush snapshot files in the log root whose address is + /// strictly less than . + /// Per-checkpoint snapshots are NOT touched here — Tsavorite's checkpoint manager + /// removes them when it deletes the parent token directory. + /// + /// Per-flush files are LOG-tied (their lifetime tracks log addresses), not + /// checkpoint-tied — they are safe to delete once Tsavorite's BeginAddress passes their + /// address. This cleanup is unconditional and independent of cluster mode or any + /// checkpoint-retention policy. + /// + internal void OnTruncateImpl(long newBeginAddress) + { + if (string.IsNullOrEmpty(riLogRoot)) + return; + if (!Directory.Exists(riLogRoot)) + return; + + try + { + foreach (var path in Directory.EnumerateFiles(riLogRoot)) + { + var name = Path.GetFileName(path); + + if (!name.EndsWith(".flush.bftree", StringComparison.Ordinal)) + continue; + + // Pattern: ..flush.bftree + // hash is 32 hex chars, then '.', then 16 hex chars (addr), then ".flush.bftree". + if (name.Length != 32 + 1 + 16 + ".flush.bftree".Length) + continue; + + var addrSegment = name.AsSpan(33, 16); + if (!long.TryParse(addrSegment, NumberStyles.HexNumber, CultureInfo.InvariantCulture, out var addr)) + continue; + + if (addr < newBeginAddress) + TryDelete(path); + } + } + catch (Exception ex) + { + logger?.LogWarning(ex, "OnTruncate: enumeration failed under {Root}", riLogRoot); + } + + void TryDelete(string p) + { + try { File.Delete(p); } + catch (Exception ex) { logger?.LogWarning(ex, "OnTruncate: failed to delete {Path}", p); } + } + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/RangeIndex/RangeIndexResult.cs b/libs/server/Resp/RangeIndex/RangeIndexResult.cs new file mode 100644 index 00000000000..fe01b4bc640 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RangeIndexResult.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Garnet.server +{ + /// + /// Result codes returned by RangeIndex storage-layer operations. + /// These codes are mapped to RESP responses by the network handler + /// (). + /// + public enum RangeIndexResult + { + /// Operation succeeded. + OK, + /// Key or field was not found in the BfTree. + NotFound, + /// Key or field was found but has been logically deleted. + Deleted, + /// Invalid key (e.g. exceeds the configured MAXKEYLEN). + InvalidKey, + /// Operation failed with an error (see accompanying error message). + Error, + /// Operation not supported for MEMORY-mode indexes. + MemoryModeNotSupported, + } +} \ No newline at end of file diff --git a/libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs b/libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs new file mode 100644 index 00000000000..2b3b8baf0c4 --- /dev/null +++ b/libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs @@ -0,0 +1,650 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.common; +using Garnet.server.BfTreeInterop; + +namespace Garnet.server +{ + /// + /// RESP network handler for RangeIndex commands (RI.CREATE, RI.SET, RI.GET, RI.DEL, + /// RI.SCAN, RI.RANGE, RI.EXISTS, RI.CONFIG, RI.METRICS). + /// + /// Each method parses RESP arguments from the network buffer, delegates to the + /// corresponding method, and writes the RESP response. + /// + internal sealed unsafe partial class RespServerSession + { + /// + /// Handles the RI.CREATE command. + /// Syntax: RI.CREATE key [MEMORY | DISK] [CACHESIZE n] [MINRECORD n] [MAXRECORD n] [MAXKEYLEN n] [PAGESIZE n] + /// + /// + /// All numeric parameters must be greater than zero. MINRECORD must not exceed MAXRECORD. + /// If PAGESIZE is not specified (or 0), it is auto-computed from MAXRECORD via + /// . + /// Duplicate RI.CREATE on the same key returns an error ("ERR index already exists"). + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRICREATE(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count < 1) + { + return AbortWithWrongNumberOfArguments("RI.CREATE"); + } + + var key = parseState.GetArgSliceByRef(0); + + // Defaults + byte storageBackend = 0; // Disk by default + long cacheSize = 16 * 1024 * 1024; // 16 MiB + long minRecordSize = 64; + long maxRecordSize = 1024; + long maxKeyLen = 128; + long leafPageSize = 0; // 0 = auto-compute from maxRecordSize + + // Parse optional keyword arguments + var idx = 1; + while (idx < parseState.Count) + { + var arg = parseState.GetArgSliceByRef(idx).ReadOnlySpan; + + if (arg.EqualsUpperCaseSpanIgnoringCase("MEMORY"u8)) + { + storageBackend = 1; + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("DISK"u8)) + { + storageBackend = 0; + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("CACHESIZE"u8)) + { + idx++; + if (idx >= parseState.Count) + { + while (!RespWriteUtils.TryWriteError("ERR CACHESIZE requires a value"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + cacheSize = parseState.GetLong(idx); + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("MINRECORD"u8)) + { + idx++; + if (idx >= parseState.Count) + { + while (!RespWriteUtils.TryWriteError("ERR MINRECORD requires a value"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + minRecordSize = parseState.GetLong(idx); + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("MAXRECORD"u8)) + { + idx++; + if (idx >= parseState.Count) + { + while (!RespWriteUtils.TryWriteError("ERR MAXRECORD requires a value"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + maxRecordSize = parseState.GetLong(idx); + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("MAXKEYLEN"u8)) + { + idx++; + if (idx >= parseState.Count) + { + while (!RespWriteUtils.TryWriteError("ERR MAXKEYLEN requires a value"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + maxKeyLen = parseState.GetLong(idx); + idx++; + } + else if (arg.EqualsUpperCaseSpanIgnoringCase("PAGESIZE"u8)) + { + idx++; + if (idx >= parseState.Count) + { + while (!RespWriteUtils.TryWriteError("ERR PAGESIZE requires a value"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + leafPageSize = parseState.GetLong(idx); + idx++; + } + else + { + while (!RespWriteUtils.TryWriteError("ERR unknown option"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + } + + // Validate numeric options + if (cacheSize <= 0 || minRecordSize <= 0 || maxRecordSize <= 0 || maxKeyLen <= 0) + { + while (!RespWriteUtils.TryWriteError("ERR numeric options must be greater than zero"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (minRecordSize > maxRecordSize) + { + while (!RespWriteUtils.TryWriteError("ERR MINRECORD must not exceed MAXRECORD"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var status = storageApi.RangeIndexCreate(key, storageBackend, + (ulong)cacheSize, (uint)minRecordSize, (uint)maxRecordSize, (uint)maxKeyLen, (uint)leafPageSize, + out var result, out var errorMsg); + + if (result == RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + } + else + { + if (errorMsg.Length > 0) + { + while (!RespWriteUtils.TryWriteError(errorMsg, ref dcurr, dend)) + SendAndReset(); + } + else + { + while (!RespWriteUtils.TryWriteError("ERR range index creation failed"u8, ref dcurr, dend)) + SendAndReset(); + } + } + + return true; + } + + /// + /// Handles the RI.SET command. + /// Syntax: RI.SET key field value + /// + /// + /// Returns WRONGTYPE error if the key exists but is not a RangeIndex. + /// Returns an error if the key doesn't exist or the field/value size exceeds limits. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRISET(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 3) + return AbortWithWrongNumberOfArguments("RI.SET"); + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + var value = parseState.GetArgSliceByRef(2); + + var status = storageApi.RangeIndexSet(key, field, value, out var result, out var errorMsg); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result == RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + } + else + { + if (errorMsg.Length > 0) + while (!RespWriteUtils.TryWriteError(errorMsg, ref dcurr, dend)) + SendAndReset(); + else + while (!RespWriteUtils.TryWriteError("ERR range index operation failed"u8, ref dcurr, dend)) + SendAndReset(); + } + + return true; + } + + /// + /// Handles the RI.GET command. + /// Syntax: RI.GET key field + /// + /// + /// Returns the value as a RESP bulk string if found, null bulk string if the field + /// doesn't exist, or WRONGTYPE error if the key is not a RangeIndex. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRIGET(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 2) + return AbortWithWrongNumberOfArguments("RI.GET"); + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + + var output = GetStringOutput(); + var status = storageApi.RangeIndexGet(key, field, ref output, out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + SendAndReset(); + return true; + } + + if (result == RangeIndexResult.OK) + { + ProcessOutput(output.SpanByteAndMemory); + } + else if (result == RangeIndexResult.NotFound) + { + WriteNull(); + } + else + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + } + + return true; + } + + /// + /// Handles the RI.DEL command. + /// Syntax: RI.DEL key field + /// + /// + /// Returns :1 on success or WRONGTYPE error if the key is not a RangeIndex. + /// Note: This deletes a field within the BfTree, not the entire RangeIndex key + /// (use the standard DEL command to delete the key and free the BfTree). + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRIDEL(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 2) + return AbortWithWrongNumberOfArguments("RI.DEL"); + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + + var status = storageApi.RangeIndexDel(key, field, out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + SendAndReset(); + return true; + } + + if (result == RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteInt32(1, ref dcurr, dend)) + SendAndReset(); + } + else + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + } + + return true; + } + + /// + /// Handles the RI.SCAN command. + /// Syntax: RI.SCAN key start COUNT n [FIELDS KEY|VALUE|BOTH] + /// + /// + /// Returns an array of results. Each element is either a bulk string (KEY or VALUE mode) + /// or a 2-element array of [key, value] (BOTH mode, the default). + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRISCAN(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count < 4) + return AbortWithWrongNumberOfArguments("RI.SCAN"); + + var key = parseState.GetArgSliceByRef(0); + var startKey = parseState.GetArgSliceByRef(1); + + if (!parseState.GetArgSliceByRef(2).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase("COUNT"u8)) + { + while (!RespWriteUtils.TryWriteError("ERR syntax error, expected COUNT"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (!parseState.TryGetInt(3, out var count) || count <= 0) + { + while (!RespWriteUtils.TryWriteError("ERR invalid count"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var returnField = ScanReturnField.KeyAndValue; + if (parseState.Count >= 6 && parseState.GetArgSliceByRef(4).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase("FIELDS"u8)) + { + var fieldsVal = parseState.GetArgSliceByRef(5).ReadOnlySpan; + if (fieldsVal.EqualsUpperCaseSpanIgnoringCase("KEY"u8)) + returnField = ScanReturnField.Key; + else if (fieldsVal.EqualsUpperCaseSpanIgnoringCase("VALUE"u8)) + returnField = ScanReturnField.Value; + else + returnField = ScanReturnField.KeyAndValue; + } + + var output = GetStringOutput(); + var status = storageApi.RangeIndexScan(key, startKey, count, returnField, + ref output, out _, out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result == RangeIndexResult.MemoryModeNotSupported) + { + while (!RespWriteUtils.TryWriteError("ERR RI.SCAN is not supported for MEMORY-mode indexes"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result != RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + ProcessOutput(output.SpanByteAndMemory); + return true; + } + + /// + /// Handles the RI.RANGE command. + /// Syntax: RI.RANGE key start end [FIELDS KEY|VALUE|BOTH] + /// + /// + /// Returns all entries in the closed range [start, end], ordered by key. + /// Same response format as RI.SCAN. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRIRANGE(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count < 3) + return AbortWithWrongNumberOfArguments("RI.RANGE"); + + var key = parseState.GetArgSliceByRef(0); + var startKey = parseState.GetArgSliceByRef(1); + var endKey = parseState.GetArgSliceByRef(2); + + var returnField = ScanReturnField.KeyAndValue; + if (parseState.Count >= 5 && parseState.GetArgSliceByRef(3).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase("FIELDS"u8)) + { + var fieldsVal = parseState.GetArgSliceByRef(4).ReadOnlySpan; + if (fieldsVal.EqualsUpperCaseSpanIgnoringCase("KEY"u8)) + returnField = ScanReturnField.Key; + else if (fieldsVal.EqualsUpperCaseSpanIgnoringCase("VALUE"u8)) + returnField = ScanReturnField.Value; + else + returnField = ScanReturnField.KeyAndValue; + } + + var output = GetStringOutput(); + var status = storageApi.RangeIndexRange(key, startKey, endKey, returnField, + ref output, out _, out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result == RangeIndexResult.MemoryModeNotSupported) + { + while (!RespWriteUtils.TryWriteError("ERR RI.RANGE is not supported for MEMORY-mode indexes"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result != RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + ProcessOutput(output.SpanByteAndMemory); + return true; + } + + /// + /// Handles the RI.EXISTS command. + /// Syntax: RI.EXISTS key + /// + /// + /// Returns :1 if the key exists and is a RangeIndex, :0 otherwise. + /// Unlike other RI commands, does NOT return WRONGTYPE for non-RI keys — + /// it simply returns :0 for any key that is not an RI key. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRIEXISTS(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 1) + return AbortWithWrongNumberOfArguments("RI.EXISTS"); + + var key = parseState.GetArgSliceByRef(0); + + storageApi.RangeIndexExists(key, out var exists); + + while (!RespWriteUtils.TryWriteInt32(exists ? 1 : 0, ref dcurr, dend)) + SendAndReset(); + + return true; + } + + /// + /// Handles the RI.CONFIG command. + /// Syntax: RI.CONFIG key + /// + /// + /// Returns the configuration of the index as 12 alternating field-value bulk strings: + /// storage_backend, cache_size, min_record_size, max_record_size, max_key_len, leaf_page_size. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRICONFIG(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 1) + return AbortWithWrongNumberOfArguments("RI.CONFIG"); + + var key = parseState.GetArgSliceByRef(0); + + var status = storageApi.RangeIndexConfig(key, out var storageBackend, out var cacheSize, + out var minRecordSize, out var maxRecordSize, out var maxKeyLen, out var leafPageSize, + out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result != RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + // 6 fields × 2 (field name + value) = 12 elements + while (!RespWriteUtils.TryWriteArrayLength(12, ref dcurr, dend)) + SendAndReset(); + + // storage_backend + while (!RespWriteUtils.TryWriteBulkString("storage_backend"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteBulkString(storageBackend == 0 ? "DISK"u8 : "MEMORY"u8, ref dcurr, dend)) + SendAndReset(); + + // cache_size + while (!RespWriteUtils.TryWriteBulkString("cache_size"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(cacheSize.ToString(), ref dcurr, dend)) + SendAndReset(); + + // min_record_size + while (!RespWriteUtils.TryWriteBulkString("min_record_size"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(minRecordSize.ToString(), ref dcurr, dend)) + SendAndReset(); + + // max_record_size + while (!RespWriteUtils.TryWriteBulkString("max_record_size"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(maxRecordSize.ToString(), ref dcurr, dend)) + SendAndReset(); + + // max_key_len + while (!RespWriteUtils.TryWriteBulkString("max_key_len"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(maxKeyLen.ToString(), ref dcurr, dend)) + SendAndReset(); + + // leaf_page_size + while (!RespWriteUtils.TryWriteBulkString("leaf_page_size"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(leafPageSize.ToString(), ref dcurr, dend)) + SendAndReset(); + + return true; + } + + /// + /// Handles the RI.METRICS command. + /// Syntax: RI.METRICS key + /// + /// + /// Returns 8 alternating field-value bulk strings: + /// tree_handle (native pointer), is_live, is_flushed, is_recovered. + /// Useful for diagnostics and testing the stub lifecycle. + /// + /// The Garnet API type. + /// Reference to the storage API. + /// Always true (command fully processed). + private bool NetworkRIMETRICS(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (storeWrapper.rangeIndexManager is null) + return AbortWithErrorMessage("ERR Range Index (preview) commands are not enabled"); + + if (parseState.Count != 1) + return AbortWithWrongNumberOfArguments("RI.METRICS"); + + var key = parseState.GetArgSliceByRef(0); + + var status = storageApi.RangeIndexMetrics(key, out var treeHandle, out var isLive, out var isFlushed, out var isRecovered, out var result); + + if (status == GarnetStatus.WRONGTYPE) + { + while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (result != RangeIndexResult.OK) + { + while (!RespWriteUtils.TryWriteError("ERR range index not found"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + // 4 fields × 2 = 8 elements + while (!RespWriteUtils.TryWriteArrayLength(8, ref dcurr, dend)) + SendAndReset(); + + // tree_handle + while (!RespWriteUtils.TryWriteBulkString("tree_handle"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteAsciiBulkString(treeHandle.ToString(), ref dcurr, dend)) + SendAndReset(); + + // is_live + while (!RespWriteUtils.TryWriteBulkString("is_live"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteBulkString(isLive ? "true"u8 : "false"u8, ref dcurr, dend)) + SendAndReset(); + + // is_flushed + while (!RespWriteUtils.TryWriteBulkString("is_flushed"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteBulkString(isFlushed ? "true"u8 : "false"u8, ref dcurr, dend)) + SendAndReset(); + + // is_recovered + while (!RespWriteUtils.TryWriteBulkString("is_recovered"u8, ref dcurr, dend)) + SendAndReset(); + while (!RespWriteUtils.TryWriteBulkString(isRecovered ? "true"u8 : "false"u8, ref dcurr, dend)) + SendAndReset(); + + return true; + } + } +} \ No newline at end of file diff --git a/libs/server/Resp/RespCommandAccessor.cs b/libs/server/Resp/RespCommandAccessor.cs index b5a3ac927fb..a9310196213 100644 --- a/libs/server/Resp/RespCommandAccessor.cs +++ b/libs/server/Resp/RespCommandAccessor.cs @@ -4,7 +4,7 @@ namespace Garnet.server { /// - /// RESP command accessor + /// Accessor to simplify access for a subset of usable by external components such as cluster plugins. /// public static class RespCommandAccessor { diff --git a/libs/server/Resp/RespCommandDocs.cs b/libs/server/Resp/RespCommandDocs.cs index b58578f7371..090155d4657 100644 --- a/libs/server/Resp/RespCommandDocs.cs +++ b/libs/server/Resp/RespCommandDocs.cs @@ -331,7 +331,7 @@ public enum RespCommandGroup : byte [Description("transactions")] Transactions, [Description("vector")] - Vector + Vector, } /// diff --git a/libs/server/Resp/RespCommandInfoFlags.cs b/libs/server/Resp/RespCommandInfoFlags.cs index bfe03845bf7..93992911071 100644 --- a/libs/server/Resp/RespCommandInfoFlags.cs +++ b/libs/server/Resp/RespCommandInfoFlags.cs @@ -115,6 +115,6 @@ public enum RespAclCategories [Description("vector")] Vector = 1 << 23, [Description("all")] - All = (Custom << 1) - 1, + All = (Vector << 1) - 1, } } \ No newline at end of file diff --git a/libs/server/Resp/RespCommandKeySpecification.cs b/libs/server/Resp/RespCommandKeySpecification.cs index 7cbc2f49452..23183827ea5 100644 --- a/libs/server/Resp/RespCommandKeySpecification.cs +++ b/libs/server/Resp/RespCommandKeySpecification.cs @@ -8,6 +8,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -328,7 +329,7 @@ public abstract class FindKeysKeySpecMethodBase : KeySpecMethodBase /// The current session parse state. /// The index from which to start extracting keys. /// The list to which extracted keys will be added. - public abstract void ExtractKeys(ref SessionParseState state, int startIndex, List keys); + public abstract void ExtractKeys(ref SessionParseState state, int startIndex, List keys); } /// @@ -381,7 +382,7 @@ public FindKeysRange(int lastKey, int keyStep, int limit) : this() } /// - public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) + public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) { int lastKey; if (LastKey < 0) @@ -461,7 +462,7 @@ public FindKeysKeyNum(int keyNumIdx, int firstKey, int keyStep) : this() } /// - public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) + public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) { int numKeys = 0; int firstKey = startIndex + FirstKey; @@ -518,7 +519,7 @@ public override void ToRespFormat(ref RespMemoryWriter writer) } /// - public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) + public override void ExtractKeys(ref SessionParseState state, int startIndex, List keys) { // Do nothing } diff --git a/libs/server/Resp/RespEnums.cs b/libs/server/Resp/RespEnums.cs index f51ce48fada..eb98f1218f8 100644 --- a/libs/server/Resp/RespEnums.cs +++ b/libs/server/Resp/RespEnums.cs @@ -13,12 +13,6 @@ internal enum ExpirationOption : byte KEEPTTL } - internal enum EtagOption : byte - { - None, - WithETag, - } - public enum ExistOptions : byte { None, diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index f91c2bfcd45..0efdfd44955 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -20,25 +20,6 @@ namespace Garnet.server { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - using LockableGarnetApi = GarnetApi, - SpanByteAllocator>>, - LockableContext>, - GenericAllocator>>>, - LockableContext, - SpanByteAllocator>>>; - /// /// RESP server session /// @@ -48,6 +29,8 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase public GarnetLatencyMetricsSession LatencyMetrics { get; } + public StoreWrapper StoreWrapper => this.storeWrapper; + readonly CommandStats commandStats; /// @@ -119,8 +102,11 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase /// public StorageSession storageSession; internal BasicGarnetApi basicGarnetApi; - internal LockableGarnetApi lockableGarnetApi; + internal TransactionalGarnetApi transactionalGarnetApi; internal TransactionManager txnManager; + internal ConsistentReadGarnetApi consistentReadGarnetApi; + internal TransactionalConsistentReadGarnetApi txnConsistentReadApi; + internal ReadSessionState readSessionState; readonly IGarnetAuthenticator _authenticator; @@ -130,9 +116,15 @@ internal sealed unsafe partial class RespServerSession : ServerSessionBase // True if multiple logical databases are enabled on this session readonly bool allowMultiDb; + // Track whether consistent read session is active + internal bool IsConsistentReadSessionActive = false; + // Map of all active database sessions (default of size 1, containing DB 0 session) private ExpandableMap databaseSessions; + // Consistent database read session + private GarnetDatabaseSession consistentReadDBSession; + /// /// The user currently authenticated in this session /// @@ -241,8 +233,6 @@ public IGarnetServer Server // Threshold for slow log in ticks (0 means disabled) readonly long slowLogThreshold; - private readonly long maximumVectorSetValueBytes; - /// /// Create a new RESP server session /// @@ -293,11 +283,15 @@ public RespServerSession( // Create the default DB session (for DB 0) & add it to the session map activeDbId = 0; var dbSession = CreateDatabaseSession(0); - var maxDbs = storeWrapper.serverOptions.MaxDatabases; + var maxDbs = storeWrapper.serverOptions.EnableCluster ? 2 : storeWrapper.serverOptions.MaxDatabases; databaseSessions = new ExpandableMap(1, 0, maxDbs - 1); if (!databaseSessions.TrySetValue(0, dbSession)) - throw new GarnetException("Failed to set initial database session in database sessions map"); + throw new GarnetException("Failed to set initialize database session in database sessions map!"); + + // Create consistent read APIs and storageSession + if (storeWrapper.serverOptions.EnableCluster && storeWrapper.serverOptions.EnableAOF && storeWrapper.serverOptions.MultiLogEnabled && storeWrapper.appendOnlyFile != null) + consistentReadDBSession = CreateConsistentReadApi(); // Set the current active session to the default session SwitchActiveDatabaseSession(dbSession); @@ -306,8 +300,7 @@ public RespServerSession( this.AuthenticateUser(Encoding.ASCII.GetBytes(this.storeWrapper.accessControlList.GetDefaultUserHandle().User.Name)); var cp = clusterProvider ?? storeWrapper.clusterProvider; - - clusterSession = cp?.CreateClusterSession(txnManager, this._authenticator, this._userHandle, sessionMetrics, basicGarnetApi, storageSession.basicContext, storageSession.vectorContext, networkSender, logger); + clusterSession = cp?.CreateClusterSession(txnManager, this._authenticator, this._userHandle, sessionMetrics, basicGarnetApi, storageSession.stringBasicContext, storageSession.vectorBasicContext, networkSender, logger); clusterSession?.SetUserHandle(this._userHandle); sessionScriptCache?.SetUserHandle(this._userHandle); @@ -324,8 +317,6 @@ public RespServerSession( if (this.networkSender.GetMaxSizeSettings?.MaxOutputSize < sizeof(int)) this.networkSender.GetMaxSizeSettings.MaxOutputSize = sizeof(int); } - - maximumVectorSetValueBytes = GarnetServerOptions.ParseSize(storeWrapper.serverOptions.PageSize, out _) - 16; // Just assume header is 16-ish bytes for now } /// @@ -388,17 +379,24 @@ public void UpdateRespProtocolVersion(byte _respProtocolVersion) public override void Dispose() { - logger?.LogDebug("Disposing RespServerSession Id={0}", this.Id); + logger?.LogDebug("Disposing RespServerSession Id={id}", this.Id); if (recvBufferPtr != null) { try { if (recvHandle.IsAllocated) recvHandle.Free(); } catch { } } + // Dispose read session state + readSessionState?.Dispose(); + // Dispose special consistent read database session + consistentReadDBSession?.Dispose(); + // Dispose all database sessions foreach (var dbSession in databaseSessions.Map) dbSession?.Dispose(); + clusterSession?.Dispose(); + if (storeWrapper.monitor != null) storeWrapper.monitor.AddMetricsHistorySessionDispose(sessionMetrics, LatencyMetrics, commandStats); @@ -466,10 +464,12 @@ internal bool CanRunModule() networkSender.IsLocalConnection()); } + bool txnSkip = false; + public override int TryConsumeMessages(byte* reqBuffer, int bytesReceived) { bytesRead = bytesReceived; - if (!txnManager.IsSkippingOperations()) + if (!txnSkip) readHead = 0; try { @@ -481,7 +481,32 @@ public override int TryConsumeMessages(byte* reqBuffer, int bytesReceived) clusterSession?.AcquireCurrentEpoch(); recvBufferPtr = reqBuffer; networkSender.EnterAndGetResponseObject(out dcurr, out dend); - ProcessMessages(); + + if (storeWrapper.EnforceConsistentRead()) + { + try + { + // We actively switch session because we aim to avoid performing any additional checks or switches on the normal processing path + // This requires us to cache txnSkip result since the txnManager instance will change when the following finally executes + // Switching is required because we cannot guarantee the role of the node outside the epoch protection + txnSkip = false; + Debug.Assert(consistentReadDBSession != null); + SwitchActiveDatabaseSession(consistentReadDBSession); + ProcessMessages(ref consistentReadGarnetApi, ref txnConsistentReadApi); + txnSkip = txnManager.IsSkippingOperations(); + } + finally + { + // Switch back to normal session in the event a failover results in this node to become a primary + SwitchActiveDatabaseSession(databaseSessions.Map[0]); + } + } + else + { + txnSkip = false; + ProcessMessages(ref basicGarnetApi, ref transactionalGarnetApi); + txnSkip = txnManager.IsSkippingOperations(); + } recvBufferPtr = null; } catch (RespParsingException ex) @@ -543,7 +568,7 @@ public override int TryConsumeMessages(byte* reqBuffer, int bytesReceived) scratchBufferAllocator.Reset(); } - if (txnManager.IsSkippingOperations()) + if (txnSkip) return 0; // so that network does not try to shift the byte array // If server processed input data successfully, update tracked metrics @@ -585,7 +610,9 @@ internal void ExitAndReturnResponseObject() internal void SetTransactionMode(bool enable) => txnManager.state = enable ? TxnState.Running : TxnState.None; - private void ProcessMessages() + private void ProcessMessages(ref TBasicApi basicApi, ref TTxnApi transactionalApi) + where TBasicApi : IGarnetApi + where TTxnApi : IGarnetApi { // #if DEBUG // logger?.LogTrace("RECV: [{recv}]", Encoding.UTF8.GetString(new Span(recvBufferPtr, bytesRead)).Replace("\n", "|").Replace("\r", "")); @@ -599,7 +626,7 @@ private void ProcessMessages() // First, parse the command, making sure we have the entire command available // We use endReadHead to track the end of the current command // On success, readHead is left at the start of the command payload for legacy operators - var cmd = ParseCommand(writeErrorOnFailure: true, out bool commandReceived); + var cmd = ParseCommand(writeErrorOnFailure: true, out var commandReceived); // If the command was not fully received, reset addresses and break out if (!commandReceived) @@ -613,16 +640,13 @@ private void ProcessMessages() { var noScriptPassed = true; - // Reset error flag unconditionally (only read when commandStats != null) - commandErrorWritten = false; - if (CheckACLPermissions(cmd) && (noScriptPassed = CheckScriptPermissions(cmd))) { if (txnManager.state != TxnState.None) { if (txnManager.state == TxnState.Running) { - _ = ProcessBasicCommands(cmd, ref lockableGarnetApi); + _ = ProcessBasicCommands(cmd, ref transactionalApi); } else _ = cmd switch { @@ -636,14 +660,17 @@ private void ProcessMessages() else { if (clusterSession == null || CanServeSlot(cmd)) - _ = ProcessBasicCommands(cmd, ref basicGarnetApi); + _ = ProcessBasicCommands(cmd, ref basicApi); } if (commandStats != null) { commandStats.IncrementCalls(cmd); if (commandErrorWritten) + { commandStats.IncrementFailed(cmd); + commandErrorWritten = false; + } } } else @@ -1063,8 +1090,20 @@ private bool ProcessOtherCommands(RespCommand command, ref TGarnetAp RespCommand.GETIFNOTMATCH => NetworkGETIFNOTMATCH(ref storageApi), RespCommand.SETIFMATCH => NetworkSETIFMATCH(ref storageApi), RespCommand.SETIFGREATER => NetworkSETIFGREATER(ref storageApi), + RespCommand.SETWITHETAG => NetworkSETWITHETAG(ref storageApi), RespCommand.DELIFGREATER => NetworkDELIFGREATER(ref storageApi), + // RangeIndex commands + RespCommand.RICREATE => NetworkRICREATE(ref storageApi), + RespCommand.RISET => NetworkRISET(ref storageApi), + RespCommand.RIGET => NetworkRIGET(ref storageApi), + RespCommand.RIDEL => NetworkRIDEL(ref storageApi), + RespCommand.RISCAN => NetworkRISCAN(ref storageApi), + RespCommand.RIRANGE => NetworkRIRANGE(ref storageApi), + RespCommand.RIEXISTS => NetworkRIEXISTS(ref storageApi), + RespCommand.RICONFIG => NetworkRICONFIG(ref storageApi), + RespCommand.RIMETRICS => NetworkRIMETRICS(ref storageApi), + _ => Process(command, ref storageApi) }; @@ -1133,7 +1172,7 @@ private bool NetworkCustomRawStringCmd(ref TGarnetApi storageApi) // Perform the operation var cmd = customCommandManagerSession.GetCustomRespCommand(currentCustomRawStringCommand.id); - TryCustomRawStringCommand(cmd, currentCustomRawStringCommand.expirationTicks, currentCustomRawStringCommand.type, ref storageApi); + TryCustomRawStringCommand(cmd, currentCustomRawStringCommand, ref storageApi); currentCustomRawStringCommand = null; return true; } @@ -1149,8 +1188,7 @@ bool NetworkCustomObjCmd(ref TGarnetApi storageApi) // Perform the operation var type = customCommandManagerSession.GetCustomGarnetObjectType(currentCustomObjectCommand.id); - TryCustomObjectCommand(type, currentCustomObjectCommand.subid, - currentCustomObjectCommand.type, ref storageApi); + TryCustomObjectCommand(type, currentCustomObjectCommand, ref storageApi); currentCustomObjectCommand = null; return true; } @@ -1261,13 +1299,9 @@ private unsafe bool Write(ref Status s, ref byte* dst, int length) private static unsafe bool Write(ref SpanByteAndMemory k, ref byte* dst, int length) { - if (k.Length > length) return false; - - var dest = new SpanByte(length, (IntPtr)dst); - if (k.IsSpanByte) - k.SpanByte.CopyTo(ref dest); - else - k.AsMemoryReadOnlySpan().CopyTo(dest.AsSpan()); + if (k.Length > length) + return false; + k.ReadOnlySpan.CopyTo(new Span(dst, length)); return true; } @@ -1385,7 +1419,7 @@ private void Send(byte* d) if ((int)(dcurr - d) > 0) { - //Debug.WriteLine("SEND: [" + Encoding.UTF8.GetString(new Span(d, (int)(dcurr - d))).Replace("\n", "|").Replace("\r", "!") + "]"); + // Debug.WriteLine("SEND: [" + Encoding.UTF8.GetString(new Span(d, (int)(dcurr - d))).Replace("\n", "|").Replace("\r", "!") + "]"); if (waitForAofBlocking) { var task = storeWrapper.WaitForCommitAsync(); @@ -1556,17 +1590,93 @@ private GarnetDatabaseSession CreateDatabaseSession(int dbId) var dbRes = storeWrapper.TryGetOrAddDatabase(dbId, out var database, out _); Debug.Assert(dbRes, "Should always find database if we're switching to it"); - var dbStorageSession = new StorageSession(storeWrapper, scratchBufferBuilder, sessionMetrics, LatencyMetrics, dbId, database.VectorManager, logger, respProtocolVersion); - var dbGarnetApi = new BasicGarnetApi(dbStorageSession, dbStorageSession.basicContext, dbStorageSession.objectStoreBasicContext); - var dbLockableGarnetApi = new LockableGarnetApi(dbStorageSession, dbStorageSession.lockableContext, dbStorageSession.objectStoreLockableContext); + var dbStorageSession = new StorageSession( + storeWrapper, + scratchBufferBuilder, + scratchBufferAllocator, + sessionMetrics, + LatencyMetrics, + dbId, + readSessionState: null, + database.VectorManager, + logger, + respProtocolVersion); + var dbGarnetApi = new BasicGarnetApi(dbStorageSession, dbStorageSession.stringBasicContext, + dbStorageSession.objectBasicContext, dbStorageSession.unifiedBasicContext); + var dbLockableGarnetApi = new TransactionalGarnetApi(dbStorageSession, + dbStorageSession.stringTransactionalContext, dbStorageSession.objectTransactionalContext, + dbStorageSession.unifiedTransactionalContext); var transactionManager = new TransactionManager(storeWrapper, this, dbGarnetApi, dbLockableGarnetApi, - dbStorageSession, scratchBufferAllocator, storeWrapper.serverOptions.EnableCluster, logger, dbId); + dbStorageSession, scratchBufferAllocator, storeWrapper.serverOptions.EnableCluster, logger: logger, dbId: dbId); dbStorageSession.txnManager = transactionManager; return new GarnetDatabaseSession(dbId, dbStorageSession, dbGarnetApi, dbLockableGarnetApi, transactionManager); } + /// + /// Create consistent read API + /// + private GarnetDatabaseSession CreateConsistentReadApi() + { + // NOTE: + // Consistent read session should point to dbId = 0 (because dbId is used to identify working database), + // though its session id = 1 to differentiate between normal session. + // Session id is set at the caller. + var dbId = 0; + var dbRes = storeWrapper.TryGetOrAddDatabase(dbId, out var database, out _); + Debug.Assert(dbRes, "Should always find database if we're switching to it"); + + readSessionState = new ReadSessionState(storeWrapper.appendOnlyFile, storeWrapper.serverOptions); + + // NOTE: We need to create storage session to tie it to the consistent read API + var dbStorageSession = new StorageSession( + storeWrapper, + scratchBufferBuilder, + scratchBufferAllocator, + sessionMetrics, + LatencyMetrics, + dbId: dbId, // NOTE: only for cluster need to retrieve default database + readSessionState: readSessionState, + database.VectorManager, + logger, + respProtocolVersion); + + var dbGarnetApi = new BasicGarnetApi(dbStorageSession, dbStorageSession.stringBasicContext, + dbStorageSession.objectBasicContext, dbStorageSession.unifiedBasicContext); + var dbLockableGarnetApi = new TransactionalGarnetApi(dbStorageSession, + dbStorageSession.stringTransactionalContext, dbStorageSession.objectTransactionalContext, + dbStorageSession.unifiedTransactionalContext); + + var consistentReadGarnetApi = new ConsistentReadGarnetApi(dbStorageSession, dbStorageSession.consistentReadContext, + dbStorageSession.objectStoreConsistentReadContext, dbStorageSession.unifiedStoreConsistentReadContext); + var txnConsistentReadApi = new TransactionalConsistentReadGarnetApi(dbStorageSession, + dbStorageSession.transactionalConsistentReadContext, dbStorageSession.objectStoreTransactionalConsistentReadContext, + dbStorageSession.unifiedStoreTransactionalConsistentReadContext); + + var consistentReadTransactionManager = new TransactionManager( + storeWrapper, + this, + dbGarnetApi, + dbLockableGarnetApi, + dbStorageSession, + scratchBufferAllocator, + storeWrapper.serverOptions.EnableCluster, + enableConsistentRead: true, + garnetConsistentApi: consistentReadGarnetApi, + transactionalConsistentGarnetApi: txnConsistentReadApi, + logger: logger, + dbId: dbId); + + return new GarnetDatabaseSession(id: dbId, // NOTE: sessionID 1 to differentiate from default session + dbStorageSession, + dbGarnetApi, + dbLockableGarnetApi, + consistentReadTransactionManager, + consistentReadGarnetApi, + txnConsistentReadApi); + } + /// /// Switch current active database session /// @@ -1577,9 +1687,22 @@ private void SwitchActiveDatabaseSession(GarnetDatabaseSession dbSession) this.txnManager = dbSession.TransactionManager; this.storageSession = dbSession.StorageSession; this.basicGarnetApi = dbSession.GarnetApi; - this.lockableGarnetApi = dbSession.LockableGarnetApi; - + this.transactionalGarnetApi = dbSession.TransactionalGarnetApi; + this.consistentReadGarnetApi = dbSession.ConsistentGarnetApi; + this.txnConsistentReadApi = dbSession.TransactionalConsistentGarnetApi; this.storageSession.UpdateRespProtocolVersion(this.respProtocolVersion); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private StringOutput GetStringOutput() + => StringOutput.FromPinnedPointer(dcurr, (int)(dend - dcurr)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private ObjectOutput GetObjectOutput() + => ObjectOutput.FromPinnedPointer(dcurr, (int)(dend - dcurr)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private UnifiedOutput GetUnifiedOutput() + => UnifiedOutput.FromPinnedPointer(dcurr, (int)(dend - dcurr)); } } \ No newline at end of file diff --git a/libs/server/Resp/RespServerSessionSlotVerify.cs b/libs/server/Resp/RespServerSessionSlotVerify.cs index bb1f6326164..efc79caeb10 100644 --- a/libs/server/Resp/RespServerSessionSlotVerify.cs +++ b/libs/server/Resp/RespServerSessionSlotVerify.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; namespace Garnet.server @@ -11,19 +10,18 @@ namespace Garnet.server /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { - /// - /// This method is used to verify slot ownership for provided array of key argslices. - /// - /// Array of key ArgSlice - /// Whether caller is going to perform a readonly or read/write operation - /// Key count if different than keys array length - /// Whether the executing command requires the containing slot be STABLE. - /// True when ownership is verified, false otherwise - bool NetworkKeyArraySlotVerify(Span keys, bool readOnly, bool waitForStableSlot, int count = -1) - => clusterSession != null && clusterSession.NetworkKeyArraySlotVerify(keys, readOnly, SessionAsking, waitForStableSlot, ref dcurr, ref dend, count); + // Key spec for CustomRawStringCmd and CustomObjCmd: a single key at arg index 1. + private static readonly SimpleRespKeySpec[] CustomCommandSingleKeySpec = + [ + new SimpleRespKeySpec + { + BeginSearch = new SimpleRespKeySpecBeginSearch(index: 1), + FindKeys = new SimpleRespKeySpecFindKeys(keyStep: 1, lastKeyOrLimit: 0, isLimit: false), + } + ]; /// - /// Validate if this command can be served based on the current slot assignment + /// Validates whether a command can be served based on the current slot assignment /// /// /// @@ -31,26 +29,53 @@ bool CanServeSlot(RespCommand cmd) { Debug.Assert(clusterSession != null); - // Verify slot for command if it falls into data command category if (!cmd.IsDataCommand()) + { + // Custom commands sit outside IsDataCommand but still touch user keys. + if (cmd is RespCommand.CustomRawStringCmd or RespCommand.CustomObjCmd) + return CanServeSlotForCustomCommand(cmd); + return true; + } cmd = cmd.NormalizeForACLs(); - if (!RespCommandsInfo.TryFastGetRespCommandInfo(cmd, out var commandInfo)) + if (!RespCommandsInfo.TryGetSimpleRespCommandInfo(cmd, out var cmdInfo)) // This only happens if we failed to parse the json file return false; - // The provided command is not a data command + // The provided command does not have key specs // so we can serve without any slot restrictions - if (commandInfo == null) + if (cmdInfo.KeySpecs == null || cmdInfo.KeySpecs.Length == 0) return true; - csvi.keyNumOffset = -1; - storeWrapper.clusterProvider.ExtractKeySpecs(commandInfo, cmd, ref parseState, ref csvi); + csvi.keySpecs = cmdInfo.KeySpecs; + // BITOP's operation argument (AND/OR/XOR/NOT) is consumed by the parser, + // so key indices need a -2 offset like subcommands + csvi.isSubCommand = cmdInfo.IsSubCommand || cmd == RespCommand.BITOP; csvi.readOnly = cmd.IsReadOnly(); csvi.sessionAsking = SessionAsking; csvi.waitForStableSlot = cmd is RespCommand.VADD or RespCommand.VREM or RespCommand.VSETATTR; return !clusterSession.NetworkMultiKeySlotVerify(ref parseState, ref csvi, ref dcurr, ref dend); } + + /// + /// Validates whether a custom command can be served based on the current slot assignment + /// + /// + /// + bool CanServeSlotForCustomCommand(RespCommand cmd) + { + // cmd.IsReadOnly() can't be used here since both custom enum values sort past LastReadCommand. + var isReadOnly = cmd == RespCommand.CustomRawStringCmd + ? currentCustomRawStringCommand.type == CommandType.Read + : currentCustomObjectCommand.type == CommandType.Read; + + csvi.keySpecs = CustomCommandSingleKeySpec; + csvi.isSubCommand = false; + csvi.readOnly = isReadOnly; + csvi.sessionAsking = SessionAsking; + csvi.waitForStableSlot = false; + return !clusterSession.NetworkMultiKeySlotVerify(ref parseState, ref csvi, ref dcurr, ref dend); + } } } \ No newline at end of file diff --git a/libs/server/Resp/Vector/DiskANNService.cs b/libs/server/Resp/Vector/DiskANNService.cs index dd3f81d5413..141deb2bfeb 100644 --- a/libs/server/Resp/Vector/DiskANNService.cs +++ b/libs/server/Resp/Vector/DiskANNService.cs @@ -20,6 +20,18 @@ internal sealed unsafe class DiskANNService internal const byte InternalIdMap = 5; private const byte ExternalIdMap = 6; +#if DEBUG + /// + /// For testing purposes, in DEBUG builds the count of calls to or on this instance. + /// + internal int CreateIndexCalls; + + /// + /// For testing purposes, in DEBUG builds the count of calls to on this instance. + /// + internal int DropIndexCalls; +#endif + public nint CreateIndex( ulong context, uint dimensions, @@ -34,6 +46,10 @@ public nint CreateIndex( delegate* unmanaged[Cdecl] readModifyWriteCallback ) { +#if DEBUG + System.Threading.Interlocked.Increment(ref CreateIndexCalls); +#endif + unsafe { return NativeDiskANNMethods.create_index(context, dimensions, reduceDims, quantType, distanceMetric, buildExplorationFactor, numLinks, (nint)readCallback, (nint)writeCallback, (nint)deleteCallback, (nint)readModifyWriteCallback); @@ -57,6 +73,10 @@ public nint RecreateIndex( public void DropIndex(ulong context, nint index) { +#if DEBUG + System.Threading.Interlocked.Increment(ref DropIndexCalls); +#endif + NativeDiskANNMethods.drop_index(context, index); } @@ -146,7 +166,7 @@ out nint continuation else { outputIdsHandle = null; - output_ids = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputIds.AsSpan())); + output_ids = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputIds.Span)); } var output_ids_len = outputIds.Length; @@ -162,7 +182,7 @@ out nint continuation else { outputDistancesHandle = null; - output_distances = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputDistances.AsSpan())); + output_distances = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputDistances.Span)); } var output_distances_len = outputDistances.Length / sizeof(float); @@ -233,7 +253,7 @@ out nint continuation else { outputIdsHandle = null; - output_ids = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputIds.AsSpan())); + output_ids = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputIds.Span)); } var output_ids_len = outputIds.Length; @@ -249,7 +269,7 @@ out nint continuation else { outputDistancesHandle = null; - output_distances = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputDistances.AsSpan())); + output_distances = Unsafe.AsPointer(ref MemoryMarshal.GetReference(outputDistances.Span)); } var output_distances_len = outputDistances.Length / sizeof(float); diff --git a/libs/server/Resp/Vector/RespServerSessionVectors.cs b/libs/server/Resp/Vector/RespServerSessionVectors.cs index cf3213dc822..bdfe5e2b1ce 100644 --- a/libs/server/Resp/Vector/RespServerSessionVectors.cs +++ b/libs/server/Resp/Vector/RespServerSessionVectors.cs @@ -51,6 +51,7 @@ private bool NetworkVADD(ref TGarnetApi storageApi) } var valueType = VectorValueType.Invalid; + int vectorDims = 0; byte[] rentedValues = null; Span values = stackalloc byte[64 * sizeof(float)]; @@ -70,6 +71,12 @@ private bool NetworkVADD(ref TGarnetApi storageApi) return AbortWithErrorMessage("ERR invalid vector specification"); } + vectorDims = asBytes.Length / sizeof(float); + if (vectorDims > VectorManager.MaxVectorDimensions) + { + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); + } + curIx++; valueType = VectorValueType.FP32; values = asBytes; @@ -82,27 +89,33 @@ private bool NetworkVADD(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("VADD"); } - if (!parseState.TryGetInt(curIx, out var valueCount) || valueCount <= 0) + if (!parseState.TryGetInt(curIx, out vectorDims) || vectorDims <= 0) { return AbortWithErrorMessage("ERR invalid vector specification"); } + curIx++; - if (valueCount * sizeof(float) > values.Length) + if (vectorDims > VectorManager.MaxVectorDimensions) { - values = rentedValues = ArrayPool.Shared.Rent(valueCount * sizeof(float)); + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); } - values = values[..(valueCount * sizeof(float))]; - if (curIx + valueCount > parseState.Count) + if (curIx + vectorDims > parseState.Count) { return AbortWithWrongNumberOfArguments("VADD"); } + if (vectorDims * sizeof(float) > values.Length) + { + values = rentedValues = ArrayPool.Shared.Rent(vectorDims * sizeof(float)); + } + values = values[..(vectorDims * sizeof(float))]; + valueType = VectorValueType.FP32; var floatValues = MemoryMarshal.Cast(values); - for (var valueIx = 0; valueIx < valueCount; valueIx++) + for (var valueIx = 0; valueIx < vectorDims; valueIx++) { if (!parseState.TryGetFloat(curIx, out floatValues[valueIx])) { @@ -121,11 +134,26 @@ private bool NetworkVADD(ref TGarnetApi storageApi) } var asBytes = parseState.GetArgSliceByRef(curIx).Span; - curIx++; + vectorDims = asBytes.Length; + if (vectorDims > VectorManager.MaxVectorDimensions) + { + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); + } + + curIx++; valueType = VectorValueType.XB8; values = asBytes; } + else + { + return AbortWithErrorMessage("ERR invalid vector specification"); + } + + if (reduceDim > vectorDims) + { + return AbortWithErrorMessage("ERR REDUCE dimension must be <= vector dimensions"); + } if (curIx >= parseState.Count) { @@ -139,7 +167,7 @@ private bool NetworkVADD(ref TGarnetApi storageApi) var cas = false; VectorQuantType? quantType = null; int? buildExplorationFactor = null; - ArgSlice? attributes = null; + PinnedSpanByte? attributes = null; int? numLinks = null; VectorDistanceMetricType? distanceMetric = null; @@ -231,9 +259,9 @@ private bool NetworkVADD(ref TGarnetApi storageApi) return AbortWithErrorMessage("ERR invalid option after element"); } - if (!parseState.TryGetInt(curIx, out var buildExplorationFactorNonNull) || buildExplorationFactorNonNull <= 0) + if (!parseState.TryGetInt(curIx, out var buildExplorationFactorNonNull) || buildExplorationFactorNonNull <= 0 || buildExplorationFactorNonNull > VectorManager.MaxExplorationFactor) { - return AbortWithErrorMessage("ERR invalid EF"); + return AbortWithErrorMessage($"ERR EF must be an integer between 1 and {VectorManager.MaxExplorationFactor}"); } buildExplorationFactor = buildExplorationFactorNonNull; @@ -279,7 +307,7 @@ private bool NetworkVADD(ref TGarnetApi storageApi) if (!parseState.TryGetInt(curIx, out var numLinksNonNull) || numLinksNonNull < MinM || numLinksNonNull > MaxM) { - return AbortWithErrorMessage("ERR invalid M"); + return AbortWithErrorMessage($"ERR M must be an integer between {MinM} and {MaxM}"); } numLinks = numLinksNonNull; @@ -346,21 +374,6 @@ private bool NetworkVADD(ref TGarnetApi storageApi) numLinks ??= 16; distanceMetric ??= VectorDistanceMetricType.L2; - // Validate that DiskANN is expected to succeed given data sizes - // - // Note that this goes away in store v2 - if (values.Length > maximumVectorSetValueBytes) - { - WriteError("ERR Vector exceed configured page size"u8); - return true; - } - - if (attributes.Value.Length > maximumVectorSetValueBytes) - { - WriteError("ERR Attribute exceed configured page size"u8); - return true; - } - if (quantType != VectorQuantType.XPreQ8 && quantType != VectorQuantType.NoQuant) { WriteError("ERR Unsupported quantization type"u8); @@ -379,7 +392,18 @@ private bool NetworkVADD(ref TGarnetApi storageApi) } else { - res = storageApi.VectorSetAdd(key, reduceDim, valueType, ArgSlice.FromPinnedSpan(values), element, quantType.Value, buildExplorationFactor.Value, attributes.Value, numLinks.Value, distanceMetric.Value, out result, out customErrMsg); + if (rentedValues != null) + { + // For large enough values we have to pay for a pin + fixed (byte* valuesPtr = rentedValues) + { + res = storageApi.VectorSetAdd(key, reduceDim, valueType, PinnedSpanByte.FromPinnedPointer(valuesPtr, values.Length), element, quantType.Value, buildExplorationFactor.Value, attributes.Value, numLinks.Value, distanceMetric.Value, out result, out customErrMsg); + } + } + else + { + res = storageApi.VectorSetAdd(key, reduceDim, valueType, PinnedSpanByte.FromPinnedSpan(values), element, quantType.Value, buildExplorationFactor.Value, attributes.Value, numLinks.Value, distanceMetric.Value, out result, out customErrMsg); + } } if (res == GarnetStatus.OK) @@ -424,10 +448,6 @@ private bool NetworkVADD(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } else { return AbortWithErrorMessage($"Unexpected GarnetStatus: {res}"); @@ -470,7 +490,7 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) var curIx = 2; - ArgSlice? element; + PinnedSpanByte? element; VectorValueType valueType = VectorValueType.Invalid; byte[] rentedValues = null; @@ -499,6 +519,11 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) return AbortWithErrorMessage("FP32 values must be multiple of 4-bytes in size"); } + if (asBytes.Length / sizeof(float) > VectorManager.MaxVectorDimensions) + { + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); + } + valueType = VectorValueType.FP32; values = asBytes; curIx++; @@ -512,6 +537,11 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) var asBytes = parseState.GetArgSliceByRef(curIx).Span; + if (asBytes.Length > VectorManager.MaxVectorDimensions) + { + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); + } + valueType = VectorValueType.XB8; values = asBytes; curIx++; @@ -527,6 +557,12 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) { return AbortWithErrorMessage("VALUES count must > 0"); } + + if (valueCount > VectorManager.MaxVectorDimensions) + { + return AbortWithErrorMessage($"ERR vector exceeds maximum of {VectorManager.MaxVectorDimensions} dimensions"); + } + curIx++; if (valueCount * sizeof(float) > values.Length) @@ -564,7 +600,7 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) int? count = null; float? delta = null; int? searchExplorationFactor = null; - ArgSlice? filter = null; + PinnedSpanByte? filter = null; int? maxFilteringEffort = null; var truth = false; var noThread = false; @@ -611,9 +647,9 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("VSIM"); } - if (!parseState.TryGetInt(curIx, out var countNonNull) || countNonNull < 0) + if (!parseState.TryGetInt(curIx, out var countNonNull) || countNonNull < 0 || countNonNull > VectorManager.MaxRetrieveCount) { - return AbortWithErrorMessage("COUNT must be integer >= 0"); + return AbortWithErrorMessage($"ERR COUNT must be an integer between 0 and {VectorManager.MaxRetrieveCount}"); } count = countNonNull; @@ -659,9 +695,9 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("VSIM"); } - if (!parseState.TryGetInt(curIx, out var searchExplorationFactorNonNull) || searchExplorationFactorNonNull < 0) + if (!parseState.TryGetInt(curIx, out var searchExplorationFactorNonNull) || searchExplorationFactorNonNull <= 0 || searchExplorationFactorNonNull > VectorManager.MaxExplorationFactor) { - return AbortWithErrorMessage("EF must be >= 0"); + return AbortWithErrorMessage($"ERR EF must be an integer between 1 and {VectorManager.MaxExplorationFactor}"); } searchExplorationFactor = searchExplorationFactorNonNull; @@ -705,9 +741,9 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("VSIM"); } - if (!parseState.TryGetInt(curIx, out var maxFilteringEffortNonNull) || maxFilteringEffortNonNull < 0) + if (!parseState.TryGetInt(curIx, out var maxFilteringEffortNonNull) || maxFilteringEffortNonNull < 0 || maxFilteringEffortNonNull > VectorManager.MaxRetrieveCount) { - return AbortWithErrorMessage("FILTER-EF must be >= 0"); + return AbortWithErrorMessage($"ERR FILTER-EF must be an integer between 0 and {VectorManager.MaxRetrieveCount}"); } maxFilteringEffort = maxFilteringEffortNonNull; @@ -754,7 +790,7 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) delta ??= 2f; searchExplorationFactor ??= 100; filter ??= default; - maxFilteringEffort ??= count.Value * 200; + maxFilteringEffort ??= (int)Math.Min((long)count.Value * 200, VectorManager.MaxRetrieveCount); // TODO: these stackallocs are dangerous, need logic to avoid stack overflow Span idSpace = stackalloc byte[(DefaultResultSetSize * DefaultIdSize) + (DefaultResultSetSize * sizeof(int))]; @@ -777,7 +813,18 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) VectorIdFormat idFormat; if (!element.HasValue) { - res = storageApi.VectorSetValueSimilarity(key, valueType, ArgSlice.FromPinnedSpan(values), count.Value, delta.Value, searchExplorationFactor.Value, filter.Value, maxFilteringEffort.Value, withAttributes.Value, ref idResult, out idFormat, ref distanceResult, ref attributeResult, out vectorRes, ref filterBitmapResult); + if (rentedValues != null) + { + // For large enough values we have to pay for a pin + fixed (byte* valuesPtr = rentedValues) + { + res = storageApi.VectorSetValueSimilarity(key, valueType, PinnedSpanByte.FromPinnedPointer(valuesPtr, values.Length), count.Value, delta.Value, searchExplorationFactor.Value, filter.Value, maxFilteringEffort.Value, withAttributes.Value, ref idResult, out idFormat, ref distanceResult, ref attributeResult, out vectorRes, ref filterBitmapResult); + } + } + else + { + res = storageApi.VectorSetValueSimilarity(key, valueType, PinnedSpanByte.FromPinnedSpan(values), count.Value, delta.Value, searchExplorationFactor.Value, filter.Value, maxFilteringEffort.Value, withAttributes.Value, ref idResult, out idFormat, ref distanceResult, ref attributeResult, out vectorRes, ref filterBitmapResult); + } } else { @@ -807,11 +854,11 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) } else { - var remainingIds = idResult.AsReadOnlySpan(); - var distancesSpan = MemoryMarshal.Cast(distanceResult.AsReadOnlySpan()); + var remainingIds = idResult.ReadOnlySpan; + var distancesSpan = MemoryMarshal.Cast(distanceResult.ReadOnlySpan); var hasFilter = filterBitmapResult.Length > 0; - var filterBitmap = hasFilter ? filterBitmapResult.AsReadOnlySpan() : default; - var remaininingAttributes = (withAttributes.Value || hasFilter) ? attributeResult.AsReadOnlySpan() : default; + var filterBitmap = hasFilter ? filterBitmapResult.ReadOnlySpan : default; + var remaininingAttributes = (withAttributes.Value || hasFilter) ? attributeResult.ReadOnlySpan : default; var totalFound = distancesSpan.Length; @@ -849,14 +896,14 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) { if (remainingIds.Length < sizeof(int)) { - throw new GarnetException($"Insufficient bytes for result id length at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.AsReadOnlySpan())}"); + throw new GarnetException($"Insufficient bytes for result id length at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.ReadOnlySpan)}"); } var elementLen = BinaryPrimitives.ReadInt32LittleEndian(remainingIds); if (remainingIds.Length < sizeof(int) + elementLen) { - throw new GarnetException($"Insufficient bytes for result of length={elementLen} at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.AsReadOnlySpan())}"); + throw new GarnetException($"Insufficient bytes for result of length={elementLen} at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.ReadOnlySpan)}"); } elementData = remainingIds.Slice(sizeof(int), elementLen); @@ -866,7 +913,7 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) { if (remainingIds.Length < sizeof(int)) { - throw new GarnetException($"Insufficient bytes for result id length at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.AsReadOnlySpan())}"); + throw new GarnetException($"Insufficient bytes for result id length at resultIndex={resultIndex}: {Convert.ToHexString(distanceResult.ReadOnlySpan)}"); } elementData = remainingIds[..sizeof(int)]; @@ -904,7 +951,7 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) { if (remaininingAttributes.Length < sizeof(int)) { - throw new GarnetException($"Insufficient bytes for attribute length at resultIndex={resultIndex}: {Convert.ToHexString(attributeResult.AsReadOnlySpan())}"); + throw new GarnetException($"Insufficient bytes for attribute length at resultIndex={resultIndex}: {Convert.ToHexString(attributeResult.ReadOnlySpan)}"); } var attrLen = BinaryPrimitives.ReadInt32LittleEndian(remaininingAttributes); @@ -932,10 +979,6 @@ private bool NetworkVSIM(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } else { throw new GarnetException($"Unexpected {nameof(GarnetStatus)}: {res}"); @@ -1007,7 +1050,7 @@ private bool NetworkVEMB(ref TGarnetApi storageApi) if (res == GarnetStatus.OK) { - var distanceSpan = MemoryMarshal.Cast(distanceResult.AsReadOnlySpan()); + var distanceSpan = MemoryMarshal.Cast(distanceResult.ReadOnlySpan); while (!RespWriteUtils.TryWriteArrayLength(distanceSpan.Length, ref dcurr, dend)) SendAndReset(); @@ -1022,10 +1065,6 @@ private bool NetworkVEMB(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } else { while (!RespWriteUtils.TryWriteEmptyArray(ref dcurr, dend)) @@ -1083,10 +1122,6 @@ private bool NetworkVDIM(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } else { while (!RespWriteUtils.TryWriteInt32(dimensions, ref dcurr, dend)) @@ -1132,15 +1167,11 @@ private bool NetworkVGETATTR(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } return AbortWithErrorMessage($"Unexpected GarnetStatus: {res}"); } - WriteSimpleString(attributesOutput.AsReadOnlySpan()); + WriteBulkString(attributesOutput.ReadOnlySpan); return true; } finally @@ -1175,10 +1206,6 @@ private bool NetworkVINFO(ref TGarnetApi storageApi) { return AbortVectorSetWrongType(); } - else if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } return AbortWithErrorMessage($"Unexpected GarnetStatus: {res}"); } @@ -1283,11 +1310,7 @@ private bool NetworkVREM(ref TGarnetApi storageApi) var res = storageApi.VectorSetRemove(key, elem); - if (res == GarnetStatus.BADSTATE) - { - return AbortVectorSetPartiallyDeleted(ref key); - } - else if (res == GarnetStatus.WRONGTYPE) + if (res == GarnetStatus.WRONGTYPE) { return AbortVectorSetWrongType(); } @@ -1318,17 +1341,6 @@ private bool NetworkVSETATTR(ref TGarnetApi storageApi) return true; } - private bool AbortVectorSetPartiallyDeleted(ref ArgSlice key) - { - // TODO: We could _finish_ the delete here... though if we do that we should do it for ALL commands, not just Vector Set commands - // That's more intrusive, and is more of a V2 thing... so lets just give a workaround for now - - while (!RespWriteUtils.TryWriteError("ERR Vector Set is in a partially deleted state - re-execute DEL to complete deletion"u8, ref dcurr, dend)) - SendAndReset(); - - return true; - } - private bool AbortVectorSetWrongType() { // Matches Redis behavior - doesn't indicate the type involved diff --git a/libs/server/Resp/Vector/VectorManager.Callbacks.cs b/libs/server/Resp/Vector/VectorManager.Callbacks.cs index 61b128af600..1327de92ae8 100644 --- a/libs/server/Resp/Vector/VectorManager.Callbacks.cs +++ b/libs/server/Resp/Vector/VectorManager.Callbacks.cs @@ -3,30 +3,30 @@ using System; using System.Buffers; -using System.Buffers.Binary; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// /// Methods which calls back into to interact with Garnet. /// public sealed partial class VectorManager { - public unsafe struct VectorReadBatch : IReadArgBatch, IDisposable + public unsafe struct VectorReadBatch : IReadArgBatch { public int Count { get; } + public readonly ReadOnlySpan Parameters + => default; + private readonly ulong context; - private readonly SpanByte lengthPrefixedKeys; + private readonly PinnedSpanByte lengthPrefixedKeys; - public readonly unsafe delegate* unmanaged[Cdecl, SuppressGCTransition] callback; + public readonly delegate* unmanaged[Cdecl, SuppressGCTransition] callback; public readonly nint callbackContext; private int currentIndex; @@ -36,7 +36,7 @@ public unsafe struct VectorReadBatch : IReadArgBatch - public void GetKey(int i, out SpanByte key) + public void GetKey(int i, out VectorElementKey key) { Debug.Assert(i >= 0 && i < Count, "Trying to advance out of bounds"); AdvanceTo(i); - key = SpanByte.FromPinnedPointer(currentPtr + 3, currentLen + 1); - key.MarkNamespace(); - key.SetNamespaceInPayload((byte)context); + ReadOnlySpan keyBytes = new(currentPtr + 4, currentLen); + + key = new((byte)context, keyBytes); } /// @@ -127,7 +124,7 @@ public readonly void GetInput(int i, out VectorInput input) } /// - public readonly void GetOutput(int i, out SpanByte output) + public readonly void GetOutput(int i, out VectorOutput output) { Debug.Assert(i >= 0 && i < Count, "Trying to advance out of bounds"); @@ -136,7 +133,7 @@ public readonly void GetOutput(int i, out SpanByte output) } /// - public readonly void SetOutput(int i, SpanByte output) + public readonly void SetOutput(int i, VectorOutput output) { Debug.Assert(i >= 0 && i < Count, "Trying to advance out of bounds"); } @@ -149,30 +146,13 @@ public void SetStatus(int i, Status status) hasPending |= status.IsPending; } - internal readonly void CompletePending(ref TContext objectContext) - where TContext : ITsavoriteContext + internal readonly void CompletePending(ref VectorBasicContext objectContext) { - // Undo mutations - *(int*)currentPtr = currentLen; - if (hasPending) { _ = objectContext.CompletePending(wait: true); } } - - /// - public void Dispose() - { - if (currentPtr == null) - { - return; - } - - // Undo mangling of prefix, if any - *(int*)currentPtr = currentLen; - currentPtr = null; - } } private unsafe delegate* unmanaged[Cdecl] ReadCallbackPtr { get; } = &ReadCallbackUnmanaged; @@ -200,116 +180,89 @@ nint dataCallbackContext { // dataCallback takes: index, dataCallbackContext, data pointer, data length, and returns nothing - var enumerable = new VectorReadBatch(dataCallback, dataCallbackContext, context, numKeys, SpanByte.FromPinnedPointer((byte*)keysData, (int)keysLength)); - try - { - ref var ctx = ref ActiveThreadSession.vectorContext; + var enumerable = new VectorReadBatch(dataCallback, dataCallbackContext, context, numKeys, PinnedSpanByte.FromPinnedPointer((byte*)keysData, (int)keysLength)); - ctx.ReadWithPrefetch(ref enumerable); + ref var ctx = ref ActiveThreadSession.vectorBasicContext; - enumerable.CompletePending(ref ctx); - } - finally - { - enumerable.Dispose(); - } + ctx.ReadWithPrefetch(ref enumerable); + + enumerable.CompletePending(ref ctx); } [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] private static unsafe byte WriteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength, nint writeData, nuint writeLength) { - var keyWithNamespace = MarkDiskANNKeyWithNamespace(context, keyData, keyLength); - try - { - - ref var ctx = ref ActiveThreadSession.vectorContext; - VectorInput input = default; - var valueSpan = SpanByte.FromPinnedPointer((byte*)writeData, (int)writeLength); - SpanByte outputSpan = default; - - var status = ctx.Upsert(ref keyWithNamespace, ref input, ref valueSpan, ref outputSpan); - if (status.IsPending) - { - CompletePending(ref status, ref outputSpan, ref ctx); - } - - return status.IsCompletedSuccessfully ? (byte)1 : default; - } - finally + var keyWithNamespace = MakeVectorElementKey(context, keyData, keyLength); + ref var ctx = ref ActiveThreadSession.vectorBasicContext; + VectorInput input = new(); + input.AlignmentExpected = true; + var valueSpan = SpanByte.FromPinnedPointer((byte*)writeData, (int)writeLength); + VectorOutput outputSpan = new(); + + var status = ctx.Upsert(keyWithNamespace, ref input, valueSpan, ref outputSpan); + if (status.IsPending) { - UnmarkDiskANNKey(keyWithNamespace); + CompletePending(ref status, ref outputSpan, ref ctx); } + + return status.IsCompletedSuccessfully ? (byte)1 : default; } [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] - private static unsafe byte DeleteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength) + private static byte DeleteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength) { - var keyWithNamespace = MarkDiskANNKeyWithNamespace(context, keyData, keyLength); + var keyWithNamespace = MakeVectorElementKey(context, keyData, keyLength); - try - { - ref var ctx = ref ActiveThreadSession.vectorContext; + ref var ctx = ref ActiveThreadSession.vectorBasicContext; - var status = ctx.Delete(ref keyWithNamespace); - Debug.Assert(!status.IsPending, "Deletes should never go async"); + var status = ctx.Delete(keyWithNamespace); + Debug.Assert(!status.IsPending, "Deletes should never go async"); - return status.IsCompletedSuccessfully && status.Found ? (byte)1 : default; - } - finally - { - UnmarkDiskANNKey(keyWithNamespace); - } + return status.IsCompletedSuccessfully && status.Found ? (byte)1 : default; } [UnmanagedCallersOnly(CallConvs = [typeof(CallConvCdecl)])] - private static unsafe byte ReadModifyWriteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength, nuint writeLength, nint dataCallback, nint dataCallbackContext) + private static byte ReadModifyWriteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength, nuint writeLength, nint dataCallback, nint dataCallbackContext) { - var keyWithNamespace = MarkDiskANNKeyWithNamespace(context, keyData, keyLength); + var keyWithNamespace = MakeVectorElementKey(context, keyData, keyLength); - try - { - ref var ctx = ref ActiveThreadSession.vectorContext; - - VectorInput input = default; - input.Callback = dataCallback; - input.CallbackContext = dataCallbackContext; - input.WriteDesiredSize = (int)writeLength; - - var status = ctx.RMW(ref keyWithNamespace, ref input); - if (status.IsPending) - { - SpanByte ignored = default; + ref var ctx = ref ActiveThreadSession.vectorBasicContext; - CompletePending(ref status, ref ignored, ref ctx); - } + VectorInput input = default; + input.Callback = dataCallback; + input.CallbackContext = dataCallbackContext; + input.WriteDesiredSize = (int)writeLength; - return status.IsCompletedSuccessfully ? (byte)1 : default; - } - finally + var status = ctx.RMW(keyWithNamespace, ref input); + if (status.IsPending) { - UnmarkDiskANNKey(keyWithNamespace); + VectorOutput ignored = new(); + + CompletePending(ref status, ref ignored, ref ctx); } + + return status.IsCompletedSuccessfully ? (byte)1 : default; } - private static unsafe bool ReadSizeUnknown(ulong context, ReadOnlySpan key, ref SpanByteAndMemory value) + private static unsafe bool ReadSizeUnknown(ulong context, bool forceAlignment, ReadOnlySpan key, ref SpanByteAndMemory value) { - Span distinctKey = stackalloc byte[key.Length + 1]; - var keyWithNamespace = SpanByte.FromPinnedSpan(distinctKey); - keyWithNamespace.MarkNamespace(); - keyWithNamespace.SetNamespaceInPayload((byte)context); - key.CopyTo(keyWithNamespace.AsSpan()); + VectorElementKey keyWithNamespace = new((byte)context, key); - ref var ctx = ref ActiveThreadSession.vectorContext; + ref var ctx = ref ActiveThreadSession.vectorBasicContext; while (true) { VectorInput input = new(); input.ReadDesiredSize = -1; - fixed (byte* ptr = value.AsSpan()) + + // Sometimes we read DiskANN written data from the .NET side + // If that's the case, we need to pad for alignment even though .NET doesn't require it + input.AlignmentExpected = forceAlignment; + fixed (byte* ptr = value.Span) { - SpanByte asSpanByte = new(value.Length, (nint)ptr); + VectorOutput asSpanByte = new(ptr, value.Length); - var status = ctx.Read(ref keyWithNamespace, ref input, ref asSpanByte); + var status = ctx.Read(keyWithNamespace, ref input, ref asSpanByte); if (status.IsPending) { CompletePending(ref status, ref asSpanByte, ref ctx); @@ -321,7 +274,7 @@ private static unsafe bool ReadSizeUnknown(ulong context, ReadOnlySpan key return false; } - if (input.ReadDesiredSize > asSpanByte.Length) + if (input.ReadDesiredSize > asSpanByte.SpanByteAndMemory.Length) { value.Memory?.Dispose(); var newAlloc = MemoryPool.Shared.Rent(input.ReadDesiredSize); @@ -329,7 +282,7 @@ private static unsafe bool ReadSizeUnknown(ulong context, ReadOnlySpan key continue; } - value.Length = asSpanByte.Length; + value.Length = asSpanByte.SpanByteAndMemory.Length; return true; } } @@ -341,32 +294,12 @@ private static unsafe bool ReadSizeUnknown(ulong context, ReadOnlySpan key /// Attempts to do this in place. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe SpanByte MarkDiskANNKeyWithNamespace(ulong context, nint keyData, nuint keyLength) + internal static unsafe VectorElementKey MakeVectorElementKey(ulong context, nint keyData, nuint keyLength) { - // DiskANN guarantees we have 4-bytes worth of unused data right before the key - var keyPtr = (byte*)keyData; - var keyNamespaceByte = keyPtr - 1; - - // TODO: if/when namespace can be > 4-bytes, we'll need to copy here - - var keyWithNamespace = SpanByte.FromPinnedPointer(keyNamespaceByte, (int)(keyLength + 1)); - keyWithNamespace.MarkNamespace(); - keyWithNamespace.SetNamespaceInPayload((byte)context); - - return keyWithNamespace; - } + // NOTE: DiskANN guarantees we have 4-bytes worth of unused data right before the key, but we aren't using it currently + ReadOnlySpan keyBytes = new((byte*)keyData, (int)keyLength); - /// - /// Inverse of . - /// - /// Used so DiskANN can keep using the same buffer for multiple calls with the same keys. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void UnmarkDiskANNKey(SpanByte keyWithNamespace) - { - var expectedLen = keyWithNamespace.Length - 1; - var start = keyWithNamespace.ToPointerWithMetadata() - 3; - BinaryPrimitives.WriteInt32LittleEndian(new Span(start, 4), expectedLen); + return new((byte)context, keyBytes); } } } \ No newline at end of file diff --git a/libs/server/Resp/Vector/VectorManager.Cleanup.cs b/libs/server/Resp/Vector/VectorManager.Cleanup.cs index ae2bdd3c6f2..74bdbf9867a 100644 --- a/libs/server/Resp/Vector/VectorManager.Cleanup.cs +++ b/libs/server/Resp/Vector/VectorManager.Cleanup.cs @@ -2,12 +2,10 @@ // Licensed under the MIT license. using System; -using System.Buffers.Binary; using System.Collections.Frozen; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; +using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; using Garnet.common; @@ -17,9 +15,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// /// Methods related to cleaning up data after a Vector Set is deleted. /// @@ -28,7 +23,7 @@ public sealed partial class VectorManager /// /// Used as part of scanning post-index-delete to cleanup abandoned data. /// - private sealed class PostDropCleanupFunctions : IScanIteratorFunctions + private sealed class PostDropCleanupFunctions : IScanIteratorFunctions { private readonly StorageSession storageSession; private readonly FrozenSet contexts; @@ -39,24 +34,26 @@ public PostDropCleanupFunctions(StorageSession storageSession, HashSet co this.storageSession = storageSession; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public void OnException(Exception exception, long numberOfRecords) { } public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + /// + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - if (key.MetadataSize != 1) + if (!logRecord.HasNamespace) { // Not Vector Set, ignore cursorRecordResult = CursorRecordResult.Skip; return true; } - var ns = key.GetNamespaceInPayload(); - var pairedContext = (ulong)ns & ~(ContextStep - 1); + // TODO: Implement variable length namespace support + Debug.Assert(logRecord.Namespace.Length == 1, "Variable length namespaces not supported"); + + ulong ns = logRecord.Namespace[0]; + var pairedContext = ns & ~(ContextStep - 1); if (!contexts.Contains(pairedContext)) { // Vector Set, but not one we're scanning for @@ -65,11 +62,13 @@ public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata re } // Delete it - var status = storageSession.vectorContext.Delete(ref key, 0); + VectorElementKey toDeleteKey = new((byte)ns, logRecord.KeyBytes); + + var status = storageSession.vectorBasicContext.Delete(toDeleteKey, 0); if (status.IsPending) { - SpanByte ignored = default; - CompletePending(ref status, ref ignored, ref storageSession.vectorContext); + VectorOutput ignored = new(); + CompletePending(ref status, ref ignored, ref storageSession.vectorBasicContext); } cursorRecordResult = CursorRecordResult.Accept; @@ -78,29 +77,56 @@ public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata re } private readonly Channel cleanupTaskChannel; + private readonly Channel<(ulong Context, TaskCompletionSource MarkCompleted)> requestCleanupTaskChannel; private readonly Task cleanupTask; + private readonly Task requestCleanupTask; private readonly Func getCleanupSession; - private async Task RunCleanupTaskAsync() + // Pause / resume coordination for the cleanup task vs concurrent Reset. + // + // Cluster re-attach paths (ReplicaDisklessSync / ReplicaDiskbasedSync) call + // storeWrapper.Reset() which tears down and rebuilds the main-store allocator. + // The cleanup task's iterator path is safe (Tsavorite's Initializing flag causes + // it to terminate cleanly). However the cleanup task ALSO does post-iterate RMWs + // on metadata records (ClearDeleteInProgress / UpdateContextMetadata) — those + // RMWs are NOT Reset-resilient and can dereference freed pagePointers and AVE. + // + // The pause/resume API serializes the entire cleanup-iteration (iterate + RMWs) + // with Reset by holding cleanupGate around the whole loop body, restoring Reset's + // documented "store is quiesced" contract. + // + // SemaphoreSlim used as an async-friendly mutex (initialCount=1, maxCount=1): + // the cleanup loop takes it around each iteration; PauseCleanupAsync takes it + // and holds until ResumeCleanup releases. Drops still enqueue items into + // cleanupTaskChannel during a pause — the cleanup task wakes, awaits the gate + // until the pause is lifted, then processes the backlog. + // + // Contract: PauseCleanupAsync callers MUST balance every successful invocation + // with ResumeCleanup, ideally in a finally block. A held pause at Dispose time + // would deadlock shutdown. + private readonly SemaphoreSlim cleanupGate = new(initialCount: 1, maxCount: 1); + + /// + /// Seaparate task thas allows for marking Vector Sets contexts as needing cleanup. + /// + /// Cleanup is actually done by the . + /// + /// Separating the two states allows for durable deletion logic, as we can block + /// deletion of Vector Sets until the context is marked as needing deletion. + /// + private async Task RunRequestCleanupTaskAsync() { - // Each drop index will queue a null object here - // We'll handle multiple at once if possible, but using a channel simplifies cancellation and dispose - await foreach (var ignored in cleanupTaskChannel.Reader.ReadAllAsync()) + while (await requestCleanupTaskChannel.Reader.WaitToReadAsync().ConfigureAwait(false)) { - try - { - HashSet needCleanup; - lock (this) - { - needCleanup = contextMetadata.GetNeedCleanup(); - } + // We do not need to take the cleanupGate here because we block in an OnDispose callback + // for this task to make progress. + // + // The fact that we're in an OnDispose means Reset() isn't running. - if (needCleanup == null) - { - // Previous run already got here, so bail - continue; - } + var completions = new List(); + try + { // TODO: this doesn't work with non-RESP impls... which maybe we don't care about? using var cleanupSession = (RespServerSession)getCleanupSession(); if (cleanupSession.activeDbId != dbId && !cleanupSession.TrySwitchActiveDatabaseSession(dbId)) @@ -108,302 +134,167 @@ private async Task RunCleanupTaskAsync() throw new GarnetException($"Could not switch VectorManager cleanup session to {dbId}, initialization failed"); } - PostDropCleanupFunctions callbacks = new(cleanupSession.storageSession, needCleanup); - - ref var ctx = ref cleanupSession.storageSession.vectorContext; + ref var delCtx = ref cleanupSession.storageSession.vectorBasicContext; - // Scan whole keyspace (sigh) and remove any associated data - // - // We don't really have a choice here, just do it - _ = ctx.Session.Iterate(ref callbacks); - - // Key is mostly ignored when deleting from InProgressDeletes - // So we just need a non-empty one to use with the context - Span basicKeySpan = new byte[1]; - unsafe + var needsUpdate = false; + lock (this) { - fixed (byte* basicKeyPtr = basicKeySpan) + // Read all pending requests so we can do one update + while (requestCleanupTaskChannel.Reader.TryRead(out var t)) { - var basicKey = SpanByte.FromPinnedPointer(basicKeyPtr, basicKeySpan.Length); - - // Generally there will already be removed, but if deletes fail in odd spots there can - // be a little bit to cleanup - so go ahead and do it. - // - // Not really worth optimizing given that we just scanned the whole key space to remove elements - // and that will dominate. - foreach (var cleanedUp in needCleanup) + if (t.MarkCompleted != null) { - ClearDeleteInProgress(ref ctx, ref basicKey, cleanedUp); + completions.Add(t.MarkCompleted); + } + + if (!contextMetadata.IsCleaningUp(t.Context)) + { + contextMetadata.MarkCleaningUp(t.Context); + + needsUpdate = true; } } } - lock (this) + if (needsUpdate) { - foreach (var cleanedUp in needCleanup) + UpdateContextMetadata(ref delCtx); + } + + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_3); + + foreach (var completion in completions) + { + try { - contextMetadata.FinishedCleaningUp(cleanedUp); + _ = completion.TrySetResult(); + } + catch (Exception innerE) + { + logger?.LogError(innerE, "While completing Vector Set cleanup request"); } } - UpdateContextMetadata(ref ctx); + // Pump the cleanup task once we're done + _ = cleanupTaskChannel.Writer.TryWrite(null); } catch (Exception e) { - logger?.LogError(e, "Failure during background cleanup of deleted vector sets, implies storage leak"); + foreach (var completion in completions) + { + try + { + _ = completion.TrySetException(e); + } + catch (Exception innerE) + { + // Best effort + logger?.LogError(innerE, "While cancelling Vector Set cleanup requests"); + } + } } } } /// - /// Called in response to or to update metadata in Tsavorite. + /// Perform cleanup of deleted Vector Set element keys. /// - /// Returns false if there is insufficient size for the value. + /// What needs cleanup is tracked as part of . /// - internal static bool TryUpdateInProgressDeletes(Span updateMessage, ref SpanByte inLogValue, ref RecordInfo recordInfo, ref RMWInfo rmwInfo) + private async Task RunCleanupTaskAsync() { - var context = BinaryPrimitives.ReadUInt64LittleEndian(updateMessage); - var len = BinaryPrimitives.ReadInt32LittleEndian(updateMessage[sizeof(ulong)..]); - var isAdding = len > 0; - var key = updateMessage[(sizeof(ulong) + sizeof(int))..]; - - Debug.Assert(key.Length == (isAdding ? len : -len), "Key length not expected"); - Debug.Assert(context is >= ContextStep, "Special context not allowed"); - - var remaining = inLogValue.AsSpan(); - while (remaining.Length >= sizeof(ulong) + sizeof(int)) + // Each drop index will queue a null object here + // We'll handle multiple at once if possible, but using a channel simplifies cancellation and dispose + await foreach (var ignored in cleanupTaskChannel.Reader.ReadAllAsync().ConfigureAwait(false)) { - var curCtx = BinaryPrimitives.ReadUInt64LittleEndian(remaining); + await cleanupGate.WaitAsync().ConfigureAwait(false); - if (curCtx == 0) - { - // Reached uninitialized data - break; - } - - var curLen = BinaryPrimitives.ReadInt32LittleEndian(remaining[sizeof(ulong)..]); - if (curCtx == context) + try { - if (isAdding) + // TODO: this doesn't work with non-RESP impls... which maybe we don't care about? + using var cleanupSession = (RespServerSession)getCleanupSession(); + if (cleanupSession.activeDbId != dbId && !cleanupSession.TrySwitchActiveDatabaseSession(dbId)) { - // Already added, ignore and make no other changes - return true; + throw new GarnetException($"Could not switch VectorManager cleanup session to {dbId}, initialization failed"); } - // Copy later values to cover the one we're removing - var afterCur = remaining[(sizeof(ulong) + sizeof(int) + curLen)..]; - afterCur.CopyTo(remaining); - - // Clear everything after that so we won't think it's valid - remaining[^(sizeof(ulong) + sizeof(int) + curLen)..].Clear(); - - // Shrink record by removed chunk size - var newSize = inLogValue.TotalSize - (sizeof(ulong) + sizeof(int) + curLen); - rmwInfo.ClearExtraValueLength(ref recordInfo, ref inLogValue, inLogValue.TotalSize); - inLogValue.ShrinkSerializedLength(inLogValue.TotalSize - newSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref inLogValue, inLogValue.TotalSize); - - return true; - } - - remaining = remaining[(sizeof(ulong) + sizeof(int) + curLen)..]; - } - - if (isAdding) - { - if (remaining.Length < sizeof(ulong) + sizeof(int) + key.Length) - { - return false; - } - - // Not already added, so slap it in - BinaryPrimitives.WriteUInt64LittleEndian(remaining, context); - BinaryPrimitives.WriteInt32LittleEndian(remaining[sizeof(ulong)..], len); - - key.CopyTo(remaining[(sizeof(ulong) + sizeof(int))..]); - - remaining = remaining[(sizeof(ulong) + sizeof(int) + key.Length)..]; - - // Record used length - var newSize = inLogValue.TotalSize - remaining.Length; - rmwInfo.ClearExtraValueLength(ref recordInfo, ref inLogValue, inLogValue.TotalSize); - inLogValue.ShrinkSerializedLength(newSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref inLogValue, inLogValue.TotalSize); - } - - return true; - } - - /// - /// Before we start smashing a for deletion, records that we started to delete it so we can recover from crashes. - /// - internal bool TryMarkDeleteInProgress(ref TContext ctx, ref SpanByte key, ulong context) - where TContext : ITsavoriteContext - { - Span keySpan = stackalloc byte[2]; - - Span dataSpan = stackalloc byte[sizeof(ulong) + sizeof(int) + key.Length]; - BinaryPrimitives.WriteUInt64LittleEndian(dataSpan, context); - - // Positive length indicates we're adding this to the list - BinaryPrimitives.WriteInt32LittleEndian(dataSpan[sizeof(ulong)..], key.LengthWithoutMetadata); - key.AsReadOnlySpan().CopyTo(dataSpan[(sizeof(ulong) + sizeof(int))..]); - - // 0:0 is ContextMetadata - // 0:1 is InProgressDeletes - var inProgressDeletesKey = SpanByte.FromPinnedSpan(keySpan); - - inProgressDeletesKey.MarkNamespace(); - inProgressDeletesKey.SetNamespaceInPayload(0); - inProgressDeletesKey.AsSpan()[0] = 1; - - VectorInput input = default; - input.Callback = 0; - - // Negative to indicate dynamic-ness - input.WriteDesiredSize = -(sizeof(ulong) + sizeof(int) + key.Length); - unsafe - { - input.CallbackContext = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(dataSpan)); - } - - var status = ctx.RMW(ref inProgressDeletesKey, ref input); + // Scan context needs to know how to handle objects and all callbacks, while VectorSessionFunctions is intentionally kept svelte + // + // So we use to different contexts, one to scan (strings) and one to delete (vectors) + ref var scanCtx = ref cleanupSession.storageSession.stringBasicContext; + ref var delCtx = ref cleanupSession.storageSession.vectorBasicContext; - if (status.IsPending) - { - SpanByte ignored = default; - CompletePending(ref status, ref ignored, ref ctx); - } + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_1); - return status.IsCompletedSuccessfully; - } + HashSet needCleanup; + lock (this) + { + needCleanup = contextMetadata.GetNeedCleanup(); + } - /// - /// Enumerate any deletes of Vector Sets that are in progress. - /// - /// Used with and to recover from interrupted deletes. - /// - internal List<(ReadOnlyMemory Key, ulong Context)> GetDeletesInProgress(StorageSession storageSession) - { - Span keySpan = stackalloc byte[1]; + if (needCleanup == null) + { + // Previous run already got here, so bail + continue; + } - // 0:1 is InProgressDeletes, but ReadSizeUnknown will attach the context for us - var inProgressDeletesKey = SpanByte.FromPinnedSpan(keySpan); + PostDropCleanupFunctions callbacks = new(cleanupSession.storageSession, needCleanup); - inProgressDeletesKey.AsSpan()[0] = 1; + // Scan whole keyspace and remove any associated data using a snapshot + // lookup-based push iterator. This avoids building a parallel tempKv (which + // would cost memory proportional to the keyspace) — IterateLookupSnapshot + // walks the log and uses hash-chain liveness checks bounded to the snapshot's + // TailAddress, so concurrent RCUs don't drop records. + _ = scanCtx.Session.IterateLookupSnapshot(ref callbacks); - SpanByteAndMemory readValue = default; + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_2); - List<(ReadOnlyMemory Key, ulong Context)> ret = []; - try - { - ActiveThreadSession = storageSession; - try - { - if (!ReadSizeUnknown(context: 0, keySpan, ref readValue)) + lock (this) { - return ret; + foreach (var cleanedUp in needCleanup) + { + contextMetadata.FinishedCleaningUp(cleanedUp); + } } + + UpdateContextMetadata(ref delCtx); } - finally + catch (Exception e) { - ActiveThreadSession = null; + logger?.LogError(e, "Failure during background cleanup of deleted vector sets, implies storage leak"); } - - var remaining = readValue.AsReadOnlySpan(); - while (remaining.Length >= sizeof(ulong) + sizeof(int)) + finally { - var ctx = BinaryPrimitives.ReadUInt64LittleEndian(remaining); - if (ctx == 0) - { - // Encountered uninitialized data - break; - } - - var len = BinaryPrimitives.ReadInt32LittleEndian(remaining[sizeof(ulong)..]); - - var key = remaining.Slice(sizeof(ulong) + sizeof(int), len); - - ret.Add((key.ToArray(), ctx)); - - remaining = remaining[(sizeof(ulong) + sizeof(int) + len)..]; + _ = cleanupGate.Release(); } - - return ret; - } - finally - { - readValue.Memory?.Dispose(); - } - } - - /// - /// After a delete has completed, removes the given key from metadata. - /// - internal void ClearDeleteInProgress(ref TContext ctx, ref SpanByte key, ulong context) - where TContext : ITsavoriteContext - { - Span keySpan = stackalloc byte[2]; - - Span dataSpan = stackalloc byte[sizeof(ulong) + sizeof(int) + key.Length]; - BinaryPrimitives.WriteUInt64LittleEndian(dataSpan, context); - - // Negative length indicates we're removing this from the list - BinaryPrimitives.WriteInt32LittleEndian(dataSpan[sizeof(ulong)..], -key.LengthWithoutMetadata); - key.AsReadOnlySpan().CopyTo(dataSpan[(sizeof(ulong) + sizeof(int))..]); - - // 0:0 is ContextMetadata - // 0:1 is InProgressDeletes - var inProgressDeletesKey = SpanByte.FromPinnedSpan(keySpan); - - inProgressDeletesKey.MarkNamespace(); - inProgressDeletesKey.SetNamespaceInPayload(0); - inProgressDeletesKey.AsSpan()[0] = 1; - - VectorInput input = default; - input.Callback = 0; - - // Negative to indicate dynamic-ness - input.WriteDesiredSize = -(sizeof(ulong) + sizeof(int) + key.Length); - unsafe - { - input.CallbackContext = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(dataSpan)); - } - - var status = ctx.RMW(ref inProgressDeletesKey, ref input); - - if (status.IsPending) - { - SpanByte ignored = default; - CompletePending(ref status, ref ignored, ref ctx); } } /// - /// After an index is dropped, called to start the process of removing ancillary data (elements, neighbor lists, attributes, etc.). + /// Block any new cleanup-task iteration from starting and wait for the current one + /// (if any) to finish. Callers (e.g., cluster re-attach paths) MUST balance every + /// invocation with , ideally in a finally block. + /// + /// While paused, drops still enqueue items into ; + /// the cleanup task wakes, awaits the gate until the pause is lifted, then + /// processes the backlog — so no work is lost. + /// + /// Use this before invoking on a running store, to + /// avoid the cleanup-task scan iterator racing with the allocator teardown. + /// + /// The optional aborts the wait if the cleanup + /// task is mid-iteration over a large keyspace and the caller (e.g., cluster + /// re-attach) needs to give up. If cancellation throws , + /// the gate was NOT acquired and the caller MUST NOT call . /// - internal void CleanupDroppedIndex(ref TContext ctx, ulong context) - where TContext : ITsavoriteContext - { - lock (this) - { - contextMetadata.MarkCleaningUp(context); - } - - UpdateContextMetadata(ref ctx); - - // Wake up cleanup task - var writeRes = cleanupTaskChannel.Writer.TryWrite(null); - Debug.Assert(writeRes, "Request for cleanup failed, this should never happen"); - } + public Task PauseCleanupAsync(CancellationToken cancellationToken = default) + => cleanupGate.WaitAsync(cancellationToken); /// - /// Detects if a Vector Set index read out of the main store is in the middle of being deleted. + /// Lift the pause acquired by . Queued cleanup + /// events resume processing immediately. Must be called exactly once per + /// successful PauseCleanupAsync — typically from a finally block. /// - private static bool PartiallyDeleted(ReadOnlySpan indexConfig) - { - ReadIndex(indexConfig, out var context, out _, out _, out _, out _, out _, out _, out _, out _); - return context == 0; - } + public void ResumeCleanup() => cleanupGate.Release(); } } \ No newline at end of file diff --git a/libs/server/Resp/Vector/VectorManager.ContextMetadata.cs b/libs/server/Resp/Vector/VectorManager.ContextMetadata.cs index 62f338ac4c6..d9d052b1185 100644 --- a/libs/server/Resp/Vector/VectorManager.ContextMetadata.cs +++ b/libs/server/Resp/Vector/VectorManager.ContextMetadata.cs @@ -11,13 +11,9 @@ using System.Threading; using Garnet.common; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// /// Methods for managing , which tracks process wide /// information about different contexts. @@ -81,6 +77,18 @@ public readonly bool IsMigrating(ulong context) return (migrating & mask) != 0; } + public readonly bool IsCleaningUp(ulong context) + { + Debug.Assert(context > 0, "Context 0 is reserved, should never queried"); + Debug.Assert((context % ContextStep) == 0, "Should only consider whole block of context, not a sub-bit"); + Debug.Assert(context <= byte.MaxValue, "Context larger than expected"); + + var bitIx = context / ContextStep; + var mask = 1UL << (byte)bitIx; + + return (cleaningUp & mask) == mask; + } + public readonly HashSet GetNamespacesForHashSlots(HashSet hashSlots) { HashSet ret = null; @@ -120,9 +128,15 @@ public readonly HashSet GetNamespacesForHashSlots(HashSet hashSlots) public readonly ulong NextNotInUse() { - var ignoringZero = inUse | 1; + var ignoringUnusuable = inUse; - var bit = (ulong)BitOperations.TrailingZeroCount(~ignoringZero & (ulong)-(long)(~ignoringZero)); + ignoringUnusuable |= 1; // Context 0 is reserved + + // We cannot use namespaces > 127 + // TODO: Once Variable length namespaces work, remove this constraint + ignoringUnusuable |= ~((1UL << 15) - 1); + + var bit = (ulong)BitOperations.TrailingZeroCount(~ignoringUnusuable & (ulong)-(long)(~ignoringUnusuable)); if (bit == 64) { @@ -233,6 +247,22 @@ public void MarkCleaningUp(ulong context) Version++; } + public void ClearIsCleaningUp(ulong context) + { + Debug.Assert(context > 0, "Context 0 is reserved, should never queried"); + Debug.Assert((context % ContextStep) == 0, "Should only consider whole block of context, not a sub-bit"); + Debug.Assert(context <= byte.MaxValue, "Context larger than expected"); + + var bitIx = context / ContextStep; + var mask = 1UL << (byte)bitIx; + + Debug.Assert((inUse & mask) != 0, "Should be in use if was marked for cleanup"); + Debug.Assert((cleaningUp & mask) != 0, "About to clear cleanup when not marked for cleanup"); + cleaningUp &= ~mask; + + Version++; + } + public void FinishedCleaningUp(ulong context) { Debug.Assert(context > 0, "Context 0 is reserved, should never queried"); @@ -328,6 +358,45 @@ public override readonly string ToString() } } + /// + /// Used to prevent new contexts from being issued during a FLUSHDB / FLUSHALL, as well as any new Vector Set operations from starting. + /// + /// Also updates and clears cached upon disposal. + /// + internal readonly struct FlushGuard : IDisposable + { + private readonly VectorManager manager; + + internal FlushGuard(VectorManager manager) + { + this.manager = manager; + + // Stop other Vector Set operations + this.manager.vectorSetLocks.AcquireAllExclusiveLock(); + + // Acquire a lock that will block all other attempts to issue a new context + Monitor.Enter(this.manager); + } + + /// + public readonly void Dispose() + { + if (manager == null) + { + // This is the default instance, ignore disposal + return; + } + + manager.contextMetadata = default; + + // Allow Vector Set operations again + manager.vectorSetLocks.ReleaseAllExclusiveLock(); + + // Allow new contexts to be issued + Monitor.Exit(manager); + } + } + private ContextMetadata contextMetadata; /// @@ -380,13 +449,30 @@ private ulong NextVectorSetContext(ushort hashSlot) } } + /// + /// During a FLUSHDB (or FLUSHALL) we need to prevent new contexts and other updates to context metadata. + /// + /// This method is called at the start of a flush and returns a guard instance which will block such + /// new creations until it is disposed. + /// + /// This is pretty expensive, but flush should be rare and is @slow anyway. + /// + internal FlushGuard BeginFlush() + { + if (!IsEnabled) + { + return default; + } + + return new FlushGuard(this); + } + /// /// Obtain some number of contexts for migrating Vector Sets. /// /// The return contexts are unavailable for other use, but are not yet "live" for visibility purposes. /// - public bool TryReserveContextsForMigration(ref TContext ctx, int count, out List contexts) - where TContext : ITsavoriteContext + public bool TryReserveContextsForMigration(ref VectorBasicContext ctx, int count, out List contexts) { lock (this) { @@ -405,10 +491,8 @@ public bool TryReserveContextsForMigration(ref TContext ctx, int count /// /// Called when an index creation succeeds to flush into the store. /// - private void UpdateContextMetadata(ref TContext ctx) - where TContext : ITsavoriteContext + private void UpdateContextMetadata(ref VectorBasicContext ctx) { - Span keySpan = stackalloc byte[1]; Span dataSpan = stackalloc byte[ContextMetadata.Size]; lock (this) @@ -416,10 +500,8 @@ private void UpdateContextMetadata(ref TContext ctx) MemoryMarshal.Cast(dataSpan)[0] = contextMetadata; } - var key = SpanByte.FromPinnedSpan(keySpan); - - key.MarkNamespace(); - key.SetNamespaceInPayload(0); + // empty is context metadata + VectorElementKey key = new(MetadataNamespace, []); VectorInput input = default; input.Callback = 0; @@ -429,11 +511,11 @@ private void UpdateContextMetadata(ref TContext ctx) input.CallbackContext = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(dataSpan)); } - var status = ctx.RMW(ref key, ref input); + var status = ctx.RMW(key, ref input); if (status.IsPending) { - SpanByte ignored = default; + VectorOutput ignored = new(); CompletePending(ref status, ref ignored, ref ctx); } } diff --git a/libs/server/Resp/Vector/VectorManager.Filter.cs b/libs/server/Resp/Vector/VectorManager.Filter.cs index 1a1eedacf9d..52b25ae8f43 100644 --- a/libs/server/Resp/Vector/VectorManager.Filter.cs +++ b/libs/server/Resp/Vector/VectorManager.Filter.cs @@ -178,7 +178,7 @@ internal static int ApplyPostFilter( } finally { - scratchBufferBuilder.RewindScratchBuffer(ref bufferSlice); + scratchBufferBuilder.RewindScratchBuffer(bufferSlice); } } diff --git a/libs/server/Resp/Vector/VectorManager.Index.cs b/libs/server/Resp/Vector/VectorManager.Index.cs index b82500c69f1..e06f3dc19a2 100644 --- a/libs/server/Resp/Vector/VectorManager.Index.cs +++ b/libs/server/Resp/Vector/VectorManager.Index.cs @@ -7,7 +7,6 @@ using System.Runtime.InteropServices; using Garnet.common; using Microsoft.Extensions.Logging; -using Tsavorite.core; namespace Garnet.server { @@ -19,6 +18,8 @@ namespace Garnet.server /// public sealed partial class VectorManager { + public const int IndexSize = Index.Size; + [StructLayout(LayoutKind.Explicit, Size = Size)] private struct Index { @@ -40,8 +41,12 @@ private struct Index public VectorQuantType QuantType; [FieldOffset(36)] public VectorDistanceMetricType DistanceMetric; + + // These used to be allocated for a GUID [FieldOffset(40)] - public Guid ProcessInstanceId; + private ulong unused0; + [FieldOffset(48)] + private ulong unused1; } /// @@ -56,12 +61,10 @@ internal void CreateIndex( VectorDistanceMetricType distanceMetric, ulong newContext, nint newIndexPtr, - ref SpanByte indexValue) + Span indexSpan) { AssertHaveStorageSession(); - var indexSpan = indexValue.AsSpan(); - Debug.Assert((newContext % 8) == 0 && newContext != 0, "Illegal context provided"); Debug.Assert(Unsafe.SizeOf() == Index.Size, "Constant index size is incorrect"); @@ -80,7 +83,6 @@ internal void CreateIndex( asIndex.NumLinks = numLinks; asIndex.DistanceMetric = distanceMetric; asIndex.IndexPtr = (ulong)newIndexPtr; - asIndex.ProcessInstanceId = processInstanceId; } /// @@ -88,24 +90,21 @@ internal void CreateIndex( /// /// This implies the index still has element data, but the pointer is garbage. /// - internal void RecreateIndex(nint newIndexPtr, ref SpanByte indexValue) + internal void RecreateIndex(nint newIndexPtr, Span indexSpan) { AssertHaveStorageSession(); - var indexSpan = indexValue.AsSpan(); - if (indexSpan.Length != Index.Size) { logger?.LogCritical("Acquired space for vector set index does not match expectations, {Length} != {Size}", indexSpan.Length, Index.Size); throw new GarnetException($"Acquired space for vector set index does not match expectations, {indexSpan.Length} != {Index.Size}"); } - ReadIndex(indexSpan, out var context, out _, out _, out _, out _, out _, out _, out _, out var indexProcessInstanceId); - Debug.Assert(processInstanceId != indexProcessInstanceId, "Shouldn't be recreating an index that matched our instance id"); + ReadIndex(indexSpan, out _, out _, out _, out _, out _, out _, out _, out var indexPtr); + Debug.Assert(indexPtr == 0, "Shouldn't be recreating an index if we already have a pointer"); ref var asIndex = ref Unsafe.As(ref MemoryMarshal.GetReference(indexSpan)); asIndex.IndexPtr = (ulong)newIndexPtr; - asIndex.ProcessInstanceId = processInstanceId; } /// @@ -113,11 +112,9 @@ internal void RecreateIndex(nint newIndexPtr, ref SpanByte indexValue) /// internal void DropIndex(ReadOnlySpan indexValue) { - AssertHaveStorageSession(); - - ReadIndex(indexValue, out var context, out _, out _, out _, out _, out _, out _, out var indexPtr, out var indexProcessInstanceId); + ReadIndex(indexValue, out var context, out _, out _, out _, out _, out _, out _, out var indexPtr); - if (indexProcessInstanceId != processInstanceId) + if (indexPtr == 0) { // We never actually spun this index up, so nothing to drop return; @@ -138,8 +135,7 @@ public static void ReadIndex( out uint buildExplorationFactor, out uint numLinks, out VectorDistanceMetricType distanceMetric, - out nint indexPtr, - out Guid processInstanceId + out nint indexPtr ) { Debug.Assert(indexValue.Length == Index.Size, $"Index size is incorrect ({indexValue.Length} != {Index.Size}), implies vector set index is probably corrupted"); @@ -154,7 +150,6 @@ out Guid processInstanceId numLinks = asIndex.NumLinks; distanceMetric = asIndex.DistanceMetric; indexPtr = (nint)asIndex.IndexPtr; - processInstanceId = asIndex.ProcessInstanceId; Debug.Assert((context % ContextStep) == 0, $"Context ({context}) not as expected (% 4 == {context % 4}), vector set index is probably corrupted"); } @@ -162,7 +157,7 @@ out Guid processInstanceId /// /// Update the context (which defines a range of namespaces) stored in a given index. /// - /// Doing this also smashes the ProcessInstanceId, so the destination node won't + /// Doing this also smashes the index pointer, so the destination node won't /// think it's already creating this index. /// public static void SetContextForMigration(Span indexValue, ulong newContext) @@ -173,7 +168,7 @@ public static void SetContextForMigration(Span indexValue, ulong newContex ref var asIndex = ref Unsafe.As(ref MemoryMarshal.GetReference(indexValue)); asIndex.Context = newContext; - asIndex.ProcessInstanceId = MigratedInstanceId; + asIndex.IndexPtr = 0; } } } \ No newline at end of file diff --git a/libs/server/Resp/Vector/VectorManager.Locking.cs b/libs/server/Resp/Vector/VectorManager.Locking.cs index 718232a9e5e..00fda41b874 100644 --- a/libs/server/Resp/Vector/VectorManager.Locking.cs +++ b/libs/server/Resp/Vector/VectorManager.Locking.cs @@ -88,9 +88,9 @@ public void Dispose() /// internal bool NeedsRecreate(ReadOnlySpan indexConfig) { - ReadIndex(indexConfig, out _, out _, out _, out _, out _, out _, out _, out _, out var indexProcessInstanceId); + ReadIndex(indexConfig, out _, out _, out _, out _, out _, out _, out _, out var indexPtr); - return indexProcessInstanceId != processInstanceId; + return indexPtr == 0; } /// @@ -100,7 +100,7 @@ internal bool NeedsRecreate(ReadOnlySpan indexConfig) /// /// Returns a disposable that prevents the index from being deleted while undisposed. /// - internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanByte key, ref RawStringInput input, scoped Span indexSpan, out GarnetStatus status) + internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ReadOnlySpan key, ref StringInput input, scoped Span indexSpan, out GarnetStatus status) { Debug.Assert(indexSpan.Length == IndexSizeBytes, "Insufficient space for index"); @@ -108,9 +108,9 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB ActiveThreadSession = storageSession; try { - var keyHash = storageSession.basicContext.GetKeyHash(ref key); + var keyHash = storageSession.stringBasicContext.GetKeyHash((FixedSpanByteKey)key); - var indexConfig = SpanByteAndMemory.FromPinnedSpan(indexSpan); + var indexConfigOutput = StringOutput.FromPinnedSpan(indexSpan); var readCmd = input.header.cmd; @@ -124,8 +124,8 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB GarnetStatus readRes; try { - readRes = storageSession.Read_MainStore(ref key, ref input, ref indexConfig, ref storageSession.basicContext); - Debug.Assert(indexConfig.IsSpanByte, "Should never need to move index onto the heap"); + readRes = storageSession.Read_MainStore(key, ref input, ref indexConfigOutput, ref storageSession.stringBasicContext); + Debug.Assert(indexConfigOutput.SpanByteAndMemory.IsSpanByte, "Should never need to move index onto the heap"); } catch { @@ -137,15 +137,7 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB bool needsRecreate; if (readRes == GarnetStatus.OK) { - if (PartiallyDeleted(indexConfig.AsReadOnlySpan())) - { - status = GarnetStatus.BADSTATE; - - vectorSetLocks.ReleaseSharedLock(sharedLockToken); - return default; - } - - needsRecreate = NeedsRecreate(indexConfig.AsReadOnlySpan()); + needsRecreate = NeedsRecreate(indexConfigOutput.SpanByteAndMemory.ReadOnlySpan); } else { @@ -162,7 +154,7 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB continue; } - ReadIndex(indexSpan, out var indexContext, out var dims, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out _, out _); + ReadIndex(indexSpan, out var indexContext, out var dims, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out _); input.arg1 = RecreateIndexArg; @@ -178,15 +170,15 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB input.parseState.EnsureCapacity(12); // Save off for recreation - input.parseState.SetArgument(10, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); // Strictly we don't _need_ this, but it keeps everything else aligned nicely - input.parseState.SetArgument(11, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); + input.parseState.SetArgument(10, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); // Strictly we don't _need_ this, but it keeps everything else aligned nicely + input.parseState.SetArgument(11, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); GarnetStatus writeRes; try { try { - writeRes = storageSession.RMW_MainStore(ref key, ref input, ref indexConfig, ref storageSession.basicContext); + writeRes = storageSession.RMW_MainStore(key, ref input, ref indexConfigOutput, ref storageSession.stringBasicContext); if (writeRes != GarnetStatus.OK) { @@ -252,8 +244,8 @@ internal ReadVectorLock ReadVectorIndex(StorageSession storageSession, ref SpanB /// internal ReadVectorLock ReadOrCreateVectorIndex( StorageSession storageSession, - ref SpanByte key, - ref RawStringInput input, + ReadOnlySpan key, + ref StringInput input, scoped Span indexSpan, out GarnetStatus status ) @@ -264,9 +256,9 @@ out GarnetStatus status ActiveThreadSession = storageSession; try { - var keyHash = storageSession.basicContext.GetKeyHash(ref key); + var keyHash = storageSession.stringBasicContext.GetKeyHash((FixedSpanByteKey)key); - var indexConfig = SpanByteAndMemory.FromPinnedSpan(indexSpan); + var indexConfigOutput = StringOutput.FromPinnedSpan(indexSpan); while (true) { @@ -277,8 +269,8 @@ out GarnetStatus status GarnetStatus readRes; try { - readRes = storageSession.Read_MainStore(ref key, ref input, ref indexConfig, ref storageSession.basicContext); - Debug.Assert(indexConfig.IsSpanByte, "Should never need to move index onto the heap"); + readRes = storageSession.Read_MainStore(key, ref input, ref indexConfigOutput, ref storageSession.stringBasicContext); + Debug.Assert(indexConfigOutput.SpanByteAndMemory.IsSpanByte, "Should never need to move index onto the heap"); } catch { @@ -290,15 +282,7 @@ out GarnetStatus status bool needsRecreate; if (readRes == GarnetStatus.OK) { - if (PartiallyDeleted(indexConfig.AsReadOnlySpan())) - { - status = GarnetStatus.BADSTATE; - - vectorSetLocks.ReleaseSharedLock(sharedLockToken); - return default; - } - - needsRecreate = NeedsRecreate(indexConfig.AsReadOnlySpan()); + needsRecreate = NeedsRecreate(indexConfigOutput.SpanByteAndMemory.ReadOnlySpan); } else { @@ -319,7 +303,7 @@ out GarnetStatus status nint newlyAllocatedIndex; if (needsRecreate) { - ReadIndex(indexSpan, out indexContext, out var dims, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out _, out _); + ReadIndex(indexSpan, out indexContext, out var dims, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out _); input.arg1 = RecreateIndexArg; @@ -331,8 +315,8 @@ out GarnetStatus status input.parseState.EnsureCapacity(12); // Save off for recreation - input.parseState.SetArgument(10, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); // Strictly we don't _need_ this, but it keeps everything else aligned nicely - input.parseState.SetArgument(11, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); + input.parseState.SetArgument(10, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); // Strictly we don't _need_ this, but it keeps everything else aligned nicely + input.parseState.SetArgument(11, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); } else { @@ -340,7 +324,7 @@ out GarnetStatus status // We must associate the index with a hash slot at creation time to enable future migrations // TODO: RENAME and friends need to also update this data - var slot = HashSlotUtils.HashSlot(ref key); + var slot = HashSlotUtils.HashSlot(key); indexContext = NextVectorSetContext(slot); @@ -363,8 +347,8 @@ out GarnetStatus status input.parseState.EnsureCapacity(12); // Save off for insertion - input.parseState.SetArgument(10, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); - input.parseState.SetArgument(11, ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); + input.parseState.SetArgument(10, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref indexContext, 1)))); + input.parseState.SetArgument(11, PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1)))); } GarnetStatus writeRes; @@ -372,18 +356,12 @@ out GarnetStatus status { try { - writeRes = storageSession.RMW_MainStore(ref key, ref input, ref indexConfig, ref storageSession.basicContext); + writeRes = storageSession.RMW_MainStore(key, ref input, ref indexConfigOutput, ref storageSession.stringBasicContext); if (writeRes != GarnetStatus.OK) { // Insertion failed, drop index Service.DropIndex(indexContext, newlyAllocatedIndex); - - // If the failure was for a brand new index, free up the context too - if (!needsRecreate) - { - CleanupDroppedIndex(ref ActiveThreadSession.vectorContext, indexContext); - } } } catch @@ -392,12 +370,6 @@ out GarnetStatus status { // Drop to avoid a leak on error Service.DropIndex(indexContext, newlyAllocatedIndex); - - // If the failure was for a brand new index, free up the context too - if (!needsRecreate) - { - CleanupDroppedIndex(ref ActiveThreadSession.vectorContext, indexContext); - } } throw; @@ -405,7 +377,7 @@ out GarnetStatus status if (!needsRecreate) { - UpdateContextMetadata(ref storageSession.vectorContext); + UpdateContextMetadata(ref storageSession.vectorBasicContext); } } catch @@ -455,9 +427,9 @@ out GarnetStatus status /// /// Acquire exclusive lock over a given key. /// - private ExclusiveVectorLock AcquireExclusiveLocks(StorageSession storageSession, ref SpanByte key) + private ExclusiveVectorLock AcquireExclusiveLocks(StorageSession storageSession, ReadOnlySpan key) { - var keyHash = storageSession.lockableContext.GetKeyHash(key); + var keyHash = storageSession.stringTransactionalContext.GetKeyHash((FixedSpanByteKey)key); vectorSetLocks.AcquireExclusiveLock(keyHash, out var exclusiveLockToken); @@ -466,23 +438,22 @@ private ExclusiveVectorLock AcquireExclusiveLocks(StorageSession storageSession, /// /// Utility method that will read vector set index out, and acquire exclusive locks to allow it to be deleted. - /// - /// If the index is partially deleted, will be set to but the locks will be still acquired. /// - internal ExclusiveVectorLock ReadForDeleteVectorIndex(StorageSession storageSession, ref SpanByte key, ref RawStringInput input, scoped Span indexSpan, out GarnetStatus status) + internal ExclusiveVectorLock ReadForDeleteVectorIndex(StorageSession storageSession, ReadOnlySpan key, ref StringInput input, scoped Span indexSpan, out GarnetStatus status) { Debug.Assert(indexSpan.Length == IndexSizeBytes, "Insufficient space for index"); Debug.Assert(ActiveThreadSession == null, "Shouldn't enter context when already in one"); ActiveThreadSession = storageSession; - var indexConfig = SpanByteAndMemory.FromPinnedSpan(indexSpan); + var indexConfigOutput = StringOutput.FromPinnedSpan(indexSpan); // Get the index - var acquiredLock = AcquireExclusiveLocks(storageSession, ref key); + var acquiredLock = AcquireExclusiveLocks(storageSession, key); try { - status = storageSession.Read_MainStore(ref key, ref input, ref indexConfig, ref storageSession.basicContext); + status = storageSession.Read_MainStore(key, ref input, ref indexConfigOutput, ref storageSession.stringBasicContext); + Debug.Assert(indexConfigOutput.SpanByteAndMemory.IsSpanByte, "Should never need to move index onto the heap"); } catch { @@ -491,15 +462,6 @@ internal ExclusiveVectorLock ReadForDeleteVectorIndex(StorageSession storageSess throw; } - if (status == GarnetStatus.OK) - { - // Even if we read the value, it might be in a bad state due to a prior delete - if (PartiallyDeleted(indexConfig.AsReadOnlySpan())) - { - status = GarnetStatus.BADSTATE; - } - } - return acquiredLock; } } diff --git a/libs/server/Resp/Vector/VectorManager.Migration.cs b/libs/server/Resp/Vector/VectorManager.Migration.cs index 69235d18aca..a3dfe8775b5 100644 --- a/libs/server/Resp/Vector/VectorManager.Migration.cs +++ b/libs/server/Resp/Vector/VectorManager.Migration.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; @@ -11,9 +12,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// /// Methods related to migrating Vector Sets between different primaries. /// @@ -21,12 +19,6 @@ namespace Garnet.server /// public sealed partial class VectorManager { - // This is a V8 GUID based on 'GARNET MIGRATION' ASCII string - // It cannot collide with processInstanceIds because it's v8 - // It's unlikely other projects will select the value, so it's unlikely to collide with other v8s - // If it ends up in logs, it's ASCII equivalent looks suspicious enough to lead back here - private static readonly Guid MigratedInstanceId = new("4e524147-5445-8d20-8947-524154494f4e"); - /// /// Called to handle a key in a namespace being received during a migration. /// @@ -35,19 +27,19 @@ public sealed partial class VectorManager /// The index is handled specially by . /// public void HandleMigratedElementKey( - ref BasicContext basicCtx, - ref BasicContext vectorCtx, - ref SpanByte key, - ref SpanByte value + ref StringBasicContext basicCtx, + ref VectorBasicContext vectorCtx, + ReadOnlySpan namespaceBytes, + ReadOnlySpan keyWithoutNamespace, + ReadOnlySpan value ) { - Debug.Assert(key.MetadataSize == 1, "Should have namespace if we're migrating a key"); - + Debug.Assert(namespaceBytes.Length == 1 && namespaceBytes[0] <= 127, "Larger than byte namespaces not yet supported"); #if DEBUG // Do some extra sanity checking in DEBUG builds lock (this) { - var ns = key.GetNamespaceInPayload(); + var ns = (byte)namespaceBytes[0]; var context = (ulong)(ns & ~(ContextStep - 1)); Debug.Assert(contextMetadata.IsInUse(context), "Shouldn't be migrating to an unused context"); Debug.Assert(contextMetadata.IsMigrating(context), "Shouldn't be migrating to context not marked for it"); @@ -56,9 +48,12 @@ ref SpanByte value #endif VectorInput input = default; - SpanByte outputSpan = default; + input.AlignmentExpected = true; + VectorOutput outputSpan = new(new SpanByteAndMemory()); + + VectorElementKey key = new((byte)namespaceBytes[0], keyWithoutNamespace); - var status = vectorCtx.Upsert(ref key, ref input, ref value, ref outputSpan); + var status = vectorCtx.Upsert(key, ref input, value, ref outputSpan); if (status.IsPending) { CompletePending(ref status, ref outputSpan, ref vectorCtx); @@ -69,22 +64,29 @@ ref SpanByte value throw new GarnetException("Failed to migrate key, this should fail migration"); } - ReplicateMigratedElementKey(ref basicCtx, ref key, ref value, logger); + ReplicateMigratedElementKey(ref basicCtx, key, value, logger); // Fake a write for post-migration replication - static void ReplicateMigratedElementKey(ref BasicContext basicCtx, ref SpanByte key, ref SpanByte value, ILogger logger) + static void ReplicateMigratedElementKey(ref StringBasicContext basicCtx, VectorElementKey key, ReadOnlySpan value, ILogger logger) { - RawStringInput input = default; + StringInput input = default; + + // Serialize namespace and key data explicitly, we'll deserialize it in HandleVectorSetAddReplication + Span serializedKeyBytes = stackalloc byte[sizeof(int) + key.NamespaceBytes.Length + sizeof(int) + key.KeyBytes.Length]; + BinaryPrimitives.WriteInt32LittleEndian(serializedKeyBytes, key.NamespaceBytes.Length); + key.NamespaceBytes.CopyTo(serializedKeyBytes[sizeof(int)..]); + BinaryPrimitives.WriteInt32LittleEndian(serializedKeyBytes[(sizeof(int) + key.NamespaceBytes.Length)..], key.KeyBytes.Length); + key.KeyBytes.CopyTo(serializedKeyBytes[(sizeof(int) + key.NamespaceBytes.Length + sizeof(int))..]); input.header.cmd = RespCommand.VADD; input.arg1 = MigrateElementKeyLogArg; - input.parseState.InitializeWithArguments([ArgSlice.FromPinnedSpan(key.AsReadOnlySpanWithMetadata()), ArgSlice.FromPinnedSpan(value.AsReadOnlySpan())]); + input.parseState.InitializeWithArguments([PinnedSpanByte.FromPinnedSpan(serializedKeyBytes), PinnedSpanByte.FromPinnedSpan(value)]); - SpanByte dummyKey = default; - SpanByteAndMemory dummyOutput = default; + ReadOnlySpan dummyKey = []; + StringOutput dummyOutput = new(); - var res = basicCtx.RMW(ref dummyKey, ref input, ref dummyOutput); + var res = basicCtx.RMW((FixedSpanByteKey)dummyKey, ref input, ref dummyOutput); if (res.IsPending) { @@ -98,7 +100,7 @@ static void ReplicateMigratedElementKey(ref BasicContext basicCtx) + static void CompletePending(ref Status status, ref StringOutput output, ref StringBasicContext basicCtx) { _ = basicCtx.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); @@ -122,18 +124,16 @@ static void CompletePending(ref Status status, ref SpanByteAndMemory output, ref public void HandleMigratedIndexKey( GarnetDatabase db, StoreWrapper storeWrapper, - ref SpanByte key, - ref SpanByte value) + ReadOnlySpan key, + ReadOnlySpan value) { - Debug.Assert(key.MetadataSize != 1, "Shouldn't have a namespace if we're migrating a Vector Set index"); - - RawStringInput input = default; + StringInput input = default; input.header.cmd = RespCommand.VADD; input.arg1 = RecreateIndexArg; - ReadIndex(value.AsReadOnlySpan(), out var context, out var dimensions, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out _, out var processInstanceId); + ReadIndex(value, out var context, out var dimensions, out var reduceDims, out var quantType, out var buildExplorationFactor, out var numLinks, out var distanceMetric, out var indexPtr); - Debug.Assert(processInstanceId == MigratedInstanceId, "Shouldn't receive a real process instance id during a migration"); + Debug.Assert(indexPtr == 0, "Shouldn't receive an index pointer during a migration"); // Extra validation in DEBUG #if DEBUG @@ -151,7 +151,7 @@ public void HandleMigratedIndexKey( Debug.Assert(db != null, "Must have DB if session is not already set"); Debug.Assert(storeWrapper != null, "Must have StoreWrapper if session is not already set"); - ActiveThreadSession = newStorageSession = new StorageSession(storeWrapper, new(), null, null, db.Id, this, this.logger); + ActiveThreadSession = newStorageSession = new StorageSession(storeWrapper, new(), new(), null, null, db.Id, null, this, this.logger); } else { @@ -161,16 +161,16 @@ public void HandleMigratedIndexKey( try { // Prepare as a psuedo-VADD - var dimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dimensions, 1))); - var reduceDimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); - ArgSlice valueTypeArg = default; - ArgSlice valuesArg = default; - ArgSlice elementArg = default; - var quantizerArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantType, 1))); - var buildExplorationFactorArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); - ArgSlice attributesArg = default; - var numLinksArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); - var distanceMetricArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); + var dimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dimensions, 1))); + var reduceDimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); + PinnedSpanByte valueTypeArg = default; + PinnedSpanByte valuesArg = default; + PinnedSpanByte elementArg = default; + var quantizerArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantType, 1))); + var buildExplorationFactorArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); + PinnedSpanByte attributesArg = default; + var numLinksArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); + var distanceMetricArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); nint newlyAllocatedIndex; unsafe @@ -178,37 +178,42 @@ public void HandleMigratedIndexKey( newlyAllocatedIndex = Service.RecreateIndex(context, dimensions, reduceDims, quantType, buildExplorationFactor, numLinks, distanceMetric, ReadCallbackPtr, WriteCallbackPtr, DeleteCallbackPtr, ReadModifyWriteCallbackPtr); } - var ctxArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref context, 1))); - var indexArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1))); + var ctxArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref context, 1))); + var indexArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref newlyAllocatedIndex, 1))); input.parseState.InitializeWithArguments([dimsArg, reduceDimsArg, valueTypeArg, valuesArg, elementArg, quantizerArg, buildExplorationFactorArg, attributesArg, numLinksArg, distanceMetricArg, ctxArg, indexArg]); Span indexSpan = stackalloc byte[Index.Size]; var indexConfig = SpanByteAndMemory.FromPinnedSpan(indexSpan); + StringOutput indexConfigOutput = new(indexConfig); // Exclusive lock to prevent other modification of this key - using (AcquireExclusiveLocks(ActiveThreadSession, ref key)) + using (AcquireExclusiveLocks(ActiveThreadSession, key)) { // Perform the write - var writeRes = ActiveThreadSession.RMW_MainStore(ref key, ref input, ref indexConfig, ref ActiveThreadSession.basicContext); + var writeRes = ActiveThreadSession.RMW_MainStore(key, ref input, ref indexConfigOutput, ref ActiveThreadSession.stringBasicContext); if (writeRes != GarnetStatus.OK) { + indexConfigOutput.SpanByteAndMemory.Memory?.Dispose(); + Service.DropIndex(context, newlyAllocatedIndex); throw new GarnetException("Failed to import migrated Vector Set index, aborting migration"); } - var hashSlot = HashSlotUtils.HashSlot(ref key); + Debug.Assert(indexConfigOutput.SpanByteAndMemory.IsSpanByte, "Should never allocate"); + + var hashSlot = HashSlotUtils.HashSlot(key); lock (this) { contextMetadata.MarkMigrationComplete(context, hashSlot); } - UpdateContextMetadata(ref ActiveThreadSession.vectorContext); + UpdateContextMetadata(ref ActiveThreadSession.vectorBasicContext); // For REPLICAs which are following, we need to fake up a write - ReplicateMigratedIndexKey(ref ActiveThreadSession.basicContext, ref key, ref value, context, logger); + ReplicateMigratedIndexKey(ref ActiveThreadSession.stringBasicContext, key, value, context, logger); } } finally @@ -221,25 +226,25 @@ public void HandleMigratedIndexKey( // Fake a write for post-migration replication static void ReplicateMigratedIndexKey( - ref BasicContext basicCtx, - ref SpanByte key, - ref SpanByte value, + ref StringBasicContext basicCtx, + ReadOnlySpan key, + ReadOnlySpan value, ulong context, ILogger logger) { - RawStringInput input = default; + StringInput input = default; input.header.cmd = RespCommand.VADD; input.arg1 = MigrateIndexKeyLogArg; - var contextArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref context, 1))); + var contextArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref context, 1))); - input.parseState.InitializeWithArguments([ArgSlice.FromPinnedSpan(key.AsReadOnlySpanWithMetadata()), ArgSlice.FromPinnedSpan(value.AsReadOnlySpan()), contextArg]); + input.parseState.InitializeWithArguments([PinnedSpanByte.FromPinnedSpan(key), PinnedSpanByte.FromPinnedSpan(value), contextArg]); - SpanByte dummyKey = default; - SpanByteAndMemory dummyOutput = default; + var dummyKey = (FixedSpanByteKey)default(ReadOnlySpan); + StringOutput dummyOutput = new(); - var res = basicCtx.RMW(ref dummyKey, ref input, ref dummyOutput); + var res = basicCtx.RMW(dummyKey, ref input, ref dummyOutput); if (res.IsPending) { @@ -253,7 +258,7 @@ static void ReplicateMigratedIndexKey( } // Helper to complete read/writes during vector set synthetic op goes async - static void CompletePending(ref Status status, ref SpanByteAndMemory output, ref BasicContext basicCtx) + static void CompletePending(ref Status status, ref StringOutput output, ref StringBasicContext basicCtx) { _ = basicCtx.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); @@ -275,7 +280,7 @@ static void CompletePending(ref Status status, ref SpanByteAndMemory output, ref public unsafe HashSet GetNamespacesForKeys(StoreWrapper storeWrapper, IEnumerable keys, Dictionary vectorSetKeys) { // TODO: Ideally we wouldn't make a new session for this, but it's fine for now - using var storageSession = new StorageSession(storeWrapper, new(), null, null, storeWrapper.DefaultDatabase.Id, this, logger); + using var storageSession = new StorageSession(storeWrapper, new(), new(), null, null, storeWrapper.DefaultDatabase.Id, null, this, logger); HashSet namespaces = null; @@ -288,10 +293,10 @@ public unsafe HashSet GetNamespacesForKeys(StoreWrapper storeWrapper, IEn var keySpan = SpanByte.FromPinnedPointer(keyPtr, key.Length); // Dummy command, we just need something Vector Set-y - RawStringInput input = default; + StringInput input = default; input.header.cmd = RespCommand.VSIM; - using (ReadVectorIndex(storageSession, ref keySpan, ref input, indexSpan, out var status)) + using (ReadVectorIndex(storageSession, keySpan, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -300,7 +305,7 @@ public unsafe HashSet GetNamespacesForKeys(StoreWrapper storeWrapper, IEn namespaces ??= []; - ReadIndex(indexSpan, out var context, out _, out _, out _, out _, out _, out _, out _, out _); + ReadIndex(indexSpan, out var context, out _, out _, out _, out _, out _, out _, out _); for (var i = 0UL; i < ContextStep; i++) { _ = namespaces.Add(context + i); diff --git a/libs/server/Resp/Vector/VectorManager.Replication.cs b/libs/server/Resp/Vector/VectorManager.Replication.cs index 5f5249a66ba..25154f0259f 100644 --- a/libs/server/Resp/Vector/VectorManager.Replication.cs +++ b/libs/server/Resp/Vector/VectorManager.Replication.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; +using System.Buffers.Binary; using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; @@ -16,9 +17,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - /// /// Methods for managing the replication of Vector Sets from primaries to other replicas. /// @@ -61,13 +59,9 @@ public async Task StartReplicationTasksAsync(CancellationToken cancellationToken replicationReplayCancellation = cancellationToken; - using var cts = new CancellationTokenSource(); - - _ = cancellationToken.Register(() => cts.Cancel()); - try { - await Task.Delay(Timeout.InfiniteTimeSpan, cts.Token).ConfigureAwait(false); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken).ConfigureAwait(false); } catch { } @@ -89,21 +83,14 @@ public async Task StartReplicationTasksAsync(CancellationToken cancellationToken /// /// This the Primary part, on a Replica runs. /// - internal void ReplicateVectorSetAdd(ref SpanByte key, ref RawStringInput input, ref TContext context) - where TContext : ITsavoriteContext + internal void ReplicateVectorSetAdd(ReadOnlySpan key, ref StringInput input, ref StringBasicContext context) { Debug.Assert(input.header.cmd == RespCommand.VADD, "Shouldn't be called with anything but VADD inputs"); var inputCopy = input; inputCopy.arg1 = VADDAppendLogArg; - Span keyWithNamespaceBytes = stackalloc byte[key.Length + 1]; - var keyWithNamespace = SpanByte.FromPinnedSpan(keyWithNamespaceBytes); - keyWithNamespace.MarkNamespace(); - keyWithNamespace.SetNamespaceInPayload(0); - key.AsReadOnlySpan().CopyTo(keyWithNamespace.AsSpan()); - - var res = context.RMW(ref keyWithNamespace, ref inputCopy); + var res = context.RMW((FixedSpanByteKey)key, ref inputCopy); if (res.IsPending) { @@ -126,23 +113,16 @@ internal void ReplicateVectorSetAdd(ref SpanByte key, ref RawStringInp /// /// This the Primary part, on a Replica runs. /// - internal void ReplicateVectorSetRemove(ref SpanByte key, ref SpanByte element, ref RawStringInput input, ref TContext context) - where TContext : ITsavoriteContext + internal void ReplicateVectorSetRemove(ReadOnlySpan key, ReadOnlySpan element, ref StringInput input, ref StringBasicContext context) { Debug.Assert(input.header.cmd == RespCommand.VREM, "Shouldn't be called with anything but VREM inputs"); var inputCopy = input; inputCopy.arg1 = VREMAppendLogArg; - Span keyWithNamespaceBytes = stackalloc byte[key.Length + 1]; - var keyWithNamespace = SpanByte.FromPinnedSpan(keyWithNamespaceBytes); - keyWithNamespace.MarkNamespace(); - keyWithNamespace.SetNamespaceInPayload(0); - key.AsReadOnlySpan().CopyTo(keyWithNamespace.AsSpan()); - - inputCopy.parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(element.AsReadOnlySpan())); + inputCopy.parseState.InitializeWithArgument(PinnedSpanByte.FromPinnedSpan(element)); - var res = context.RMW(ref keyWithNamespace, ref inputCopy); + var res = context.RMW((FixedSpanByteKey)key, ref inputCopy); if (res.IsPending) { @@ -156,36 +136,14 @@ internal void ReplicateVectorSetRemove(ref SpanByte key, ref SpanByte } } - /// - /// After an index is dropped, called to cleanup state injected by - /// - /// Amounts to delete a synthetic key in namespace 0. - /// - internal bool TryDropVectorSetReplicationKey(SpanByte key, ref TContext context) - where TContext : ITsavoriteContext - { - Span keyWithNamespaceBytes = stackalloc byte[key.Length + 1]; - var keyWithNamespace = SpanByte.FromPinnedSpan(keyWithNamespaceBytes); - keyWithNamespace.MarkNamespace(); - keyWithNamespace.SetNamespaceInPayload(0); - key.AsReadOnlySpan().CopyTo(keyWithNamespace.AsSpan()); - - var res = context.Delete(ref keyWithNamespace); - - if (res.IsPending) - { - CompletePending(ref res, ref context); - } - - return res.IsCompletedSuccessfully; - } - /// /// Vector Set adds are phrased as reads (once the index is created), so they require special handling. /// /// Operations that are faked up by running on the Primary get diverted here on a Replica. /// - internal void HandleVectorSetAddReplication(StorageSession currentSession, Func obtainServerSession, ref SpanByte keyWithNamespace, ref RawStringInput input) + internal void HandleVectorSetAddReplication( + StorageSession currentSession, + Func obtainServerSession, ReadOnlySpan key, ref StringInput input) { if (input.arg1 == MigrateElementKeyLogArg) { @@ -193,14 +151,19 @@ internal void HandleVectorSetAddReplication(StorageSession currentSession, Func< // These get replayed on REPLICAs typically, though role changes might still cause these // to get replayed on now-primary nodes - var key = input.parseState.GetArgSliceByRef(0).SpanByte; - var value = input.parseState.GetArgSliceByRef(1).SpanByte; + // Serialized len + ns + len + key in ReplicateMigratedElementKey + var elementNamespaceAndKey = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + + var elementNsLen = BinaryPrimitives.ReadInt32LittleEndian(elementNamespaceAndKey); + var elementNsBytes = elementNamespaceAndKey.Slice(sizeof(int), elementNsLen); + var elementKeyLen = BinaryPrimitives.ReadInt32LittleEndian(elementNamespaceAndKey[(sizeof(int) + elementNsLen)..]); + var elementKeyBytes = elementNamespaceAndKey.Slice(sizeof(int) + elementNsLen + sizeof(int), elementKeyLen); + + var value = input.parseState.GetArgSliceByRef(1); - // TODO: Namespace is present, but not actually transmitted - // This presumably becomes unnecessary in Store v2 - key.MarkNamespace(); + Debug.Assert(elementNsBytes.Length == 1, "Longer length namespaces not supported"); - var ns = key.GetNamespaceInPayload(); + var ns = (ulong)elementNsBytes[0]; // REPLICAs wouldn't have seen a reservation message, so allocate this on demand var ctx = ns & ~(ContextStep - 1); @@ -221,19 +184,19 @@ internal void HandleVectorSetAddReplication(StorageSession currentSession, Func< if (needsUpdate) { - UpdateContextMetadata(ref currentSession.vectorContext); + UpdateContextMetadata(ref currentSession.vectorBasicContext); } } - HandleMigratedElementKey(ref currentSession.basicContext, ref currentSession.vectorContext, ref key, ref value); + HandleMigratedElementKey(ref currentSession.stringBasicContext, ref currentSession.vectorBasicContext, elementNsBytes, elementKeyBytes, value); return; } else if (input.arg1 == MigrateIndexKeyLogArg) { // These also injected by a PRIMARY applying migration operations - var key = input.parseState.GetArgSliceByRef(0).SpanByte; - var value = input.parseState.GetArgSliceByRef(1).SpanByte; + var indexKey = input.parseState.GetArgSliceByRef(0); + var value = input.parseState.GetArgSliceByRef(1); var context = MemoryMarshal.Cast(input.parseState.GetArgSliceByRef(2).Span)[0]; // Most of the time a replica will have seen an element moving before now @@ -257,14 +220,14 @@ internal void HandleVectorSetAddReplication(StorageSession currentSession, Func< if (needsUpdate) { - UpdateContextMetadata(ref currentSession.vectorContext); + UpdateContextMetadata(ref currentSession.vectorBasicContext); } } ActiveThreadSession = currentSession; try { - HandleMigratedIndexKey(null, null, ref key, ref value); + HandleMigratedIndexKey(null, null, indexKey, value); } finally { @@ -278,10 +241,12 @@ internal void HandleVectorSetAddReplication(StorageSession currentSession, Func< // Undo mangling that got replication going var inputCopy = input; inputCopy.arg1 = default; - var keyBytesArr = ArrayPool.Shared.Rent(keyWithNamespace.Length - 1); - var keyBytes = keyBytesArr.AsMemory()[..(keyWithNamespace.Length - 1)]; - keyWithNamespace.AsReadOnlySpan().CopyTo(keyBytes.Span); + // Copy key onto + var keyBytesArr = ArrayPool.Shared.Rent(key.Length); + var keyBytes = keyBytesArr.AsMemory()[..key.Length]; + + key.CopyTo(keyBytes.Span); var dims = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(0).Span); var reduceDims = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(1).Span); @@ -423,30 +388,30 @@ static unsafe void ApplyVectorSetAdd(VectorManager self, StorageSession storageS var indexBytes = stackalloc byte[IndexSizeBytes]; - var dimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dims, 1))); - var reduceDimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); - var valueTypeArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref valueType, 1))); - var valuesArg = ArgSlice.FromPinnedSpan(values.AsReadOnlySpan()); - var elementArg = ArgSlice.FromPinnedSpan(element.AsReadOnlySpan()); - var quantizerArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantizer, 1))); - var buildExplorationFactorArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); - var attributesArg = ArgSlice.FromPinnedSpan(attributes.AsReadOnlySpan()); - var numLinksArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); - var distanceMetricArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); + var dimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dims, 1))); + var reduceDimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); + var valueTypeArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref valueType, 1))); + var valuesArg = PinnedSpanByte.FromPinnedSpan(values); + var elementArg = PinnedSpanByte.FromPinnedSpan(element); + var quantizerArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantizer, 1))); + var buildExplorationFactorArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); + var attributesArg = PinnedSpanByte.FromPinnedSpan(attributes); + var numLinksArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); + var distanceMetricArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); reusableParseState.InitializeWithArguments([dimsArg, reduceDimsArg, valueTypeArg, valuesArg, elementArg, quantizerArg, buildExplorationFactorArg, attributesArg, numLinksArg, distanceMetricArg]); - var input = new RawStringInput(RespCommand.VADD, ref reusableParseState); + StringInput input = new(RespCommand.VADD, ref reusableParseState); // Equivalent to VectorStoreOps.VectorSetAdd // // We still need locking here because the replays may proceed in parallel - using (self.ReadOrCreateVectorIndex(storageSession, ref key, ref input, indexSpan, out var status)) + using (self.ReadOrCreateVectorIndex(storageSession, key, ref input, indexSpan, out var status)) { Debug.Assert(status == GarnetStatus.OK, "Replication should only occur when an add is successful, so index must exist"); - var addRes = self.TryAdd(indexSpan, element.AsReadOnlySpan(), valueType, values.AsReadOnlySpan(), attributes.AsReadOnlySpan(), reduceDims, quantizer, buildExplorationFactor, numLinks, distanceMetric, out _); + var addRes = self.TryAdd(indexSpan, element, valueType, values, attributes, reduceDims, quantizer, buildExplorationFactor, numLinks, distanceMetric, out _); if (addRes != VectorManagerResult.OK) { @@ -523,20 +488,15 @@ public void ShutdownReplayTasks() /// /// Operations that are faked up by running on the Primary get diverted here on a Replica. /// - internal void HandleVectorSetRemoveReplication(StorageSession storageSession, ref SpanByte key, ref RawStringInput input) + internal void HandleVectorSetRemoveReplication(StorageSession storageSession, ReadOnlySpan key, ref StringInput input) { Span indexSpan = stackalloc byte[IndexSizeBytes]; var element = input.parseState.GetArgSliceByRef(0); - // Replication adds a (0) namespace - remove it - Span keyWithoutNamespaceSpan = stackalloc byte[key.Length - 1]; - key.AsReadOnlySpan().CopyTo(keyWithoutNamespaceSpan); - var keyWithoutNamespace = SpanByte.FromPinnedSpan(keyWithoutNamespaceSpan); - var inputCopy = input; inputCopy.arg1 = default; - using (ReadVectorIndex(storageSession, ref keyWithoutNamespace, ref inputCopy, indexSpan, out var status)) + using (ReadVectorIndex(storageSession, key, ref inputCopy, indexSpan, out var status)) { Debug.Assert(status == GarnetStatus.OK, "Replication should only occur when a remove is successful, so index must exist"); @@ -556,7 +516,7 @@ public void WaitForVectorOperationsToComplete() { try { - replicationBlockEvent.Wait(); + _ = replicationBlockEvent.Wait(); } catch (ObjectDisposedException) { @@ -566,8 +526,7 @@ public void WaitForVectorOperationsToComplete() } } // Helper to complete read/writes during vector set synthetic op goes async - private static void CompletePending(ref Status status, ref TContext context) - where TContext : ITsavoriteContext + private static void CompletePending(ref Status status, ref VectorBasicContext context) { _ = context.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); diff --git a/libs/server/Resp/Vector/VectorManager.cs b/libs/server/Resp/Vector/VectorManager.cs index c00d5752d95..6e8b34da4c0 100644 --- a/libs/server/Resp/Vector/VectorManager.cs +++ b/libs/server/Resp/Vector/VectorManager.cs @@ -4,6 +4,7 @@ using System; using System.Buffers; using System.Buffers.Binary; +using System.Collections.Concurrent; using System.Diagnostics; using System.Runtime.InteropServices; using System.Text; @@ -16,9 +17,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - public enum VectorManagerResult { Invalid = 0, @@ -37,6 +35,9 @@ public sealed partial class VectorManager : IDisposable // MUST BE A POWER OF 2 public const ulong ContextStep = 8; + // We reserve the first 7 namespaces (we can't use 0, so it's off limits) for store-wide metadata about Vector Sets + internal const byte MetadataNamespace = 1; + internal const int IndexSizeBytes = Index.Size; internal const long VADDAppendLogArg = long.MinValue; internal const long DeleteAfterDropArg = VADDAppendLogArg + 1; @@ -45,36 +46,100 @@ public sealed partial class VectorManager : IDisposable internal const long MigrateElementKeyLogArg = VREMAppendLogArg + 1; internal const long MigrateIndexKeyLogArg = MigrateElementKeyLogArg + 1; + /// + /// Byte stored on log records to distinguish the INDEX key as a Vector Set + /// Element keys are tracked in separate namespaces and are not marked with a special RecordType + /// + public const byte RecordType = 1; + /// /// Minimum size of an id is assumed to be at least 8 bytes + a length prefix. /// private const int MinimumSpacePerId = sizeof(int) + 8; /// - /// The process wide instances of DiskANN. - /// - /// We only need the one, even if we have multiple DBs, because all context is provided by DiskANN instances and Garnet storage. + /// Maximum number of dimensions a vector can have. + /// Matches Redis's VSET_MAX_VECTOR_DIM (65,536). /// - private DiskANNService Service { get; } = new DiskANNService(); + internal const int MaxVectorDimensions = 1 << 16; /// - /// Whether or not Vector Set preview is enabled. + /// Maximum number of results that can be requested in a single VSIM query. + /// Practical limit to prevent integer overflow when computing buffer sizes + /// (e.g. retrieveCount * MinimumSpacePerId) and to avoid excessive allocations + /// from a single command (at 100M: ~400 MB for distances + ~1.2 GB for ids). + /// + internal const int MaxRetrieveCount = 100_000_000; + + /// + /// Maximum exploration factor (EF) for build and search operations. + /// Matches Redis's hardcoded limit of 1,000,000. + /// + internal const int MaxExplorationFactor = 1_000_000; + + /// + /// Ensures the VSIM distance output buffer has at least * sizeof(float) bytes. + /// Rents from if the current buffer is too small. + /// + private static void EnsureDistanceBufferSize(ref SpanByteAndMemory buffer, int retrieveCount) + { + // Verify no overflow: checked() ensures MaxRetrieveCount * sizeof(float) fits in int32 + Debug.Assert(retrieveCount <= MaxRetrieveCount && checked(MaxRetrieveCount * sizeof(float)) > 0); + var sizeBytes = retrieveCount * sizeof(float); + if (sizeBytes > buffer.Length) + { + if (!buffer.IsSpanByte) + { + buffer.Memory.Dispose(); + } + + buffer = new SpanByteAndMemory(MemoryPool.Shared.Rent(sizeBytes), sizeBytes); + } + + buffer.Length = sizeBytes; + } + + /// + /// Ensures the VSIM id output buffer has at least * bytes. + /// Rents from if the current buffer is too small. + /// If we're still wrong, we'll end up using continuation callbacks which have more overhead. + /// + private static void EnsureIdBufferSize(ref SpanByteAndMemory buffer, int retrieveCount) + { + // Verify no overflow: checked() ensures MaxRetrieveCount * MinimumSpacePerId fits in int32 + Debug.Assert(retrieveCount <= MaxRetrieveCount && checked(MaxRetrieveCount * MinimumSpacePerId) > 0); + var sizeBytes = retrieveCount * MinimumSpacePerId; + if (sizeBytes > buffer.Length) + { + if (!buffer.IsSpanByte) + { + buffer.Memory.Dispose(); + } + + buffer = new SpanByteAndMemory(MemoryPool.Shared.Rent(sizeBytes), sizeBytes); + } + } + + /// + /// This managers instance of . /// - /// TODO: This goes away once we're stable. + /// We could probably share these, but its not a big loss to scope to the instance. /// - public bool IsEnabled { get; } + internal DiskANNService Service { get; } = new DiskANNService(); /// - /// Unique id for this . + /// Whether or not Vector Set preview is enabled. /// - /// Is used to determine if an is backed by a DiskANN index that was created in this process. + /// TODO: This goes away once we're stable. /// - private readonly Guid processInstanceId = Guid.NewGuid(); + public bool IsEnabled { get; } private readonly ILogger logger; private readonly int dbId; + private ConcurrentDictionary recoveredIndexes; + public VectorManager(int dbId, GarnetServerOptions serverOptions, Func getCleanupSession, ILoggerFactory loggerFactory) { this.dbId = dbId; @@ -82,10 +147,11 @@ public VectorManager(int dbId, GarnetServerOptions serverOptions, Func(new() { SingleWriter = true, SingleReader = false, AllowSynchronousContinuations = false }); + // NOTE: for multi-log we need to disable single writer since multiple AOF replay tasks may append to this common channel. + replicationReplayChannel = Channel.CreateUnbounded(new() { SingleWriter = !serverOptions.MultiLogEnabled, SingleReader = false, AllowSynchronousContinuations = false }); if (serverOptions.VectorSetReplayTaskCount < 0 || serverOptions.VectorSetReplayTaskCount > Environment.ProcessorCount) throw new GarnetException($"VectorSetReplayTaskCount should be in range [0,{Environment.ProcessorCount}]!"); @@ -100,7 +166,11 @@ public VectorManager(int dbId, GarnetServerOptions serverOptions, Func(new() { SingleWriter = false, SingleReader = true, AllowSynchronousContinuations = false }); + requestCleanupTaskChannel = Channel.CreateUnbounded<(ulong Context, TaskCompletionSource Completion)>(new() { SingleWriter = false, SingleReader = true, AllowSynchronousContinuations = false }); cleanupTask = RunCleanupTaskAsync(); + requestCleanupTask = RunRequestCleanupTaskAsync(); + + recoveredIndexes = new(); logger?.LogInformation("Created VectorManager"); } @@ -118,23 +188,19 @@ public void Initialize() throw new GarnetException($"Could not switch VectorManager cleanup session to {dbId}, initialization failed"); } - Span keySpan = stackalloc byte[1]; - Span dataSpan = stackalloc byte[ContextMetadata.Size]; - - var key = SpanByte.FromPinnedSpan(keySpan); + VectorElementKey key = new(MetadataNamespace, []); - key.MarkNamespace(); - key.SetNamespaceInPayload(0); + Span dataSpan = stackalloc byte[ContextMetadata.Size]; - var data = SpanByte.FromPinnedSpan(dataSpan); + VectorOutput data = new(dataSpan); - ref var ctx = ref session.storageSession.vectorContext; + ref var ctx = ref session.storageSession.vectorBasicContext; - var status = ctx.Read(ref key, ref data); + var status = ctx.Read(key, ref data); if (status.IsPending) { - SpanByte ignored = default; + VectorOutput ignored = new(); CompletePending(ref status, ref ignored, ref ctx); } @@ -146,7 +212,6 @@ public void Initialize() contextMetadata = MemoryMarshal.Cast(dataSpan)[0]; } } - } /// @@ -158,13 +223,14 @@ public void ResumePostRecovery() using var session = (RespServerSession)getCleanupSession(); - ref var ctx = ref session.storageSession.vectorContext; + ref var ctx = ref session.storageSession.vectorBasicContext; - // If we come up and contexts are marked for migration, that means the migration FAILED - // and we'd like those contexts back ASAP lock (this) { + // If we come up and contexts are marked for migration, that means the migration FAILED + // and we'd like those contexts back ASAP var abandonedMigrations = contextMetadata.GetMigrating(); + var needsUpdated = false; if (abandonedMigrations != null) { @@ -174,111 +240,40 @@ public void ResumePostRecovery() contextMetadata.MarkCleaningUp(abandoned); } - UpdateContextMetadata(ref ctx); + needsUpdated = true; } - } - - Span indexSpan = stackalloc byte[Index.Size]; - - // Finish any deletes that were in progress before we restarted - var failedDeletes = GetDeletesInProgress(session.storageSession); - var clearInProgressDeletes = true; - foreach (var (toDeleteKey, toDeleteCtx) in failedDeletes) - { - logger?.LogInformation("Cleaning up in progress Vector Set delete of {key} (context: {ctx})", Encoding.UTF8.GetString(toDeleteKey.Span), toDeleteCtx); - unsafe + // Any non-deleted records we recovered for contexts being deleted, we need to undo that + foreach (var (context, _) in recoveredIndexes) { - fixed (byte* toDeleteKeyPtr = toDeleteKey.Span) + if (contextMetadata.IsCleaningUp(context)) { - var toDeleteKeySpanByte = SpanByte.FromPinnedPointer(toDeleteKeyPtr, toDeleteKey.Span.Length); - - RawStringInput input = new(RespCommand.VADD); - - // Check if delete got far enough that we should re-apply it - using (ReadForDeleteVectorIndex(session.storageSession, ref toDeleteKeySpanByte, ref input, indexSpan, out var garnetStatus)) - { - if (garnetStatus is not (GarnetStatus.BADSTATE or GarnetStatus.NOTFOUND)) - { - // It didn't - so don't re-apply (But do remove the "we're deleting"-entry later) - continue; - } - } - - try - { - if (TryDeleteVectorSet(session.storageSession, ref toDeleteKeySpanByte, out var garnetStatus).IsCompletedSuccessfully && garnetStatus != GarnetStatus.BADSTATE) - { - // Normal delete worked, easy enough - // - // This happens if we fail between the "remember we're deleting" and "zero everything out" steps - logger?.LogInformation("Vector Set under {key} (context: {ctx}) deleted normally", Encoding.UTF8.GetString(toDeleteKey.Span), toDeleteCtx); - continue; - } - } - catch (Exception ex) - { - logger?.LogError(ex, "Attempt at normal cleanup of {key} failed", Encoding.UTF8.GetString(toDeleteKey.Span)); - } - - // Partial delete, do these bits directly - // 1. Try to zero out the index key - // 2. Try to delete the index key - // 3. Try to drop the replication key - // 4. Mark the context as needing cleanup - - // Zero out the index (which may already be zero'd, but that's fine to redo) - RawStringInput updateToDroppableVectorSet = new(RespCommand.VADD, arg1: DeleteAfterDropArg); - var update = session.storageSession.basicContext.RMW(ref toDeleteKeySpanByte, ref updateToDroppableVectorSet); - if (!update.IsCompletedSuccessfully) - { - throw new GarnetException("Failed to make Vector Set delete-able, this should never happen but will leave vector sets corrupted"); - } - - // Note that we don't need to DROP the index because we know we haven't re-created it yet - - // Actually delete the value - var del = session.storageSession.basicContext.Delete(ref toDeleteKeySpanByte); - if (!(del.Found || del.NotFound)) - { - logger?.LogCritical("Failed to cleanup delete dropped Vector Set {key} (context: {ctx}), Vector Set will remain corrupted", Encoding.UTF8.GetString(toDeleteKey.Span), toDeleteCtx); - clearInProgressDeletes = false; - continue; - } - - // Cleanup incidental additional state - if (!TryDropVectorSetReplicationKey(toDeleteKeySpanByte, ref session.storageSession.basicContext)) - { - logger?.LogCritical("Failed to cleanup delete dropped Vector Set {key} (context: {ctx}), Vector Set will remain corrupted", Encoding.UTF8.GetString(toDeleteKey.Span), toDeleteCtx); - clearInProgressDeletes = false; - continue; - } + contextMetadata.ClearIsCleaningUp(context); + needsUpdated = true; + } - // Schedule cleanup of element data - CleanupDroppedIndex(ref session.storageSession.vectorContext, toDeleteCtx); + recoveredIndexes = null; + } - logger?.LogInformation("Vector Set under {key} (context: {ctx}) deleted normally", Encoding.UTF8.GetString(toDeleteKey.Span), toDeleteCtx); - } + if (needsUpdated) + { + UpdateContextMetadata(ref ctx); } } - if (clearInProgressDeletes) - { - // We successfully dealt with all pending deletes, we can delete the metadata key - Span toDeleteKeySpan = stackalloc byte[2]; - var toDeleteKey = SpanByte.FromPinnedSpan(toDeleteKeySpan); - - // 0:1 is InProgressDeletes - toDeleteKey.MarkNamespace(); - toDeleteKey.SetNamespaceInPayload(0); - toDeleteKey.AsSpan()[0] = 1; + // Resume any cleanups we didn't complete before recovery + _ = cleanupTaskChannel.Writer.TryWrite(null); + } - var deleteStatus = session.storageSession.vectorContext.Delete(ref toDeleteKey); - Debug.Assert(!deleteStatus.IsPending, "Delete shouldn't go async"); + public void RecoveredVectorSetIndexKey(ref LogRecord record) + { + if (record.ValueSpan.Length != IndexSize) + { + return; } - // Resume any cleanups we didn't complete before recovery - _ = cleanupTaskChannel.Writer.TryWrite(null); + ReadIndex(record.ValueSpan, out var context, out _, out _, out _, out _, out _, out _, out _); + recoveredIndexes[context] = 0; } /// @@ -291,14 +286,25 @@ public void Dispose() replicationBlockEvent.Dispose(); - // Wait for any in progress cleanup to finish + // Wait for any _marking_ of cleanup state to finish. PauseCleanupAsync callers MUST + // have called ResumeCleanup before reaching here, otherwise the cleanup task + // is permanently blocked on cleanupGate.WaitAsync() and Dispose will hang. + requestCleanupTaskChannel.Writer.Complete(); + AsyncUtils.BlockingWait(requestCleanupTaskChannel.Reader.Completion); + AsyncUtils.BlockingWait(requestCleanupTask); + + // Wait for any in progress cleanup to finish. PauseCleanupAsync callers MUST + // have called ResumeCleanup before reaching here, otherwise the cleanup task + // is permanently blocked on cleanupGate.WaitAsync() and Dispose will hang. cleanupTaskChannel.Writer.Complete(); AsyncUtils.BlockingWait(cleanupTaskChannel.Reader.Completion); AsyncUtils.BlockingWait(cleanupTask); + + // Cleanup task has fully drained, so nothing else can take this gate. + cleanupGate.Dispose(); } - private static void CompletePending(ref Status status, ref SpanByte output, ref TContext ctx) - where TContext : ITsavoriteContext + private static void CompletePending(ref Status status, ref VectorOutput output, ref VectorBasicContext ctx) { _ = ctx.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); @@ -309,6 +315,16 @@ private static void CompletePending(ref Status status, ref SpanByte ou completedOutputs.Dispose(); } + private static void CompletePending(ref Status status, ref StringBasicContext ctx) + { + _ = ctx.CompletePendingWithOutputs(out var completedOutputs, wait: true); + var more = completedOutputs.Next(); + Debug.Assert(more); + status = completedOutputs.Current.Status; + Debug.Assert(!completedOutputs.Next()); + completedOutputs.Dispose(); + } + /// /// Add a vector to a vector set encoded by . /// @@ -333,7 +349,7 @@ out ReadOnlySpan errorMsg errorMsg = default; - ReadIndex(indexValue, out var context, out var dimensions, out var reduceDims, out var quantType, out _, out var numLinks, out var distanceMetric, out var indexPtr, out _); + ReadIndex(indexValue, out var context, out var dimensions, out var reduceDims, out var quantType, out _, out var numLinks, out var distanceMetric, out var indexPtr); var valueDims = CalculateValueDimensions(valueType, values); @@ -398,7 +414,7 @@ internal VectorManagerResult TryRemove(ReadOnlySpan indexValue, ReadOnlySp { AssertHaveStorageSession(); - ReadIndex(indexValue, out var context, out _, out _, out var quantType, out _, out _, out _, out var indexPtr, out _); + ReadIndex(indexValue, out var context, out _, out _, out var quantType, out _, out _, out _, out var indexPtr); var del = Service.Remove(context, indexPtr, element); @@ -406,75 +422,64 @@ internal VectorManagerResult TryRemove(ReadOnlySpan indexValue, ReadOnlySp } /// - /// Deletion of a Vector Set needs special handling. - /// - /// This is called by DEL and UNLINK after a naive delete fails for us to _try_ and delete a Vector Set. + /// Request deletion of a Vector Set given the VALUE of the index key. /// - internal Status TryDeleteVectorSet(StorageSession storageSession, ref SpanByte key, out GarnetStatus status) + internal void RequestDeletion(Span value) { - storageSession.parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan())); - - var input = new RawStringInput(RespCommand.VADD, ref storageSession.parseState); - - Span indexSpan = stackalloc byte[Index.Size]; - - using (ReadForDeleteVectorIndex(storageSession, ref key, ref input, indexSpan, out status)) + if (value.Length != IndexSize) { - if (status != GarnetStatus.OK) - { - // This can happen is something else successfully deleted before we acquired the lock - return Status.CreateNotFound(); - } - - ReadIndex(indexSpan, out var context, out _, out _, out _, out _, out _, out _, out _, out _); - - if (!TryMarkDeleteInProgress(ref storageSession.vectorContext, ref key, context)) - { - // We can't recover from a crash or error, so fail the delete for safety - return Status.CreateError(); - } + logger?.LogWarning($"Ignored Vector Set deletion due to size mismatch"); + return; + } - ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_0); + ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_0); - // Update the index to be delete-able - RawStringInput updateToDroppableVectorSet = new(RespCommand.VADD, arg1: DeleteAfterDropArg); + ReadIndex(value, out var context, out _, out _, out _, out _, out _, out _, out _); - var update = storageSession.basicContext.RMW(ref key, ref updateToDroppableVectorSet); - if (!update.IsCompletedSuccessfully) - { - throw new GarnetException("Failed to make Vector Set delete-able, this should never happen but will leave vector sets corrupted"); - } + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - // Drop the native side of the index now - we can't fault between the two unless the process is torn down - DropIndex(indexSpan); - - ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_1); + if (!requestCleanupTaskChannel.Writer.TryWrite((context, tcs))) + { + throw new GarnetException("Could not submit request for Vector Set cleanup, aborting delete"); + } - // Actually delete the value - var del = storageSession.basicContext.Delete(ref key); - if (!del.IsCompletedSuccessfully) - { - throw new GarnetException("Failed to delete dropped Vector Set, this should never happen but will leave vector sets corrupted"); - } + // Wait until the context is _marked_ for cleanup, but not the actual cleanup + AsyncUtils.BlockingWait(tcs.Task); - ExceptionInjectionHelper.TriggerException(ExceptionInjectionType.VectorSet_Interrupt_Delete_2); + // Tell DiskANN to clean itself up + DropIndex(value); + } - // Cleanup incidental additional state - if (!TryDropVectorSetReplicationKey(key, ref storageSession.basicContext)) - { - logger?.LogCritical("Couldn't synthesize Vector Set delete operation for replication, data loss will occur"); - } + /// + /// Ask DiskANN to drop its index. + /// + internal void DropInMemoryIndex(ReadOnlySpan value) + { + if (value.Length != IndexSize) + { + logger?.LogWarning($"Ignored Vector Set drop index due to size mismatch"); + return; + } - // Schedule cleanup of element data - CleanupDroppedIndex(ref storageSession.vectorContext, context); + ReadIndex(value, out var context, out _, out _, out _, out _, out _, out _, out var indexPtr); - // Delete has finished, so remove the in progress metadata - // - // A crash or error before this will cause some work to be retried, but no correctness issues - ClearDeleteInProgress(ref storageSession.vectorContext, ref key, context); + Service.DropIndex(context, indexPtr); + } - return Status.CreateFound(); + /// + /// Clear out the index pointer stored in this record value. + /// + /// Next time the record is touched, we'll recreate the index. + /// + internal static void ClearIndexPointer(Span value) + { + if (value.Length != IndexSize) + { + return; } + + ref var index = ref MemoryMarshal.Cast(value)[0]; + index.IndexPtr = 0; } /// @@ -499,7 +504,7 @@ ref SpanByteAndMemory filterBitmap { AssertHaveStorageSession(); - ReadIndex(indexValue, out var context, out var dimensions, out _, out var quantType, out _, out _, out _, out var indexPtr, out _); + ReadIndex(indexValue, out var context, out var dimensions, out _, out var quantType, out _, out _, out _, out var indexPtr); var valueDims = CalculateValueDimensions(valueType, values); if (dimensions != valueDims) @@ -525,32 +530,8 @@ ref SpanByteAndMemory filterBitmap retrieveCount = effectiveEF; } - // Make sure enough space in distances for requested count - if (retrieveCount > outputDistances.Length) - { - if (!outputDistances.IsSpanByte) - { - outputDistances.Memory.Dispose(); - } - - outputDistances = new SpanByteAndMemory(MemoryPool.Shared.Rent(retrieveCount * sizeof(float)), retrieveCount * sizeof(float)); - } - - // Indicate requested # of matches - outputDistances.Length = retrieveCount * sizeof(float); - - // If we're fairly sure the ids won't fit, go ahead and grab more memory now - // - // If we're still wrong, we'll end up using continuation callbacks which have more overhead - if (retrieveCount * MinimumSpacePerId > outputIds.Length) - { - if (!outputIds.IsSpanByte) - { - outputIds.Memory.Dispose(); - } - - outputIds = new SpanByteAndMemory(MemoryPool.Shared.Rent(retrieveCount * MinimumSpacePerId), retrieveCount * MinimumSpacePerId); - } + EnsureDistanceBufferSize(ref outputDistances, retrieveCount); + EnsureIdBufferSize(ref outputIds, retrieveCount); var found = Service.SearchVector( @@ -594,7 +575,7 @@ out var continuation filterBitmap = new SpanByteAndMemory(MemoryPool.Shared.Rent(requiredBitmapBytes), requiredBitmapBytes); } - ApplyPostFilter(filter, found, outputAttributes.AsReadOnlySpan(), filterBitmap.AsSpan(), ActiveThreadSession.scratchBufferBuilder); + ApplyPostFilter(filter, found, outputAttributes.ReadOnlySpan, filterBitmap.Span, ActiveThreadSession.scratchBufferBuilder); } if (continuation != 0) @@ -639,7 +620,7 @@ ref SpanByteAndMemory filterBitmap { AssertHaveStorageSession(); - ReadIndex(indexValue, out var context, out _, out _, out var quantType, out _, out _, out _, out var indexPtr, out _); + ReadIndex(indexValue, out var context, out _, out _, out var quantType, out _, out _, out _, out var indexPtr); // When a filter is present, over-retrieve candidates from DiskANN var retrieveCount = !filter.IsEmpty ? maxFilteringEffort : count; @@ -653,32 +634,8 @@ ref SpanByteAndMemory filterBitmap retrieveCount = effectiveEF; } - // Make sure enough space in distances for requested count - if (retrieveCount * sizeof(float) > outputDistances.Length) - { - if (!outputDistances.IsSpanByte) - { - outputDistances.Memory.Dispose(); - } - - outputDistances = new SpanByteAndMemory(MemoryPool.Shared.Rent(retrieveCount * sizeof(float)), retrieveCount * sizeof(float)); - } - - // Indicate requested # of matches - outputDistances.Length = retrieveCount * sizeof(float); - - // If we're fairly sure the ids won't fit, go ahead and grab more memory now - // - // If we're still wrong, we'll end up using continuation callbacks which have more overhead - if (retrieveCount * MinimumSpacePerId > outputIds.Length) - { - if (!outputIds.IsSpanByte) - { - outputIds.Memory.Dispose(); - } - - outputIds = new SpanByteAndMemory(MemoryPool.Shared.Rent(retrieveCount * MinimumSpacePerId), retrieveCount * MinimumSpacePerId); - } + EnsureDistanceBufferSize(ref outputDistances, retrieveCount); + EnsureIdBufferSize(ref outputIds, retrieveCount); var found = Service.SearchElement( @@ -721,7 +678,7 @@ out var continuation filterBitmap = new SpanByteAndMemory(MemoryPool.Shared.Rent(requiredBitmapBytes), requiredBitmapBytes); } - ApplyPostFilter(filter, found, outputAttributes.AsReadOnlySpan(), filterBitmap.AsSpan(), ActiveThreadSession.scratchBufferBuilder); + ApplyPostFilter(filter, found, outputAttributes.ReadOnlySpan, filterBitmap.Span, ActiveThreadSession.scratchBufferBuilder); } if (continuation != 0) @@ -753,11 +710,11 @@ out var continuation /// IMPORTANT: outputAttributes may be replaced with an allocated memory, so the caller needs to check /// if the buffer is stack-based or heap-based, and dispose if it's the latter. /// - internal VectorManagerResult FetchSingleVectorElementAttributes(ReadOnlySpan indexValue, SpanByte element, ref SpanByteAndMemory outputAttributes) + internal VectorManagerResult FetchSingleVectorElementAttributes(ReadOnlySpan indexValue, PinnedSpanByte element, ref SpanByteAndMemory outputAttributes) { AssertHaveStorageSession(); - ReadIndex(indexValue, out var context, out _, out _, out _, out _, out _, out _, out _, out _); - var found = ReadSizeUnknown(context | DiskANNService.Attributes, element.AsReadOnlySpan(), ref outputAttributes); + ReadIndex(indexValue, out var context, out _, out _, out _, out _, out _, out _, out _); + var found = ReadSizeUnknown(context | DiskANNService.Attributes, forceAlignment: true, element, ref outputAttributes); return found ? VectorManagerResult.OK : VectorManagerResult.MissingElement; } @@ -768,7 +725,7 @@ internal VectorManagerResult FetchSingleVectorElementAttributes(ReadOnlySpan private void FetchVectorElementAttributes(ulong context, int numIds, SpanByteAndMemory ids, ref SpanByteAndMemory attributes) { - var remainingIds = ids.AsReadOnlySpan(); + var remainingIds = ids.ReadOnlySpan; GCHandle idPin = default; byte[] idWithNamespaceArr = null; @@ -817,25 +774,25 @@ private void FetchVectorElementAttributes(ulong context, int numIds, SpanByteAnd attributeMem.Length = attributeMem.SpanByte.Length; } - var found = ReadSizeUnknown(context | DiskANNService.Attributes, id, ref attributeMem); + var found = ReadSizeUnknown(context | DiskANNService.Attributes, forceAlignment: true, id, ref attributeMem); // Copy attribute into output buffer, length prefixed, resizing as necessary var neededSpace = 4 + (found ? attributeMem.Length : 0); - var destSpan = attributes.AsSpan()[attributesNextIx..]; + var destSpan = attributes.Span[attributesNextIx..]; if (destSpan.Length < neededSpace) { var newAttrArr = MemoryPool.Shared.Rent(attributes.Length + neededSpace); - attributes.AsReadOnlySpan().CopyTo(newAttrArr.Memory.Span); + attributes.ReadOnlySpan.CopyTo(newAttrArr.Memory.Span); attributes.Memory?.Dispose(); attributes = new SpanByteAndMemory(newAttrArr, newAttrArr.Memory.Length); - destSpan = attributes.AsSpan()[attributesNextIx..]; + destSpan = attributes.Span[attributesNextIx..]; } BinaryPrimitives.WriteInt32LittleEndian(destSpan, attributeMem.Length); - attributeMem.AsReadOnlySpan().CopyTo(destSpan[sizeof(int)..]); + attributeMem.ReadOnlySpan.CopyTo(destSpan[sizeof(int)..]); attributesNextIx += neededSpace; @@ -863,7 +820,7 @@ internal bool TryGetEmbedding(ReadOnlySpan indexValue, ReadOnlySpan { AssertHaveStorageSession(); - ReadIndex(indexValue, out var context, out var dimensions, out _, out var quantType, out _, out _, out _, out var indexPtr, out _); + ReadIndex(indexValue, out var context, out var dimensions, out _, out var quantType, out _, out _, out _, out var indexPtr); // Make sure enough space in distances for requested count if (dimensions * sizeof(float) > outputDistances.Length) @@ -884,7 +841,7 @@ internal bool TryGetEmbedding(ReadOnlySpan indexValue, ReadOnlySpan var internalIdBytes = SpanByteAndMemory.FromPinnedSpan(internalId); try { - if (!ReadSizeUnknown(context | DiskANNService.InternalIdMap, element, ref internalIdBytes)) + if (!ReadSizeUnknown(context | DiskANNService.InternalIdMap, forceAlignment: true, element, ref internalIdBytes)) { return false; } @@ -896,18 +853,18 @@ internal bool TryGetEmbedding(ReadOnlySpan indexValue, ReadOnlySpan internalIdBytes.Memory?.Dispose(); } - Span asBytesSpan = stackalloc byte[(int)dimensions]; + Span asBytesSpan = stackalloc byte[1024]; var asBytes = SpanByteAndMemory.FromPinnedSpan(asBytesSpan); try { - if (!ReadSizeUnknown(context | DiskANNService.FullVector, internalId, ref asBytes)) + if (!ReadSizeUnknown(context | DiskANNService.FullVector, forceAlignment: true, internalId, ref asBytes)) { return false; } - var into = MemoryMarshal.Cast(outputDistances.AsSpan()); + var into = MemoryMarshal.Cast(outputDistances.Span); - var from = asBytes.AsReadOnlySpan(); + var from = asBytes.ReadOnlySpan; if (quantType == VectorQuantType.NoQuant) { var fromFloat = MemoryMarshal.Cast(from); diff --git a/libs/server/ServerConfig.cs b/libs/server/ServerConfig.cs index 02856cccd79..2c2836a1699 100644 --- a/libs/server/ServerConfig.cs +++ b/libs/server/ServerConfig.cs @@ -109,9 +109,7 @@ ReadOnlySpan GetDatabases() private unsafe bool NetworkCONFIG_REWRITE() { if (parseState.Count != 0) - { return AbortWithWrongNumberOfArguments($"{nameof(RespCommand.CONFIG)}|{nameof(CmdStrings.REWRITE)}"); - } storeWrapper.clusterProvider?.FlushConfig(); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) @@ -123,19 +121,15 @@ private unsafe bool NetworkCONFIG_REWRITE() private unsafe bool NetworkCONFIG_SET() { if (parseState.Count == 0 || parseState.Count % 2 != 0) - { return AbortWithWrongNumberOfArguments($"{nameof(RespCommand.CONFIG)}|{nameof(CmdStrings.SET)}"); - } string certFileName = null; string certPassword = null; string clusterUsername = null; string clusterPassword = null; - string memorySize = null; - string objLogMemory = null; - string objHeapMemory = null; + string mainLogMemorySize = null; + string readCacheMemorySize = null; string index = null; - string objIndex = null; var unknownOption = false; var unknownKey = ""; @@ -145,16 +139,12 @@ private unsafe bool NetworkCONFIG_SET() var key = parseState.GetArgSliceByRef(c).ReadOnlySpan; var value = parseState.GetArgSliceByRef(c + 1).ReadOnlySpan; - if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.Memory, allowNonAlphabeticChars: false)) - memorySize = Encoding.ASCII.GetString(value); - else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.ObjLogMemory, allowNonAlphabeticChars: true)) - objLogMemory = Encoding.ASCII.GetString(value); - else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.ObjHeapMemory, allowNonAlphabeticChars: true)) - objHeapMemory = Encoding.ASCII.GetString(value); + if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.MainLogMemory, allowNonAlphabeticChars: false)) + mainLogMemorySize = Encoding.ASCII.GetString(value); + else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.ReadCacheMemory, allowNonAlphabeticChars: false)) + readCacheMemorySize = Encoding.ASCII.GetString(value); else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.Index, allowNonAlphabeticChars: false)) index = Encoding.ASCII.GetString(value); - else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.ObjIndex, allowNonAlphabeticChars: true)) - objIndex = Encoding.ASCII.GetString(value); else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.CertFileName, allowNonAlphabeticChars: true)) certFileName = Encoding.ASCII.GetString(value); else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.CertPassword, allowNonAlphabeticChars: true)) @@ -163,22 +153,17 @@ private unsafe bool NetworkCONFIG_SET() clusterUsername = Encoding.ASCII.GetString(value); else if (key.EqualsLowerCaseSpanIgnoringCase(CmdStrings.ClusterPassword, allowNonAlphabeticChars: true)) clusterPassword = Encoding.ASCII.GetString(value); - else + else if (!unknownOption) { - if (!unknownOption) - { - unknownOption = true; - unknownKey = Encoding.ASCII.GetString(key); - } + unknownOption = true; + unknownKey = Encoding.ASCII.GetString(key); } } var sbErrorMsg = new StringBuilder(); if (unknownOption) - { AppendError(sbErrorMsg, string.Format(CmdStrings.GenericErrUnknownOptionConfigSet, unknownKey)); - } else { if (clusterUsername != null || clusterPassword != null) @@ -188,9 +173,7 @@ private unsafe bool NetworkCONFIG_SET() if (storeWrapper.clusterProvider != null) storeWrapper.clusterProvider?.UpdateClusterAuth(clusterUsername, clusterPassword); else - { AppendError(sbErrorMsg, "ERR Cluster is disabled."); - } } if (certFileName != null || certPassword != null) @@ -198,36 +181,21 @@ private unsafe bool NetworkCONFIG_SET() if (storeWrapper.serverOptions.TlsOptions != null) { if (!storeWrapper.serverOptions.TlsOptions.UpdateCertFile(certFileName, certPassword, out var certErrorMessage)) - { AppendError(sbErrorMsg, certErrorMessage); - } } else - { - sbErrorMsg.AppendLine("ERR TLS is disabled."); - } + _ = sbErrorMsg.AppendLine("ERR TLS is disabled."); } - if (memorySize != null) - HandleMemorySizeChange(memorySize, sbErrorMsg); - - if (objLogMemory != null) - HandleMemorySizeChange(objLogMemory, sbErrorMsg, mainStore: false); - + if (mainLogMemorySize != null) + HandleMemorySizeChange(mainLogMemorySize, sbErrorMsg, isReadCache: false); + if (readCacheMemorySize != null) + HandleMemorySizeChange(readCacheMemorySize, sbErrorMsg, isReadCache: true); if (index != null) { // Must block, we're on the network thread AsyncUtils.BlockingWait(HandleIndexSizeChangeAsync(index, sbErrorMsg)); } - - if (objIndex != null) - { - // Must block, we're on the network thread - AsyncUtils.BlockingWait(HandleIndexSizeChangeAsync(objIndex, sbErrorMsg, mainStore: false)); - } - - if (objHeapMemory != null) - HandleObjHeapMemorySizeChange(objHeapMemory, sbErrorMsg); } if (sbErrorMsg.Length == 0) @@ -244,58 +212,56 @@ private unsafe bool NetworkCONFIG_SET() return true; } - private void HandleMemorySizeChange(string memorySize, StringBuilder sbErrorMsg, bool mainStore = true) + private void HandleMemorySizeChange(string memorySize, StringBuilder sbErrorMsg, bool isReadCache) { - var option = mainStore ? CmdStrings.Memory : CmdStrings.ObjLogMemory; - if (!ServerOptions.TryParseSize(memorySize, out var newMemorySize)) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIncorrectSizeFormat, option); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIncorrectSizeFormat, CmdStrings.MainLogMemory); return; } // Parse the configured memory size - var confMemorySize = ServerOptions.ParseSize( - mainStore ? storeWrapper.serverOptions.MemorySize - : storeWrapper.serverOptions.ObjectStoreLogMemorySize, out _); - // If the new memory size is the same as the configured memory size, nothing to do + var confMemorySize = ServerOptions.ParseSize(storeWrapper.serverOptions.LogMemorySize, out _); if (newMemorySize == confMemorySize) return; // Calculate the buffer size based on the configured memory size - confMemorySize = ServerOptions.NextPowerOf2(confMemorySize); - // If the new memory size is greater than the configured memory size, return an error + confMemorySize = ServerOptions.NextPowerOf2(confMemorySize); if (newMemorySize > confMemorySize) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrMemorySizeGreaterThanBuffer, option); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrMemorySizeGreaterThanBuffer, CmdStrings.MainLogMemory); return; } - // Parse & adjust the configured page size - var pageSize = ServerOptions.ParseSize( - mainStore ? storeWrapper.serverOptions.PageSize : storeWrapper.serverOptions.ObjectStorePageSize, - out _); - pageSize = ServerOptions.PreviousPowerOf2(pageSize); - - // Compute the new minimum empty page count and update the store's log accessor - var newMinEmptyPageCount = (int)((confMemorySize - newMemorySize) / pageSize); - if (mainStore) + // If the size tracker is not running for the specified allocator, return an error + if (isReadCache) { - storeWrapper.store.Log.MinEmptyPageCount = newMinEmptyPageCount; + if (storeWrapper.sizeTracker?.readCacheTracker is null) + { + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrReadCacheMemorySizeTrackerNotRunning, CmdStrings.ReadCacheMemory); + return; + } } - else + else if (storeWrapper.sizeTracker?.mainLogTracker is null) { - storeWrapper.objectStore.Log.MinEmptyPageCount = newMinEmptyPageCount; + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrMainLogMemorySizeTrackerNotRunning, CmdStrings.MainLogMemory); + return; } + + // Set the new target size for the object store size tracker + if (isReadCache) + storeWrapper.sizeTracker.ReadCacheTargetSize = newMemorySize; + else + storeWrapper.sizeTracker.TargetSize = newMemorySize; } - private async Task HandleIndexSizeChangeAsync(string indexSize, StringBuilder sbErrorMsg, bool mainStore = true) + private async Task HandleIndexSizeChangeAsync(string indexSize, StringBuilder sbErrorMsg) { if (!ServerOptions.TryParseSize(indexSize, out var newIndexSize)) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIncorrectSizeFormat, GetOption(mainStore)); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIncorrectSizeFormat, CmdStrings.Index); return; } @@ -303,82 +269,45 @@ private async Task HandleIndexSizeChangeAsync(string indexSize, StringBuilder sb var adjNewIndexSize = ServerOptions.PreviousPowerOf2(newIndexSize); if (adjNewIndexSize != newIndexSize) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizePowerOfTwo, GetOption(mainStore)); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizePowerOfTwo, CmdStrings.Index); return; } // Check if the index auto-grow task is running. If so - return an error. - if ((mainStore && storeWrapper.serverOptions.AdjustedIndexMaxCacheLines > 0) || - (!mainStore && storeWrapper.serverOptions.AdjustedObjectStoreIndexMaxCacheLines > 0)) + if (storeWrapper.serverOptions.AdjustedIndexMaxCacheLines > 0) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeAutoGrow, GetOption(mainStore)); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeAutoGrow, CmdStrings.Index); return; } - var currIndexSize = mainStore ? storeWrapper.store.IndexSize : storeWrapper.objectStore.IndexSize; + var currIndexSize = storeWrapper.store.IndexSize; // Convert new index size to cache lines - adjNewIndexSize /= 64; - // If the current index size is the same as the new index size, nothing to do + adjNewIndexSize /= 64; if (currIndexSize == adjNewIndexSize) return; // If the new index size is smaller than the current index size, return an error if (currIndexSize > adjNewIndexSize) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeSmallerThanCurrent, GetOption(mainStore)); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeSmallerThanCurrent, CmdStrings.Index); return; } // Try to grow the index size by doubling it until it reaches the new size - while (currIndexSize < adjNewIndexSize) + for (; currIndexSize < adjNewIndexSize; currIndexSize *= 2) { - var isSuccessful = - await - (mainStore - ? storeWrapper.store.GrowIndexAsync() - : storeWrapper.objectStore.GrowIndexAsync() - ) - .ConfigureAwait(false); - - if (!isSuccessful) + if (!AsyncUtils.BlockingWait(storeWrapper.store.GrowIndexAsync())) { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeGrowFailed, GetOption(mainStore)); + AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIndexSizeGrowFailed, CmdStrings.Index); return; } - - currIndexSize *= 2; - } - - // Can't keep Span local in async method, so helper to get one on demand - static ReadOnlySpan GetOption(bool mainStore) - => mainStore ? CmdStrings.Index : CmdStrings.ObjIndex; - } - - private void HandleObjHeapMemorySizeChange(string heapMemorySize, StringBuilder sbErrorMsg) - { - if (!ServerOptions.TryParseSize(heapMemorySize, out var newHeapMemorySize)) - { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrIncorrectSizeFormat, CmdStrings.ObjHeapMemory); - return; } - - // If the object store size tracker is not running, return an error - if (storeWrapper.objectStoreSizeTracker == null) - { - AppendErrorWithTemplate(sbErrorMsg, CmdStrings.GenericErrHeapMemorySizeTrackerNotRunning, CmdStrings.ObjHeapMemory); - return; - } - - // Set the new target size for the object store size tracker - storeWrapper.objectStoreSizeTracker.TargetSize = newHeapMemorySize; } private static void AppendError(StringBuilder sbErrorMsg, string error) - { - sbErrorMsg.Append($"{(sbErrorMsg.Length == 0 ? error : $"; {error.Substring(4)}")}"); - } + => _ = sbErrorMsg.Append($"{(sbErrorMsg.Length == 0 ? error : $"; {error.Substring(4)}")}"); private static void AppendErrorWithTemplate(StringBuilder sbErrorMsg, string template, ReadOnlySpan option) { diff --git a/libs/server/Servers/GarnetServerBase.cs b/libs/server/Servers/GarnetServerBase.cs index b326c9f3d30..77ebdfaff89 100644 --- a/libs/server/Servers/GarnetServerBase.cs +++ b/libs/server/Servers/GarnetServerBase.cs @@ -168,8 +168,9 @@ public virtual void Dispose() internal void DisposeActiveHandlers() { logger?.LogTrace("Begin disposing active handlers"); -#if HANGDETECT - int count = 0; +#if DEBUG + var sw = System.Diagnostics.Stopwatch.StartNew(); + var diagnosed = false; #endif while (activeHandlerCount >= 0) { @@ -180,11 +181,16 @@ internal void DisposeActiveHandlers() var _handler = kvp.Key; _handler?.Dispose(); } - Thread.Yield(); -#if HANGDETECT - if (++count % 10000 == 0) - logger?.LogTrace("Dispose iteration {count}, {activeHandlerCount}", count, activeHandlerCount); +#if DEBUG + if (!diagnosed && sw.ElapsedMilliseconds > 5_000) + { + diagnosed = true; + logger?.LogError("DisposeActiveHandlers blocked with activeHandlerCount={activeHandlerCount}. Active handlers:", activeHandlerCount); + foreach (var kvp in activeHandlers) + logger?.LogError(" Stuck handler: {handlerType}", kvp.Key?.GetType().FullName); + } #endif + Thread.Yield(); } if (Interlocked.CompareExchange(ref activeHandlerCount, int.MinValue, 0) == 0) break; diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index ad3c0db7846..b496df59635 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -21,42 +21,6 @@ public class GarnetServerOptions : ServerOptions /// public bool DisableObjects = false; - /// - /// Heap memory size limit of object store. - /// - public string ObjectStoreHeapMemorySize = ""; - - /// - /// Object store log memory used in bytes excluding heap memory. - /// - public string ObjectStoreLogMemorySize = "32m"; - - /// - /// Size of each object store page in bytes (rounds down to power of 2). - /// - public string ObjectStorePageSize = "4k"; - - /// - /// Size of each object store log segment in bytes on disk (rounds down to power of 2). - /// - public string ObjectStoreSegmentSize = "32m"; - - /// - /// Size of object store hash index in bytes (rounds down to power of 2). - /// - public string ObjectStoreIndexSize = "16m"; - - /// - /// Max size of object store hash index in bytes (rounds down to power of 2). - /// If unspecified, index size doesn't grow (default behavior). - /// - public string ObjectStoreIndexMaxSize = string.Empty; - - /// - /// Percentage of object store log memory that is kept mutable. - /// - public int ObjectStoreMutablePercent = 90; - /// /// Enable cluster. /// @@ -87,10 +51,14 @@ public class GarnetServerOptions : ServerOptions /// public bool EnableAOF = false; - // Enable Lua scripts on server + /// + /// Enable Lua scripts on server + /// public bool EnableLua = false; - // Run Lua scripts as a transaction (lock keys - run script - unlock keys) + /// + /// Run Lua scripts as a transaction (lock keys - run script - unlock keys) + /// public bool LuaTransactionMode = false; /// @@ -104,14 +72,25 @@ public class GarnetServerOptions : ServerOptions public string AofPageSize = "4m"; /// - /// AOF replication (safe tail address) refresh frequency in milliseconds. 0 = auto refresh after every enqueue. + /// Size of each AOF segment (file) in bytes on disk (rounds down to power of 2). + /// This is the granularity at which AOF files are created and truncated. /// - public int AofReplicationRefreshFrequencyMs = 10; + public string AofSegmentSize = "1g"; /// - /// Subscriber (safe tail address) refresh frequency in milliseconds (for pub-sub). 0 = auto refresh after every enqueue. + /// Number of AOF physical sublogs (i.e. TsavoriteLog instances) used (=1 equivalent to the legacy single log implementation >1: sharded log implementation. /// - public int SubscriberRefreshFrequencyMs = 0; + public int AofPhysicalSublogCount = 1; + + /// + /// Number of replay tasks per physical sublog at the replica. + /// + public int AofReplayTaskCount = 1; + + /// + /// Polling frequency of the background task responsible for moving time ahead for all physical sublogs (Used only with physical sublog value >1). + /// + public int AofTailWitnessFreqMs = 100; /// /// Write ahead logging (append-only file) commit issue frequency in milliseconds. @@ -135,6 +114,12 @@ public class GarnetServerOptions : ServerOptions /// public int IndexResizeThreshold = 50; + /// + /// The size at which a value string becomes an overflow byte[]. Accepts bytes or k/m/g suffixes (e.g. "4k", "1m"). + /// Valid range: 64 bytes to 256m. Rounds down to previous power of 2 by Tsavorite. + /// + public string ValueOverflowThreshold = "16k"; + /// /// Wait for AOF to commit before returning results to client. /// Warning: will greatly increase operation latency. @@ -181,11 +166,6 @@ public class GarnetServerOptions : ServerOptions /// public int CompactionMaxSegments = 32; - /// - /// Number of object store log segments created on disk before compaction triggers. - /// - public int ObjectStoreCompactionMaxSegments = 32; - /// /// Percent of cluster nodes to gossip with at each gossip iteration. /// @@ -262,21 +242,11 @@ public class GarnetServerOptions : ServerOptions /// public bool QuietMode = false; - /// - /// SAVE and BGSAVE: Enable incremental snapshots, try to write only changes compared to base snapshot - /// - public bool EnableIncrementalSnapshots = false; - /// /// SAVE and BGSAVE: We will take a full (index + log) checkpoint when ReadOnlyAddress of log increases by this amount, from the last full checkpoint. /// public long FullCheckpointLogInterval = 1L << 30; - /// - /// SAVE and BGSAVE: Limit on size of delta log for incremental snapshot, we perform a non-incremental checkpoint after this limit is reached. - /// - public long IncrementalSnapshotLogSizeLimit = 1L << 30; - /// /// SAVE and BGSAVE: Use fold-over checkpoints instead of snapshots. /// @@ -319,10 +289,6 @@ public class GarnetServerOptions : ServerOptions /// public int CheckpointThrottleFlushDelayMs = 0; - /// - /// Enable FastCommit mode for TsavoriteLog - /// - public bool EnableFastCommit = true; /// /// Throttle FastCommit to write metadata once every K commits @@ -390,9 +356,9 @@ public class GarnetServerOptions : ServerOptions /// public bool UseAofNullDevice = false; - // - // Use specified device type - // + /// + /// Use specified device type + /// public DeviceType DeviceType = DeviceType.Default; /// @@ -437,11 +403,6 @@ public class GarnetServerOptions : ServerOptions /// public bool RevivInChainOnly; - /// - /// Number of records in the single free record bin for the object store. - /// - public int RevivObjBinRecordCount; - /// Max size of hash index (cache lines) after rounding down size in bytes to power of 2. public int AdjustedIndexMaxCacheLines; @@ -471,20 +432,25 @@ public class GarnetServerOptions : ServerOptions /// List of modules to load public IEnumerable LoadModuleCS; + /// Whether the read cache is enabled public bool EnableReadCache = false; - public string ReadCacheMemorySize = "16g"; - - public string ReadCachePageSize = "32m"; - - public string ObjectStoreReadCachePageSize = "1m"; - - public string ObjectStoreReadCacheLogMemorySize = "32m"; + /// + /// Total readcache-log memory (inline and heap) to use if readcache is enabled, in bytes. Does not need to be a power of 2 + /// + public string ReadCacheMemorySize = "1g"; - public string ObjectStoreReadCacheHeapMemorySize = ""; + /// + /// Size of each read cache page in bytes (rounds down to power of 2) + /// + public string ReadCachePageSize = "4m"; - public bool EnableObjectStoreReadCache = false; + /// + /// Number of readcache-log pages (rounds down to power of 2). This allows specifying less pages initially than ReadCacheMemorySize divided by ReadCachePageSize. + /// + public int ReadCachePageCount = 0; + /// Options for Lua script execution public LuaOptions LuaOptions; /// @@ -515,12 +481,7 @@ public class GarnetServerOptions : ServerOptions /// /// Gets the base directory for storing main-store checkpoints /// - public string MainStoreCheckpointBaseDirectory => Path.Combine(CheckpointBaseDirectory, "Store"); - - /// - /// Gets the base directory for storing object-store checkpoints - /// - public string ObjectStoreCheckpointBaseDirectory => Path.Combine(CheckpointBaseDirectory, "ObjectStore"); + public string StoreCheckpointBaseDirectory => Path.Combine(CheckpointBaseDirectory, "Store"); /// /// Seconds between attempts to re-establish replication between a Primary and Replica if the replication connection @@ -538,6 +499,14 @@ public class GarnetServerOptions : ServerOptions /// public bool ClusterReplicaResumeWithData = false; + /// + /// Check if the startup configuration allows the possibility of data loss during replication + /// NOTE: null device cannot or FastAofTruncate without OnDemandCheckpoint cannot guarantee the integrity of replication + /// since the AOF is being truncated aggresively. + /// + public bool AllowDataLoss + => UseAofNullDevice || (FastAofTruncate && !OnDemandCheckpoint); + /// /// If true, enable Vector Set commands. /// @@ -551,27 +520,26 @@ public class GarnetServerOptions : ServerOptions public int VectorSetReplayTaskCount = 0; /// - /// Get the directory name for database checkpoints + /// If true, enable Range Index commands (RI.CREATE, RI.SET, RI.GET, etc.). + /// + /// This is a preview feature, subject to substantial change, and should not be relied upon. /// - /// Database Id - /// Directory name - public string GetCheckpointDirectoryName(int dbId) => $"checkpoints{(dbId == 0 ? string.Empty : $"_{dbId}")}"; + public bool EnableRangeIndexPreview = false; /// - /// Get the directory for main-store database checkpoints + /// Get the directory name for database checkpoints /// /// Database Id - /// Directory - public string GetMainStoreCheckpointDirectory(int dbId) => - Path.Combine(MainStoreCheckpointBaseDirectory, GetCheckpointDirectoryName(dbId)); + /// Directory name + public static string GetCheckpointDirectoryName(int dbId) => $"checkpoints{(dbId == 0 ? string.Empty : $"_{dbId}")}"; /// - /// Get the directory for object-store database checkpoints + /// Get the directory for database checkpoints /// /// Database Id /// Directory - public string GetObjectStoreCheckpointDirectory(int dbId) => - Path.Combine(ObjectStoreCheckpointBaseDirectory, GetCheckpointDirectoryName(dbId)); + public string GetStoreCheckpointDirectory(int dbId) => + Path.Combine(StoreCheckpointBaseDirectory, GarnetServerOptions.GetCheckpointDirectoryName(dbId)); /// /// Gets the base directory for storing AOF commits @@ -583,7 +551,7 @@ public string GetObjectStoreCheckpointDirectory(int dbId) => /// /// Database Id /// Directory name - public string GetAppendOnlyFileDirectoryName(int dbId) => $"AOF{(dbId == 0 ? string.Empty : $"_{dbId}")}"; + public static string GetAppendOnlyFileDirectoryName(int dbId) => $"AOF{(dbId == 0 ? string.Empty : $"_{dbId}")}"; /// /// Get the directory for database AOF commits @@ -618,15 +586,14 @@ public void Initialize(ILoggerFactory loggerFactory = null) /// Tsavorite Log factory instance /// /// - public KVSettings GetSettings(ILoggerFactory loggerFactory, LightEpoch epoch, StateMachineDriver stateMachineDriver, - out INamedDeviceFactory logFactory) + public KVSettings GetSettings(ILoggerFactory loggerFactory, LightEpoch epoch, StateMachineDriver stateMachineDriver, out INamedDeviceFactory logFactory) { if (MutablePercent is < 10 or > 95) throw new Exception("MutablePercent must be between 10 and 95"); - var indexCacheLines = IndexSizeCachelines("hash index size", IndexSize); + var indexCacheLines = IndexSizeCachelines("hash index size", IndexMemorySize); - KVSettings kvSettings = new() + KVSettings kvSettings = new() { IndexSize = indexCacheLines * 64L, PreallocateLog = false, @@ -634,33 +601,54 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, PageSize = 1L << PageSizeBits(), Epoch = epoch, StateMachineDriver = stateMachineDriver, + MaxInlineValueSize = ValueOverflowThresholdBytes(), loggerFactory = loggerFactory, logger = loggerFactory?.CreateLogger("TsavoriteKV [main]") }; + if (!string.IsNullOrEmpty(LogMemorySize)) + kvSettings.LogMemorySize = ParseSize(LogMemorySize, out _); + + if (PageCount != 0) + kvSettings.PageCount = PageCount; + logger?.LogInformation("[Store] Using page size of {PageSize}", PrettySize(kvSettings.PageSize)); + logger?.LogInformation("[Store] Each page can hold ~{PageSize} key-value pairs of objects", kvSettings.PageSize / 24); - kvSettings.MemorySize = 1L << MemorySizeBits(MemorySize, PageSize, out var storeEmptyPageCount); - kvSettings.MinEmptyPageCount = storeEmptyPageCount; + if (kvSettings.LogMemorySize > 0) + { + var pageCount = kvSettings.LogMemorySize / kvSettings.PageSize; + if (kvSettings.PageCount > pageCount) + logger?.LogInformation("[Store] Warning: overriding specified PageCount of {kvSettingsPageCount} with smaller page count calculated from LogMemorySize limit divided by PageSize: {pageCount}", kvSettings.PageCount, pageCount); - long effectiveSize = kvSettings.MemorySize - storeEmptyPageCount * kvSettings.PageSize; - if (storeEmptyPageCount == 0) - logger?.LogInformation("[Store] Using log memory size of {MemorySize}", PrettySize(kvSettings.MemorySize)); + var bufferSize = NextPowerOf2(pageCount); + logger?.LogInformation("[Store] There are {LogPages} log pages in memory, of which {pageCount} are initially allocated", PrettySize(bufferSize), pageCount); + logger?.LogInformation("[Store] Log memory size limit of {LogMemorySize} will be enforced", PrettySize(kvSettings.LogMemorySize)); + } else - logger?.LogInformation("[Store] Using log memory size of {MemorySize}, with {storeEmptyPageCount} empty pages, for effective size of {effectiveSize}", - PrettySize(kvSettings.MemorySize), storeEmptyPageCount, PrettySize(effectiveSize)); - - logger?.LogInformation("[Store] There are {LogPages} log pages in memory", PrettySize(kvSettings.MemorySize / kvSettings.PageSize)); + { + if (kvSettings.PageCount == 0) + throw new TsavoriteException($"Store Log Memory size or PageCount must be specified"); + var bufferSize = (int)NextPowerOf2(kvSettings.PageCount); + if (kvSettings.PageCount < bufferSize) + { + logger?.LogInformation("[Store] Warning: overriding specified PageCount of {kvSettingsPageCount} with next power of 2 page count {bufferSize}", kvSettings.PageCount, bufferSize); + kvSettings.PageCount = bufferSize; + } + logger?.LogInformation("[Store] There are {LogPages} log pages in memory, all of which will be used because there is no MemorySize limit", PrettySize(bufferSize)); + logger?.LogInformation("[Store] No log memory size limit will be enforced"); + } - kvSettings.SegmentSize = 1L << SegmentSizeBits(); + kvSettings.SegmentSize = 1L << SegmentSizeBits(isObj: false); + kvSettings.ObjectLogSegmentSize = 1L << SegmentSizeBits(isObj: true); logger?.LogInformation("[Store] Using disk segment size of {SegmentSize}", PrettySize(kvSettings.SegmentSize)); - logger?.LogInformation("[Store] Using hash index size of {IndexSize} ({indexCacheLines} cache lines)", PrettySize(kvSettings.IndexSize), PrettySize(indexCacheLines)); + logger?.LogInformation("[Store] Using hash index size of {IndexMemorySize} ({indexCacheLines} cache lines)", PrettySize(kvSettings.IndexSize), PrettySize(indexCacheLines)); logger?.LogInformation("[Store] Hash index size is optimized for up to ~{distinctKeys} distinct keys", PrettySize(indexCacheLines * 4L)); - AdjustedIndexMaxCacheLines = IndexMaxSize == string.Empty ? 0 : IndexSizeCachelines("hash index max size", IndexMaxSize); + AdjustedIndexMaxCacheLines = IndexMaxMemorySize == string.Empty ? 0 : IndexSizeCachelines("hash index max size", IndexMaxMemorySize); if (AdjustedIndexMaxCacheLines != 0 && AdjustedIndexMaxCacheLines < indexCacheLines) - throw new Exception($"Index size {IndexSize} should not be less than index max size {IndexMaxSize}"); + throw new Exception($"Index size {IndexMemorySize} should not be less than index max size {IndexMaxMemorySize}"); if (AdjustedIndexMaxCacheLines > 0) { @@ -670,29 +658,49 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, logger?.LogInformation("[Store] Using log mutable percentage of {MutablePercent}%", MutablePercent); if (DeviceType == DeviceType.Default) - { DeviceType = Devices.GetDefaultDeviceType(); - } DeviceFactoryCreator ??= new LocalStorageNamedDeviceFactoryCreator(deviceType: DeviceType, logger: logger); - logger?.LogInformation("Using device type {deviceType}", DeviceType); if (LatencyMonitor && MetricsSamplingFrequency == 0) throw new Exception("LatencyMonitor requires MetricsSamplingFrequency to be set"); // Read cache related settings - if (EnableReadCache && !EnableStorageTier) - { - throw new Exception("Read cache requires storage tiering to be enabled"); - } - if (EnableReadCache) { + if (!EnableStorageTier) + throw new Exception("Read cache requires storage tiering to be enabled"); kvSettings.ReadCacheEnabled = true; - kvSettings.ReadCachePageSize = ParseSize(ReadCachePageSize, out _); + + if (ReadCachePageCount != 0) + kvSettings.ReadCachePageCount = ReadCachePageCount; + + kvSettings.ReadCachePageSize = 1L << ReadCachePageSizeBits(); kvSettings.ReadCacheMemorySize = ParseSize(ReadCacheMemorySize, out _); - logger?.LogInformation("[Store] Read cache enabled with page size of {ReadCachePageSize} and memory size of {ReadCacheMemorySize}", - PrettySize(kvSettings.ReadCachePageSize), PrettySize(kvSettings.ReadCacheMemorySize)); + + if (kvSettings.ReadCacheMemorySize > 0) + { + var pageCount = kvSettings.ReadCacheMemorySize / kvSettings.ReadCachePageSize; + if (kvSettings.PageCount > pageCount) + logger?.LogInformation("[Store] Warning: Read cache overriding specified PageCount of {kvSettingsReadCachePageCount} with smaller page count calculated from LogMemorySize limit divided by PageSize: {pageCount}", kvSettings.ReadCachePageCount, pageCount); + + var bufferSize = NextPowerOf2(pageCount); + logger?.LogInformation("[Store] Read cache enabled with {LogPages} log pages in memory, of which {pageCount} are initially allocated", PrettySize(bufferSize), pageCount); + logger?.LogInformation("[Store] Read cache Log memory size limit of {LogMemorySize} will be enforced", PrettySize(kvSettings.ReadCacheMemorySize)); + } + else + { + if (kvSettings.ReadCachePageCount == 0) + throw new TsavoriteException($"Read Cache Log Memory size or PageCount must be specified"); + var bufferSize = (int)NextPowerOf2(kvSettings.ReadCachePageCount); + if (kvSettings.PageCount < bufferSize) + { + logger?.LogInformation("[Store] Warning: Read cache overriding specified PageCount of {kvSettingsReadCachePageCount} with next power of 2 page count {bufferSize}", kvSettings.ReadCachePageCount, bufferSize); + kvSettings.PageCount = bufferSize; + } + logger?.LogInformation("[Store] Read cache enabled with {LogPages} log pages in memory, all of which will be used because there is no MemorySize limit", PrettySize(bufferSize)); + logger?.LogInformation("[Store] No Read cache log memory size limit will be enforced"); + } } if (EnableStorageTier) @@ -700,7 +708,11 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, if (LogDir is null or "") LogDir = Directory.GetCurrentDirectory(); logFactory = GetInitializedDeviceFactory(LogDir); + + // These must match GetInitializedSegmentFileDevice.GetStoreHLogDevice kvSettings.LogDevice = logFactory.Get(new FileDescriptor("Store", "hlog")); + if (!DisableObjects) + kvSettings.ObjectLogDevice = logFactory.Get(new FileDescriptor("Store", "hlog_objs")); } else { @@ -737,7 +749,7 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, FreeRecordBins = new RevivificationBin[RevivBinRecordSizes.Length], RevivifiableFraction = RevivifiableFraction }; - for (var ii = 0; ii < RevivBinRecordSizes.Length; ++ii) + for (var ii = 0; ii < RevivBinRecordSizes.Length; ii++) { var recordCount = RevivBinRecordCounts?.Length switch { @@ -765,135 +777,62 @@ public KVSettings GetSettings(ILoggerFactory loggerFactory, /// Get memory size /// /// - public static int MemorySizeBits(string memorySize, string storePageSize, out int emptyPageCount) + public int MemorySizeBits(string memorySize) { - emptyPageCount = 0; - long size = ParseSize(memorySize, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(memorySize, out _); + var adjustedSize = NextPowerOf2(size); if (size != adjustedSize) - { - adjustedSize *= 2; - long pageSize = ParseSize(storePageSize, out _); - pageSize = PreviousPowerOf2(pageSize); - emptyPageCount = (int)((adjustedSize - size) / pageSize); - } + logger?.LogInformation("Warning: using lower memory size than specified (power of 2)"); return (int)Math.Log(adjustedSize, 2); } /// - /// Get KVSettings for the object store log + /// Get read-cache page size in bits, enforcing the shared minimum. /// - public KVSettings GetObjectStoreSettings(ILoggerFactory loggerFactory, LightEpoch epoch, StateMachineDriver stateMachineDriver, - out long objHeapMemorySize, out long objReadCacheHeapMemorySize) - { - objReadCacheHeapMemorySize = default; - - if (ObjectStoreMutablePercent is < 10 or > 95) - throw new Exception("ObjectStoreMutablePercent must be between 10 and 95"); - - var indexCacheLines = IndexSizeCachelines("object store hash index size", ObjectStoreIndexSize); - KVSettings kvSettings = new() - { - IndexSize = indexCacheLines * 64L, - PreallocateLog = false, - MutableFraction = ObjectStoreMutablePercent / 100.0, - PageSize = 1L << ObjectStorePageSizeBits(), - Epoch = epoch, - StateMachineDriver = stateMachineDriver, - loggerFactory = loggerFactory, - logger = loggerFactory?.CreateLogger("TsavoriteKV [obj]") - }; - - logger?.LogInformation("[Object Store] Using page size of {PageSize}", PrettySize(kvSettings.PageSize)); - logger?.LogInformation("[Object Store] Each page can hold ~{PageSize} key-value pairs of objects", kvSettings.PageSize / 24); - - kvSettings.MemorySize = 1L << MemorySizeBits(ObjectStoreLogMemorySize, ObjectStorePageSize, out var objectStoreEmptyPageCount); - kvSettings.MinEmptyPageCount = objectStoreEmptyPageCount; + internal int ReadCachePageSizeBits() => ValidatedPageSizeBits(ReadCachePageSize, nameof(ReadCachePageSize)); - long effectiveSize = kvSettings.MemorySize - objectStoreEmptyPageCount * kvSettings.PageSize; - if (objectStoreEmptyPageCount == 0) - logger?.LogInformation("[Object Store] Using log memory size of {MemorySize}", PrettySize(kvSettings.MemorySize)); - else - logger?.LogInformation("[Object Store] Using log memory size of {MemorySize}, with {objectStoreEmptyPageCount} empty pages, for effective size of {effectiveSize}", PrettySize(kvSettings.MemorySize), objectStoreEmptyPageCount, PrettySize(effectiveSize)); - - logger?.LogInformation("[Object Store] This can hold ~{PageSize} key-value pairs of objects in memory total", effectiveSize / 24); - - logger?.LogInformation("[Object Store] There are {LogPages} log pages in memory", PrettySize(kvSettings.MemorySize / kvSettings.PageSize)); - - kvSettings.SegmentSize = 1L << ObjectStoreSegmentSizeBits(); - logger?.LogInformation("[Object Store] Using disk segment size of {SegmentSize}", PrettySize(kvSettings.SegmentSize)); - - logger?.LogInformation("[Object Store] Using hash index size of {IndexSize} ({indexCacheLines} cache lines)", PrettySize(kvSettings.IndexSize), PrettySize(indexCacheLines)); - logger?.LogInformation("[Object Store] Hash index size is optimized for up to ~{distinctKeys} distinct keys", PrettySize(indexCacheLines * 4L)); - - AdjustedObjectStoreIndexMaxCacheLines = ObjectStoreIndexMaxSize == string.Empty ? 0 : IndexSizeCachelines("hash index max size", ObjectStoreIndexMaxSize); - if (AdjustedObjectStoreIndexMaxCacheLines != 0 && AdjustedObjectStoreIndexMaxCacheLines < indexCacheLines) - throw new Exception($"Index size {IndexSize} should not be less than index max size {IndexMaxSize}"); - - if (AdjustedObjectStoreIndexMaxCacheLines > 0) - { - logger?.LogInformation("[Object Store] Using hash index max size of {MaxSize}, ({CacheLines} cache lines)", PrettySize(AdjustedObjectStoreIndexMaxCacheLines * 64L), PrettySize(AdjustedObjectStoreIndexMaxCacheLines)); - logger?.LogInformation("[Object Store] Hash index max size is optimized for up to ~{distinctKeys} distinct keys", PrettySize(AdjustedObjectStoreIndexMaxCacheLines * 4L)); - } - logger?.LogInformation("[Object Store] Using log mutable percentage of {ObjectStoreMutablePercent}%", ObjectStoreMutablePercent); - - objHeapMemorySize = ParseSize(ObjectStoreHeapMemorySize, out _); - logger?.LogInformation("[Object Store] Heap memory size is {objHeapMemorySize}", objHeapMemorySize > 0 ? PrettySize(objHeapMemorySize) : "unlimited"); - - // Read cache related settings - if (EnableObjectStoreReadCache && !EnableStorageTier) - { - throw new Exception("Read cache requires storage tiering to be enabled"); - } - - if (EnableObjectStoreReadCache) - { - kvSettings.ReadCacheEnabled = true; - kvSettings.ReadCachePageSize = ParseSize(ObjectStoreReadCachePageSize, out _); - kvSettings.ReadCacheMemorySize = ParseSize(ObjectStoreReadCacheLogMemorySize, out _); - logger?.LogInformation("[Object Store] Read cache enabled with page size of {ReadCachePageSize} and memory size of {ReadCacheMemorySize}", - PrettySize(kvSettings.ReadCachePageSize), PrettySize(kvSettings.ReadCacheMemorySize)); - - objReadCacheHeapMemorySize = ParseSize(ObjectStoreReadCacheHeapMemorySize, out _); - logger?.LogInformation("[Object Store] Read cache heap memory size is {objReadCacheHeapMemorySize}", objReadCacheHeapMemorySize > 0 ? PrettySize(objReadCacheHeapMemorySize) : "unlimited"); - } - - if (EnableStorageTier) - { - if (LogDir is null or "") - LogDir = Directory.GetCurrentDirectory(); - kvSettings.LogDevice = GetInitializedDeviceFactory(LogDir).Get(new FileDescriptor("ObjectStore", "hlog")); - kvSettings.ObjectLogDevice = GetInitializedDeviceFactory(LogDir).Get(new FileDescriptor("ObjectStore", "hlog.obj")); - } - else - { - if (LogDir != null) - throw new Exception("LogDir specified without enabling tiered storage (UseStorage)"); - kvSettings.LogDevice = kvSettings.ObjectLogDevice = new NullDevice(); - } - - if (ObjectStoreCopyReadsToTail) - kvSettings.ReadCopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog); - - if (RevivInChainOnly) - { - logger?.LogInformation("[Object Store] Using Revivification in-chain only"); - kvSettings.RevivificationSettings = RevivificationSettings.InChainOnly.Clone(); - } - else if (UseRevivBinsPowerOf2 || RevivBinRecordSizes?.Length > 0) - { - logger?.LogInformation("[Object Store] Using Revivification with a single fixed-size bin"); - kvSettings.RevivificationSettings = RevivificationSettings.DefaultFixedLength.Clone(); - kvSettings.RevivificationSettings.RevivifiableFraction = RevivifiableFraction; - kvSettings.RevivificationSettings.FreeRecordBins[0].NumberOfRecords = RevivObjBinRecordCount; - kvSettings.RevivificationSettings.FreeRecordBins[0].BestFitScanLimit = RevivBinBestFitScanLimit; - } - else + /// + /// Parse and validate as a byte count. + /// Tsavorite requires this to be at least 64 bytes (1 << LogSettings.kLowestMaxInlineSizeBits) + /// and at most 256 MB (1 << (LogSettings.kMaxStringSizeBits - 1)). The value will be rounded down to the previous power of 2. + /// Additionally, the effective (power-of-2) value must be strictly less than the effective PageSize + /// so that a value of this size, plus per-record overhead, can be allocated within a single page; + /// if not, it is clamped down to the largest valid value (with a warning). + /// + /// The byte value used for KVSettings.MaxInlineValueSize. + /// Thrown when the value cannot be parsed or is outside the allowed byte range. + public int ValueOverflowThresholdBytes() + { + const long MinBytes = 64L; // 1 << LogSettings.kLowestMaxInlineSizeBits + const long MaxBytes = 1L << 28; // 1 << (LogSettings.kMaxStringSizeBits - 1) + + if (string.IsNullOrEmpty(ValueOverflowThreshold)) + throw new Exception($"{nameof(ValueOverflowThreshold)} must be specified"); + + if (!TryParseSize(ValueOverflowThreshold, out var sizeInBytes)) + throw new Exception($"Unable to parse {nameof(ValueOverflowThreshold)} value '{ValueOverflowThreshold}'. Expected a memory size string (e.g. '4k', '1m')."); + + if (sizeInBytes < MinBytes || sizeInBytes > MaxBytes) + throw new Exception($"{nameof(ValueOverflowThreshold)} value '{ValueOverflowThreshold}' ({sizeInBytes} bytes) is outside the allowed range [{MinBytes}, {MaxBytes}] bytes."); + + // Cross-property check: a value of MaxInlineValueSize plus per-record overhead must fit on a page. + // Both PageSize and MaxInlineValueSize are rounded down to the previous power of 2 by Tsavorite, + // so we require effectiveValue < effectivePage (i.e., value bits < page bits), which guarantees the + // value occupies at most half the page and leaves room for the record header, key, and optional fields. + // If not satisfied (e.g. defaults combined with an unusually small PageSize), clamp down with a warning + // rather than failing — this preserves the "rounds down silently" behavior of other size settings. + var valueBits = (int)Math.Log(PreviousPowerOf2(sizeInBytes), 2); + var pageBits = PageSizeBits(); + if (valueBits >= pageBits) { - logger?.LogInformation("[Object Store] Not using Revivification"); + var clampedBits = pageBits - 1; + var clampedBytes = 1L << clampedBits; + logger?.LogWarning("Warning: clamping {Name} '{Value}' (effective {EffectiveValue} bytes) down to {Clamped} bytes so it fits within PageSize '{Page}' (effective {EffectivePage} bytes).", + nameof(ValueOverflowThreshold), ValueOverflowThreshold, 1L << valueBits, clampedBytes, PageSize, 1L << pageBits); + return (int)clampedBytes; } - return kvSettings; + return (int)sizeInBytes; } /// @@ -901,33 +840,50 @@ public KVSettings GetObjectStoreSettings(ILoggerFactory l /// /// DB ID /// Tsavorite log settings - public void GetAofSettings(int dbId, LightEpoch epoch, out TsavoriteLogSettings tsavoriteLogSettings) + public void GetAofSettings(int dbId, out TsavoriteLogSettings[] tsavoriteLogSettings) { - tsavoriteLogSettings = new TsavoriteLogSettings - { - MemorySizeBits = AofMemorySizeBits(), - PageSizeBits = AofPageSizeBits(), - LogDevice = GetAofDevice(dbId), - TryRecoverLatest = false, - SafeTailRefreshFrequencyMs = EnableCluster ? AofReplicationRefreshFrequencyMs : -1, - FastCommitMode = EnableFastCommit, - AutoCommit = CommitFrequencyMs == 0, - MutableFraction = 0.9, - Epoch = epoch - }; - if (tsavoriteLogSettings.PageSize > tsavoriteLogSettings.MemorySize) + // Validate sizes up-front (invariant across sublogs) so we don't allocate devices + // or commit managers that would need to be disposed if validation fails. + var memorySizeBits = AofMemorySizeBits(); + var pageSizeBits = AofPageSizeBits(); + var segmentSizeBits = AofSegmentSizeBits(); + + if (pageSizeBits > memorySizeBits) { logger?.LogError("AOF Page size cannot be more than the AOF memory size."); throw new Exception("AOF Page size cannot be more than the AOF memory size."); } - var aofDir = GetAppendOnlyFileDirectory(dbId); - // We use Tsavorite's default checkpoint manager for AOF, since cookie is not needed for AOF commits - tsavoriteLogSettings.LogCommitManager = new DeviceLogCommitCheckpointManager( - FastAofTruncate ? new NullNamedDeviceFactoryCreator() : DeviceFactoryCreator, - new DefaultCheckpointNamingScheme(aofDir), - removeOutdated: true, - fastCommitThrottleFreq: EnableFastCommit ? FastCommitThrottleFreq : 0); + if (pageSizeBits > segmentSizeBits) + { + logger?.LogError("AOF Page size cannot be more than the AOF segment size."); + throw new Exception("AOF Page size cannot be more than the AOF segment size."); + } + + tsavoriteLogSettings = new TsavoriteLogSettings[AofPhysicalSublogCount]; + for (var i = 0; i < AofPhysicalSublogCount; i++) + { + tsavoriteLogSettings[i] = new TsavoriteLogSettings + { + MemorySizeBits = memorySizeBits, + PageSizeBits = pageSizeBits, + SegmentSizeBits = segmentSizeBits, + LogDevice = GetAofDevice(dbId, subLogIdx: AofPhysicalSublogCount == 1 ? -1 : i), + TryRecoverLatest = false, + FastCommitMode = true, + AutoCommit = AofAutoCommit && (AofPhysicalSublogCount == 1), + MutableFraction = 0.9, + Epoch = null + }; + + var aofDir = GetAppendOnlyFileDirectory(dbId); + // We use Tsavorite's default checkpoint manager for AOF, since cookie is not needed for AOF commits + tsavoriteLogSettings[i].LogCommitManager = new DeviceLogCommitCheckpointManager( + FastAofTruncate ? new NullNamedDeviceFactoryCreator() : DeviceFactoryCreator, + new DefaultCheckpointNamingScheme(aofDir, AofPhysicalSublogCount == 1 ? -1 : i), + removeOutdated: true, + fastCommitThrottleFreq: FastCommitThrottleFreq); + } } /// @@ -946,8 +902,8 @@ public INamedDeviceFactory GetInitializedDeviceFactory(string baseName) /// public int AofMemorySizeBits() { - long size = ParseSize(AofMemorySize, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(AofMemorySize, out _); + var adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) logger?.LogInformation("Warning: using lower AOF memory size than specified (power of 2)"); return (int)Math.Log(adjustedSize, 2); @@ -959,36 +915,36 @@ public int AofMemorySizeBits() /// public int AofPageSizeBits() { - long size = ParseSize(AofPageSize, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(AofPageSize, out _); + var adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) logger?.LogInformation("Warning: using lower AOF page size than specified (power of 2)"); return (int)Math.Log(adjustedSize, 2); } /// - /// Get maximum AOF size in bits + /// Get AOF segment size in bits /// /// - public int AofSizeLimitSizeBits() + public int AofSegmentSizeBits() { - long size = ParseSize(AofSizeLimit, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(AofSegmentSize, out _); + var adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) - logger?.LogInformation("Warning: using lower AOF memory size than specified (power of 2)"); + logger?.LogInformation("Warning: using lower AOF segment size than specified (power of 2)"); return (int)Math.Log(adjustedSize, 2); } /// - /// Get object store page size + /// Get maximum AOF size in bits /// /// - public int ObjectStorePageSizeBits() + public int AofSizeLimitSizeBits() { - long size = ParseSize(ObjectStorePageSize, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(AofSizeLimit, out _); + var adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) - logger?.LogInformation("Warning: using lower object store page size than specified (power of 2)"); + logger?.LogInformation("Warning: using lower AOF memory size than specified (power of 2)"); return (int)Math.Log(adjustedSize, 2); } @@ -999,31 +955,45 @@ public int ObjectStorePageSizeBits() public long ReplicaDisklessSyncFullSyncAofThresholdValue() => ParseSize(string.IsNullOrEmpty(ReplicaDisklessSyncFullSyncAofThreshold) ? AofMemorySize : ReplicaDisklessSyncFullSyncAofThreshold, out _); - /// - /// Get object store segment size - /// - /// - public int ObjectStoreSegmentSizeBits() - { - long size = ParseSize(ObjectStoreSegmentSize, out _); - long adjustedSize = PreviousPowerOf2(size); - if (size != adjustedSize) - logger?.LogInformation("Warning: using lower object store disk segment size than specified (power of 2)"); - return (int)Math.Log(adjustedSize, 2); - } - /// /// Get device for AOF /// /// - IDevice GetAofDevice(int dbId) + IDevice GetAofDevice(int dbId, int subLogIdx = -1) { if (UseAofNullDevice && EnableCluster && !FastAofTruncate) throw new Exception("Cannot use null device for AOF when cluster is enabled and you are not using main memory replication"); if (UseAofNullDevice) return new NullDevice(); - return GetInitializedDeviceFactory(AppendOnlyFileBaseDirectory) - .Get(new FileDescriptor(GetAppendOnlyFileDirectoryName(dbId), "aof.log")); + if (subLogIdx == -1) + { + return GetInitializedDeviceFactory(AppendOnlyFileBaseDirectory) + .Get(new FileDescriptor(GetAppendOnlyFileDirectoryName(dbId), "aof.log")); + } + else + { + return GetInitializedDeviceFactory(AppendOnlyFileBaseDirectory) + .Get(new FileDescriptor(GetAppendOnlyFileDirectoryName(dbId), $"aof.{subLogIdx}.log")); + } } + + /// + /// Indicates whether AOF auto-commit is enabled. + /// + public bool AofAutoCommit + => CommitFrequencyMs == 0; + + /// + /// Check if multi-log is enabled + /// + /// + public bool MultiLogEnabled + => AofPhysicalSublogCount > 1 || AofReplayTaskCount > 1; + + /// + /// Number of virtual sublogs expected + /// + public int AofVirtualSublogCount + => AofPhysicalSublogCount * AofReplayTaskCount; } } \ No newline at end of file diff --git a/libs/server/Servers/GarnetServerTcp.cs b/libs/server/Servers/GarnetServerTcp.cs index 6c9d2868f77..07d53f71811 100644 --- a/libs/server/Servers/GarnetServerTcp.cs +++ b/libs/server/Servers/GarnetServerTcp.cs @@ -83,7 +83,7 @@ public GarnetServerTcp( this.networkSendThrottleMax = networkSendThrottleMax; var serverBufferSize = BufferSizeUtils.ServerBufferSize(new MaxSizeSettings()); this.networkBufferSettings = new NetworkBufferSettings(serverBufferSize, serverBufferSize); - this.networkPool = networkBufferSettings.CreateBufferPool(logger: logger); + this.networkPool = networkBufferSettings.CreateBufferPool(ownerType: PoolOwnerType.ServerNetwork, logger: logger); this.unixSocketPath = unixSocketPath; this.unixSocketPermission = unixSocketPermission; diff --git a/libs/server/Servers/IServerSerializer.cs b/libs/server/Servers/IServerSerializer.cs deleted file mode 100644 index 61e341f2efb..00000000000 --- a/libs/server/Servers/IServerSerializer.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -namespace Garnet.server -{ - /// - /// Serializer interface for server-side processing - /// - /// Key - /// Value - /// Input - /// Output - public unsafe interface IServerSerializer - { - /// - /// Write element to given destination, with length bytes of space available - /// - /// Element to write - /// Destination memory - /// Space (bytes) available at destination - /// True if write succeeded, false if not (insufficient space) - bool Write(ref TKey k, ref byte* dst, int length); - - /// - /// Write element to given destination, with length bytes of space available - /// - /// Element to write - /// Destination memory - /// Space (bytes) available at destination - /// True if write succeeded, false if not (insufficient space) - bool Write(ref TValue v, ref byte* dst, int length); - - /// - /// Write element to given destination, with length bytes of space available - /// - /// Element to write - /// Destination memory - /// Space (bytes) available at destination - /// True if write succeeded, false if not (insufficient space) - bool Write(ref TOutput o, ref byte* dst, int length); - - /// - /// Get length of given output - /// - /// - /// - int GetLength(ref TOutput o); - - /// - /// Read key by reference, from given location - /// - /// Memory location - /// Key - ref TKey ReadKeyByRef(ref byte* src); - - /// - /// Read value by reference, from given location - /// - /// Memory location - /// Value - ref TValue ReadValueByRef(ref byte* src); - - /// - /// Read input by reference, from given location - /// - /// Memory location - /// Input - ref TInput ReadInputByRef(ref byte* src); - - /// - /// Read memory as output (by reference), at given location - /// - /// Memory location - /// Length of buffer at memory - /// Output - ref TOutput AsRefOutput(byte* src, int length); - - /// - /// Skip output (increment address) - /// - /// Memory location - void SkipOutput(ref byte* src); - } -} \ No newline at end of file diff --git a/libs/server/Servers/ServerOptions.cs b/libs/server/Servers/ServerOptions.cs index bec46f021be..de519f8520d 100644 --- a/libs/server/Servers/ServerOptions.cs +++ b/libs/server/Servers/ServerOptions.cs @@ -36,29 +36,40 @@ public class ServerOptions public ClusterPreferredEndpointType ClusterPreferredEndpointType { get; set; } /// - /// Total log memory used in bytes (rounds down to power of 2). + /// Total main-log memory (inline and heap) to use, in bytes. Does not need to be a power of 2 /// - public string MemorySize = "16g"; + public string LogMemorySize = "16g"; /// - /// Size of each page in bytes (rounds down to power of 2). + /// Size of each main-log page in bytes (rounds down to power of 2). /// - public string PageSize = "32m"; + public string PageSize = "16m"; /// - /// Size of each log segment in bytes on disk (rounds down to power of 2). + /// Number of main-log pages (rounds down to power of 2). This allows specifying less pages initially than divided by + /// + /// The default empty value means to calculate based on divided by + public int PageCount = 0; + + /// + /// Size of each main-log segment in bytes on disk (rounds down to power of 2). /// public string SegmentSize = "1g"; + /// + /// Size of each object-log segment in bytes on disk (rounds down to power of 2). + /// + public string ObjectLogSegmentSize = "1g"; + /// /// Size of hash index in bytes (rounds down to power of 2). /// - public string IndexSize = "128m"; + public string IndexMemorySize = "128m"; /// /// Max size of hash index in bytes (rounds down to power of 2). If unspecified, index size doesn't grow (default behavior). /// - public string IndexMaxSize = string.Empty; + public string IndexMaxMemorySize = string.Empty; /// /// Percentage of log memory that is kept mutable. @@ -75,11 +86,6 @@ public class ServerOptions /// public bool CopyReadsToTail = false; - /// - /// When records are read from the object store's in-memory immutable region or storage device, copy them to the tail of the log. - /// - public bool ObjectStoreCopyReadsToTail = false; - /// /// Storage directory for tiered records (hybrid log), if storage tiering (UseStorage) is enabled. Uses current directory if unspecified. /// @@ -129,7 +135,7 @@ public ServerOptions(ILogger logger = null) /// public int MemorySizeBits() { - long size = ParseSize(MemorySize, out _); + long size = ParseSize(LogMemorySize, out _); long adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) logger?.LogInformation("Warning: using lower log memory size than specified (power of 2)"); @@ -137,18 +143,34 @@ public int MemorySizeBits() } /// - /// Get page size + /// Minimum main-log / read-cache page size in bytes. A worst-case inline record at default Garnet settings + /// (MaxInlineKeySize = 128B Tsavorite default, single-byte namespace, all optional fields, max length-byte + /// encoding, max filler) plus the 64-byte page header is ~490 bytes when the value-overflow threshold is + /// at its largest allowed value relative to PageSize (effective value < effective page); 512 bytes is the + /// smallest power-of-2 page size that accommodates this and prevents Tsavorite's "Entry does not fit on page". /// - /// - public int PageSizeBits() + public const long MinPageSizeBytes = 512L; + + /// + /// Validate and convert a page-size configuration value to bits, enforcing . + /// + protected int ValidatedPageSizeBits(string value, string propName) { - long size = ParseSize(PageSize, out _); - long adjustedSize = PreviousPowerOf2(size); + var size = ParseSize(value, out _); + var adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) - logger?.LogInformation("Warning: using lower page size than specified (power of 2)"); + logger?.LogInformation("Warning: using lower {PropName} than specified (power of 2)", propName); + if (adjustedSize < MinPageSizeBytes) + throw new Exception($"{propName} '{value}' (effective {adjustedSize} bytes after rounding to previous power of 2) must be at least {MinPageSizeBytes} bytes to ensure a worst-case record fits within a single page."); return (int)Math.Log(adjustedSize, 2); } + /// + /// Get page size + /// + /// + public int PageSizeBits() => ValidatedPageSizeBits(PageSize, nameof(PageSize)); + /// /// Get pub/sub page size /// @@ -163,15 +185,15 @@ public long PubSubPageSizeBytes() } /// - /// Get segment size + /// Get segment size bits for either the main log or object log /// /// - public int SegmentSizeBits() + public int SegmentSizeBits(bool isObj) { - long size = ParseSize(SegmentSize, out _); + long size = ParseSize(isObj ? ObjectLogSegmentSize : SegmentSize, out _); long adjustedSize = PreviousPowerOf2(size); if (size != adjustedSize) - logger?.LogInformation("Warning: using lower disk segment size than specified (power of 2)"); + logger?.LogInformation("Warning: using lower {SegmentType} than specified (power of 2)", isObj ? "ObjSegmentSize" : "SegmentSize"); return (int)Math.Log(adjustedSize, 2); } @@ -279,7 +301,7 @@ internal static string PrettySize(long value) /// /// /// - internal static long PreviousPowerOf2(long v) + public static long PreviousPowerOf2(long v) { v |= v >> 1; v |= v >> 2; diff --git a/libs/server/SessionParseStateExtensions.cs b/libs/server/SessionParseStateExtensions.cs index 8908f41bd1f..d901b031b07 100644 --- a/libs/server/SessionParseStateExtensions.cs +++ b/libs/server/SessionParseStateExtensions.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Text; using Garnet.common; +using Tsavorite.core; namespace Garnet.server { @@ -39,16 +40,10 @@ public static bool TryGetInfoMetricsType(this SessionParseState parseState, int value = InfoMetricsType.STATS; else if (sbArg.EqualsUpperCaseSpanIgnoringCase("STORE"u8)) value = InfoMetricsType.STORE; - else if (sbArg.EqualsUpperCaseSpanIgnoringCase("OBJECTSTORE"u8)) - value = InfoMetricsType.OBJECTSTORE; else if (sbArg.EqualsUpperCaseSpanIgnoringCase("STOREHASHTABLE"u8)) value = InfoMetricsType.STOREHASHTABLE; - else if (sbArg.EqualsUpperCaseSpanIgnoringCase("OBJECTSTOREHASHTABLE"u8)) - value = InfoMetricsType.OBJECTSTOREHASHTABLE; else if (sbArg.EqualsUpperCaseSpanIgnoringCase("STOREREVIV"u8)) value = InfoMetricsType.STOREREVIV; - else if (sbArg.EqualsUpperCaseSpanIgnoringCase("OBJECTSTOREREVIV"u8)) - value = InfoMetricsType.OBJECTSTOREREVIV; else if (sbArg.EqualsUpperCaseSpanIgnoringCase("PERSISTENCE"u8)) value = InfoMetricsType.PERSISTENCE; else if (sbArg.EqualsUpperCaseSpanIgnoringCase("CLIENTS"u8)) @@ -197,25 +192,22 @@ internal static unsafe bool TryGetBitfieldEncoding(this SessionParseState parseS isSigned = default; var encodingSlice = parseState.GetArgSliceByRef(idx); - if (encodingSlice.length <= 1) + if (encodingSlice.Length <= 1) { return false; } - var ptr = encodingSlice.ptr + 1; - - isSigned = *encodingSlice.ptr == 'i'; - - if (!isSigned && *encodingSlice.ptr != 'u') - { + var ptr = encodingSlice.ToPointer() + 1; + byte b = *encodingSlice.ToPointer(); + isSigned = b == 'i'; + if (!isSigned && b != 'u') return false; - } return - RespReadUtils.TryReadInt64Safe(ref ptr, encodingSlice.ptr + encodingSlice.length, + RespReadUtils.TryReadInt64Safe(ref ptr, encodingSlice.ToPointer() + encodingSlice.Length, out bitCount, out var bytesRead, out _, out _, allowLeadingZeros: false) && - ((int)bytesRead == encodingSlice.length - 1) && (bytesRead > 0L) && + ((int)bytesRead == encodingSlice.Length - 1) && (bytesRead > 0L) && (bitCount > 0) && ((isSigned && bitCount <= 64) || (!isSigned && bitCount < 64)); @@ -240,12 +232,12 @@ internal static unsafe bool TryGetBitfieldOffset(this SessionParseState parseSta return false; } - var ptr = offsetSlice.ptr; - var len = offsetSlice.length; + var ptr = offsetSlice.ToPointer(); + var len = offsetSlice.Length; if (*ptr == '#') { - if (offsetSlice.length == 1) + if (offsetSlice.Length == 1) return false; multiplyOffset = true; @@ -254,7 +246,7 @@ internal static unsafe bool TryGetBitfieldOffset(this SessionParseState parseSta } return - RespReadUtils.TryReadInt64Safe(ref ptr, offsetSlice.ptr + offsetSlice.length, + RespReadUtils.TryReadInt64Safe(ref ptr, offsetSlice.ToPointer() + offsetSlice.Length, out bitFieldOffset, out var bytesRead, out _, out _, allowLeadingZeros: false) && ((int)bytesRead == len) && (bytesRead > 0L) && @@ -294,7 +286,7 @@ public static bool TryGetGeoSearchOptions(this SessionParseState parseState, if (command == RespCommand.GEORADIUSBYMEMBER || command == RespCommand.GEORADIUSBYMEMBER_RO) { // From Member - searchOpts.fromMember = parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + searchOpts.fromMember = parseState.GetArgSliceByRef(currTokenIdx++).ToArray(); searchOpts.origin = GeoOriginType.FromMember; } else @@ -351,7 +343,7 @@ public static bool TryGetGeoSearchOptions(this SessionParseState parseState, break; } - searchOpts.fromMember = parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + searchOpts.fromMember = parseState.GetArgSliceByRef(currTokenIdx++).ToArray(); searchOpts.origin = GeoOriginType.FromMember; continue; } @@ -854,9 +846,9 @@ internal static bool TryGetTimeout(this SessionParseState parseState, int idx, o /// The SessionParseState instance. /// The command's simplified info /// The extracted keys - internal static ArgSlice[] ExtractCommandKeys(this ref SessionParseState state, SimpleRespCommandInfo commandInfo) + internal static PinnedSpanByte[] ExtractCommandKeys(this ref SessionParseState state, SimpleRespCommandInfo commandInfo) { - var keysIndexes = new List<(ArgSlice Key, int Index)>(); + var keysIndexes = new List<(PinnedSpanByte Key, int Index)>(); foreach (var spec in commandInfo.KeySpecs) TryAppendKeysFromSpec(ref state, spec, commandInfo.IsSubCommand, keysIndexes); @@ -870,14 +862,14 @@ internal static ArgSlice[] ExtractCommandKeys(this ref SessionParseState state, /// The SessionParseState instance. /// The command's simplified info /// The extracted keys and flags - internal static (ArgSlice, KeySpecificationFlags)[] ExtractCommandKeysAndFlags(this ref SessionParseState state, SimpleRespCommandInfo commandInfo) + internal static (PinnedSpanByte, KeySpecificationFlags)[] ExtractCommandKeysAndFlags(this ref SessionParseState state, SimpleRespCommandInfo commandInfo) { - var keysFlagsIndexes = new List<(ArgSlice Key, KeySpecificationFlags Flags, int Index)>(); + var keysFlagsIndexes = new List<(PinnedSpanByte Key, KeySpecificationFlags Flags, int Index)>(); foreach (var spec in commandInfo.KeySpecs) - TryAppendKeysAndFlagsFromSpec(ref state, spec, commandInfo.IsSubCommand, keysFlagsIndexes); + _ = TryAppendKeysAndFlagsFromSpec(ref state, spec, commandInfo.IsSubCommand, keysFlagsIndexes); - return keysFlagsIndexes.OrderBy(k => k.Index).Select(k => (k.Key, k.Flags)).ToArray(); + return [.. keysFlagsIndexes.OrderBy(k => k.Index).Select(k => (k.Key, k.Flags))]; } /// @@ -887,7 +879,7 @@ internal static (ArgSlice, KeySpecificationFlags)[] ExtractCommandKeysAndFlags(t /// The key specification to use for extraction. /// True if command is a sub-command /// The list to store extracted keys and their matching indexes - private static bool TryAppendKeysFromSpec(ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, List<(ArgSlice Key, int Index)> keysToIndexes) + private static bool TryAppendKeysFromSpec(ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, List<(PinnedSpanByte Key, int Index)> keysToIndexes) { if (!parseState.TryGetKeySearchArgsFromSimpleKeySpec(keySpec, isSubCommand, out var searchArgs)) return false; @@ -911,7 +903,7 @@ private static bool TryAppendKeysFromSpec(ref SessionParseState parseState, Simp /// The key specification to use for extraction. /// True if command is a sub-command /// The list to store extracted keys and flags and their indexes - private static bool TryAppendKeysAndFlagsFromSpec(ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, List<(ArgSlice Key, KeySpecificationFlags Flags, int Index)> keysAndFlags) + private static bool TryAppendKeysAndFlagsFromSpec(ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, List<(PinnedSpanByte Key, KeySpecificationFlags Flags, int Index)> keysAndFlags) { if (!parseState.TryGetKeySearchArgsFromSimpleKeySpec(keySpec, isSubCommand, out var searchArgs)) return false; @@ -936,7 +928,7 @@ private static bool TryAppendKeysAndFlagsFromSpec(ref SessionParseState parseSta /// True if command is a sub-command /// First, last, and step arguments for key searching /// - internal static bool TryGetKeySearchArgsFromSimpleKeySpec(this ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, out (int firstIdx, int lastIdx, int step) searchArgs) + public static bool TryGetKeySearchArgsFromSimpleKeySpec(this ref SessionParseState parseState, SimpleRespKeySpec keySpec, bool isSubCommand, out (int firstIdx, int lastIdx, int step) searchArgs) { searchArgs = (-1, -1, -1); diff --git a/libs/server/Sessions/ServerSessionBase.cs b/libs/server/Sessions/ServerSessionBase.cs index 9c6cc852718..93eeeecdb1d 100644 --- a/libs/server/Sessions/ServerSessionBase.cs +++ b/libs/server/Sessions/ServerSessionBase.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using Garnet.networking; +using Tsavorite.core; namespace Garnet.server { @@ -38,7 +39,7 @@ public ServerSessionBase(INetworkSender networkSender) /// /// /// - public abstract unsafe void Publish(ArgSlice key, ArgSlice value); + public abstract unsafe void Publish(PinnedSpanByte key, PinnedSpanByte value); /// /// Publish an update to a key to all the (pattern) subscribers of the key @@ -46,7 +47,7 @@ public ServerSessionBase(INetworkSender networkSender) /// /// /// - public abstract unsafe void PatternPublish(ArgSlice pattern, ArgSlice key, ArgSlice value); + public abstract unsafe void PatternPublish(PinnedSpanByte pattern, PinnedSpanByte key, PinnedSpanByte value); /// /// Dispose diff --git a/libs/server/SpanByteFunctionsForServer.cs b/libs/server/SpanByteFunctionsForServer.cs deleted file mode 100644 index 13c0e6cba82..00000000000 --- a/libs/server/SpanByteFunctionsForServer.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Buffers; -using Tsavorite.core; - -namespace Garnet.server -{ - /// - /// Callback functions using SpanByteAndMemory output, for SpanByte key, value, input - /// - public class SpanByteFunctionsForServer : SpanByteFunctions - { - /// - /// Constructor - /// - /// - public SpanByteFunctionsForServer(MemoryPool memoryPool = default) - : base(memoryPool) - { - } - - /// - public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) - => CopyWithHeaderTo(ref value, ref dst, memoryPool); - - /// - public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => CopyWithHeaderTo(ref value, ref dst, memoryPool); - - /// - /// Copy to given SpanByteAndMemory (header length and payload copied to actual span/memory) - /// - /// - /// - /// - private static unsafe bool CopyWithHeaderTo(ref SpanByte src, ref SpanByteAndMemory dst, MemoryPool memoryPool) - { - if (dst.IsSpanByte) - { - if (dst.Length >= src.TotalSize) - { - dst.Length = src.TotalSize; - var span = dst.SpanByte.AsSpan(); - fixed (byte* ptr = span) - *(int*)ptr = src.Length; - src.AsReadOnlySpan().CopyTo(span.Slice(sizeof(int))); - return true; - } - dst.ConvertToHeap(); - } - - dst.Length = src.TotalSize; - dst.Memory = memoryPool.Rent(src.TotalSize); - dst.Length = src.TotalSize; - fixed (byte* ptr = dst.Memory.Memory.Span) - *(int*)ptr = src.Length; - src.AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span.Slice(sizeof(int))); - return true; - } - } -} \ No newline at end of file diff --git a/libs/server/SpanByteServerSerializer.cs b/libs/server/SpanByteServerSerializer.cs deleted file mode 100644 index b40d1ece97f..00000000000 --- a/libs/server/SpanByteServerSerializer.cs +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; -using Tsavorite.core; - -namespace Garnet.server -{ - /// - /// Serializer for SpanByte. Used only on server-side. - /// - public sealed unsafe class SpanByteServerSerializer : IServerSerializer - { - readonly int keyLength; - readonly int valueLength; - - [ThreadStatic] - static SpanByteAndMemory output; - - /// - /// Constructor - /// - /// Max key length - /// Max value length - public SpanByteServerSerializer(int maxKeyLength = 512, int maxValueLength = 512) - { - keyLength = maxKeyLength; - valueLength = maxValueLength; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref SpanByte ReadKeyByRef(ref byte* src) - { - ref var ret = ref Unsafe.AsRef(src); - src += ret.TotalSize; - return ref ret; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref SpanByte ReadValueByRef(ref byte* src) - { - ref var ret = ref Unsafe.AsRef(src); - src += ret.TotalSize; - return ref ret; - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref SpanByte ReadInputByRef(ref byte* src) - { - ref var ret = ref Unsafe.AsRef(src); - src += ret.TotalSize; - return ref ret; - } - - /// - public bool Write(ref SpanByte k, ref byte* dst, int length) - { - if (k.Length > length) return false; - - *(int*)dst = k.Length; - dst += sizeof(int); - var dest = new SpanByte(k.Length, (IntPtr)dst); - k.CopyTo(ref dest); - dst += k.Length; - return true; - } - - - /// - public bool Write(ref SpanByteAndMemory k, ref byte* dst, int length) - { - if (k.Length > length) return false; - - var dest = new SpanByte(length, (IntPtr)dst); - if (k.IsSpanByte) - k.SpanByte.CopyTo(ref dest); - else - k.AsMemoryReadOnlySpan().CopyTo(dest.AsSpan()); - return true; - } - - /// - public ref SpanByteAndMemory AsRefOutput(byte* src, int length) - { - output = SpanByteAndMemory.FromPinnedSpan(new Span(src, length)); - return ref output; - } - - /// - public void SkipOutput(ref byte* src) => src += (*(int*)src) + sizeof(int); - - /// - public int GetLength(ref SpanByteAndMemory o) => o.Length; - } -} \ No newline at end of file diff --git a/libs/server/Storage/Functions/EtagState.cs b/libs/server/Storage/Functions/EtagState.cs deleted file mode 100644 index 244ff325ffa..00000000000 --- a/libs/server/Storage/Functions/EtagState.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using Tsavorite.core; - -namespace Garnet.server -{ - internal static class EtagConstants - { - public const byte EtagSize = sizeof(long); - - public const long NoETag = 0; - } - - /// - /// Indirection wrapper to provide a way to set offsets related to Etags and use the getters opaquely from outside. - /// - public struct EtagState - { - public EtagState() - { - } - - /// - /// Offset used accounting space for an etag during allocation - /// - public byte etagOffsetForVarlen { get; set; } = 0; - - /// - /// Gives an offset used to opaquely work with Etag in a payload. By calling this you can skip past the etag if it is present. - /// - public byte etagSkippedStart { get; private set; } = 0; - - /// - /// Resp response methods depend on the value for end being -1 or length of the payload. This field lets you work with providing the end opaquely. - /// - public int etagAccountedLength { get; private set; } = -1; - - /// - /// Field provides access to getting an Etag from a record, hiding whether it is actually present or not. - /// - public long etag { get; set; } = EtagConstants.NoETag; - - /// - /// Sets the values to indicate the presence of an Etag as a part of the payload value - /// - public static void SetValsForRecordWithEtag(ref EtagState curr, ref SpanByte value) - { - curr.etagOffsetForVarlen = EtagConstants.EtagSize; - curr.etagSkippedStart = EtagConstants.EtagSize; - curr.etagAccountedLength = value.LengthWithoutMetadata; - curr.etag = value.GetEtagInPayload(); - } - - public static void ResetState(ref EtagState curr) - { - curr.etagOffsetForVarlen = 0; - curr.etagSkippedStart = 0; - curr.etag = EtagConstants.NoETag; - curr.etagAccountedLength = -1; - } - } -} \ No newline at end of file diff --git a/libs/server/Storage/Functions/FunctionsState.cs b/libs/server/Storage/Functions/FunctionsState.cs index 32eddffbe4e..8f24df11f23 100644 --- a/libs/server/Storage/Functions/FunctionsState.cs +++ b/libs/server/Storage/Functions/FunctionsState.cs @@ -3,6 +3,8 @@ using System; using System.Buffers; +using Garnet.common; +using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server @@ -14,31 +16,42 @@ internal sealed class FunctionsState { private readonly CustomCommandManager customCommandManager; - public readonly TsavoriteLog appendOnlyFile; + public readonly GarnetAppendOnlyFile appendOnlyFile; public readonly WatchVersionMap watchVersionMap; public readonly MemoryPool memoryPool; - public readonly CacheSizeTracker objectStoreSizeTracker; + public readonly CacheSizeTracker cacheSizeTracker; public readonly GarnetObjectSerializer garnetObjectSerializer; - public EtagState etagState; + public IStoreFunctions storeFunctions; + public ObjectIdMap transientObjectIdMap; + public readonly RangeIndexManager rangeIndexManager; + public StoreWrapper storeWrapper; + public readonly ILogger logger; public byte respProtocolVersion; public bool StoredProcMode; public readonly VectorManager vectorManager; internal ReadOnlySpan nilResp => respProtocolVersion >= 3 ? CmdStrings.RESP3_NULL_REPLY : CmdStrings.RESP_ERRNOTFOUND; - public FunctionsState(TsavoriteLog appendOnlyFile, WatchVersionMap watchVersionMap, CustomCommandManager customCommandManager, - MemoryPool memoryPool, CacheSizeTracker objectStoreSizeTracker, GarnetObjectSerializer garnetObjectSerializer, VectorManager vectorManager, + public FunctionsState(GarnetAppendOnlyFile appendOnlyFile, WatchVersionMap watchVersionMap, StoreWrapper storeWrapper, + MemoryPool memoryPool, CacheSizeTracker objectStoreSizeTracker, VectorManager vectorManager, ILogger logger, byte respProtocolVersion = ServerOptions.DEFAULT_RESP_VERSION) { this.appendOnlyFile = appendOnlyFile; this.watchVersionMap = watchVersionMap; - this.customCommandManager = customCommandManager; + this.customCommandManager = storeWrapper.customCommandManager; this.memoryPool = memoryPool ?? MemoryPool.Shared; - this.objectStoreSizeTracker = objectStoreSizeTracker; - this.garnetObjectSerializer = garnetObjectSerializer; - this.etagState = new EtagState(); - this.vectorManager = vectorManager; + this.cacheSizeTracker = objectStoreSizeTracker; + this.garnetObjectSerializer = storeWrapper.GarnetObjectSerializer; + this.storeFunctions = storeWrapper.storeFunctions; + this.transientObjectIdMap = storeWrapper.store.TransientObjectIdMap; + + // Hang onto this for access to storeWrapper.store.Log + this.storeWrapper = storeWrapper; + + this.rangeIndexManager = storeWrapper.rangeIndexManager; + this.logger = logger; this.respProtocolVersion = respProtocolVersion; + this.vectorManager = vectorManager; } public CustomRawStringFunctions GetCustomCommandFunctions(int id) @@ -49,5 +62,56 @@ public CustomObjectFactory GetCustomObjectFactory(int id) public CustomObjectFunctions GetCustomObjectSubCommandFunctions(int id, int subId) => customCommandManager.TryGetCustomObjectSubCommand(id, subId, out var cmd) ? cmd.functions : null; + + /// + /// Copies the specified RESP response bytes into the destination buffer. + /// If the response fits within the stack-allocated buffer, it is copied directly; otherwise, the buffer is converted to heap allocation and the response is copied there. + /// + /// The response bytes to copy. + /// The destination buffer to receive the copied response. + internal void CopyDefaultResp(ReadOnlySpan resp, ref SpanByteAndMemory dst) + { + if (resp.Length < dst.SpanByte.Length) + { + resp.CopyTo(dst.SpanByte.Span); + dst.SpanByte.Length = resp.Length; + return; + } + + dst.ConvertToHeap(); + dst.Length = resp.Length; + dst.Memory = memoryPool.Rent(resp.Length); + resp.CopyTo(dst.MemorySpan); + } + + /// + /// Copies a RESP-formatted integer response into the destination buffer. + /// If the buffer has sufficient stack-allocated space, the number is written directly; otherwise, the buffer is converted to heap allocation and the response is written there. + /// + /// The integer value to encode in RESP format. + /// The destination buffer to receive the encoded response. + internal unsafe void CopyRespNumber(long number, ref SpanByteAndMemory dst) + { + var curr = dst.SpanByte.ToPointer(); + var end = curr + dst.SpanByte.Length; + if (RespWriteUtils.TryWriteInt64(number, ref curr, end, out int integerLen, out int totalLen)) + { + dst.SpanByte.Length = (int)(curr - dst.SpanByte.ToPointer()); + return; + } + + // Handle resp buffer overflow here + dst.ConvertToHeap(); + dst.Length = totalLen; + dst.Memory = memoryPool.Rent(totalLen); + fixed (byte* ptr = dst.MemorySpan) + { + var cc = ptr; + *cc++ = (byte)':'; + NumUtils.WriteInt64(number, integerLen, ref cc); + *cc++ = (byte)'\r'; + *cc = (byte)'\n'; + } + } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/GarnetRecordTriggers.cs b/libs/server/Storage/Functions/GarnetRecordTriggers.cs new file mode 100644 index 00000000000..e56611b674f --- /dev/null +++ b/libs/server/Storage/Functions/GarnetRecordTriggers.cs @@ -0,0 +1,252 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Record lifecycle triggers for Garnet's unified store. Implements + /// to handle per-record cleanup on delete, eviction, flush, copy-to-tail, log truncation, + /// and disk read for BfTree stubs (RangeIndex). + /// + public readonly struct GarnetRecordTriggers : IRecordTriggers + { + /// + /// Cache size tracker for heap size accounting. + /// Created before the store and initialized after via . + /// May be null if memory tracking is disabled. + /// + internal readonly CacheSizeTracker cacheSizeTracker; + + /// + /// Reference to the RangeIndexManager for BfTree lifecycle management. + /// May be null if RangeIndex is not enabled. + /// + internal readonly RangeIndexManager rangeIndexManager; + + /// + /// Reference to the VectorManager for Vector Set lifecycle management. + /// May be null if Vector Sets are not enabled. + /// + internal readonly VectorManager vectorManager; + + /// + /// Creates a GarnetRecordTriggers with a cache size tracker and optional RangeIndexManager. + /// + public GarnetRecordTriggers(CacheSizeTracker cacheSizeTracker, RangeIndexManager rangeIndexManager, VectorManager vectorManager) + { + this.cacheSizeTracker = cacheSizeTracker; + this.rangeIndexManager = rangeIndexManager; + this.vectorManager = vectorManager; + } + + // Trigger gates: when EnableRangeIndexPreview=false (the default), GarnetServer + // passes null in place of a manager, so these gates return false → Tsavorite skips + // ALL trigger invocations entirely → zero per-operation cost on non-RangeIndex + // workloads. Without this, every flush/evict/disk-read/CTT/truncate would invoke + // our callback only to do an early-return on the RecordType check. + + /// + public bool CallOnFlush => rangeIndexManager != null; + + /// + public bool CallOnEvict => rangeIndexManager != null || vectorManager != null; + + /// + public bool CallOnDiskRead => rangeIndexManager != null || vectorManager != null; + + /// + public bool CallPostCopyToTail => rangeIndexManager != null; + + /// + public bool CallOnTruncate => rangeIndexManager != null; + + /// + public void OnDispose(ref LogRecord logRecord, DisposeReason reason) + { + if (!logRecord.Info.ValueIsObject) + { + // Free BfTree and delete data files on key deletion. + if (reason is DisposeReason.Deleted or DisposeReason.Expired && logRecord.RecordDataHeader.RecordType == RangeIndexManager.RangeIndexRecordType) + { + rangeIndexManager?.DisposeTreeUnderLock(logRecord.Key, logRecord.ValueSpan, deleteFiles: true); + } + + // Request Vector Set cleanup when the index key is deleted. + if (reason is DisposeReason.Deleted or DisposeReason.Expired && logRecord.RecordDataHeader.RecordType == VectorManager.RecordType) + { + vectorManager?.RequestDeletion(logRecord.ValueSpan); + } + } + } + + /// + public readonly void OnFlush(ref LogRecord logRecord, long logicalAddress) + { + if (rangeIndexManager is null + || logRecord.Info.ValueIsObject + || logRecord.RecordDataHeader.RecordType != RangeIndexManager.RangeIndexRecordType) + return; + + rangeIndexManager.SnapshotTreeForFlush(logRecord.Key, logRecord.ValueSpan, logicalAddress); + } + + /// + public readonly void OnEvict(ref LogRecord logRecord, EvictionSource source) + { + if (!logRecord.Info.ValueIsObject) + { + // Free BfTree on page eviction under exclusive lock. + if (logRecord.RecordDataHeader.RecordType == RangeIndexManager.RangeIndexRecordType) + { + rangeIndexManager?.DisposeTreeUnderLock(logRecord.Key, logRecord.ValueSpan, deleteFiles: false); + } + + // Drop DiskANN side of index + if (logRecord.RecordDataHeader.RecordType == VectorManager.RecordType) + { + vectorManager?.DropInMemoryIndex(logRecord.ValueSpan); + } + } + } + + /// + public readonly void OnDiskRead(ref LogRecord logRecord) + { + if (!logRecord.Info.ValueIsObject) + { + // Invalidate stale TreeHandle bytes on records loaded from disk. + // RIPROMOTE PostCopyUpdater handles file pre-staging when this stub is later promoted. + if (logRecord.RecordDataHeader.RecordType == RangeIndexManager.RangeIndexRecordType) + { + RangeIndexManager.InvalidateStub(logRecord.ValueSpan); + } + + // Clear DiskANN index pointer so we'll recreate it on first touch + if (logRecord.RecordDataHeader.RecordType == VectorManager.RecordType) + { + VectorManager.ClearIndexPointer(logRecord.ValueSpan); + } + } + } + + /// + public readonly void OnRecovery(Guid checkpointToken) + { + rangeIndexManager?.SetRecoveredCheckpointToken(checkpointToken); + } + + /// + public readonly void OnRecoverySnapshotRead(ref LogRecord logRecord) + { + if (!logRecord.Info.ValueIsObject) + { + // Above-FUA-at-checkpoint stubs: pre-stage data.bftree from the checkpoint snapshot + // file DURING recovery (snapshot files may be deleted post-recovery). Below-FUA + // stubs are handled lazily by RIPROMOTE PostCopyUpdater on first access. + if (rangeIndexManager is not null && logRecord.RecordDataHeader.RecordType == RangeIndexManager.RangeIndexRecordType) + { + RangeIndexManager.MarkRecoveredFromCheckpoint(logRecord.ValueSpan); + rangeIndexManager.RebuildFromSnapshotIfPending(logRecord.Key); + } + + // If we're recovering we might have a context marked as deleting, but the record itself isn't deleted + if (vectorManager is not null && !logRecord.Info.Tombstone && logRecord.RecordDataHeader.RecordType == VectorManager.RecordType) + { + vectorManager.RecoveredVectorSetIndexKey(ref logRecord); + } + } + } + + /// + public readonly void OnCheckpoint(CheckpointTrigger trigger, Guid checkpointToken) + { + if (rangeIndexManager == null) + return; + + switch (trigger) + { + case CheckpointTrigger.VersionShift: + rangeIndexManager.SetCheckpointBarrier(checkpointToken); + break; + case CheckpointTrigger.FlushBegin: + rangeIndexManager.SnapshotAllTreesForCheckpoint(checkpointToken); + rangeIndexManager.ClearCheckpointBarrier(); + break; + case CheckpointTrigger.CheckpointCompleted: + // No action — Tsavorite's checkpoint manager removes per-token snapshot dirs + // when removeOutdated is true; per-flush snapshots are cleaned by OnTruncate. + break; + } + } + + /// + public readonly void PostCopyToTail(in TSourceLogRecord srcLogRecord, long srcLogicalAddress, + ref LogRecord dstLogRecord, long dstLogicalAddress) + where TSourceLogRecord : ISourceLogRecord +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // Only act on RangeIndex records. + // Check the SOURCE's RecordType, not the destination's: Tsavorite's + // CopyReadsToTail / compaction CTT path allocates dst at tail and copies the + // value bytes, but does NOT carry the RecordDataHeader.RecordType through. After + // CTT, dst.RecordType == 0 by default. We must inspect src to recognize this is + // a RangeIndex record, and explicitly propagate the RecordType to dst below + // before the next Read-with-RangeIndex-cmd can pass the type-mismatch guard in + // ReadMethods.CheckRecordTypeMismatch. + if (rangeIndexManager is null + || dstLogRecord.Info.ValueIsObject + || srcLogRecord.RecordType != RangeIndexManager.RangeIndexRecordType) + return; + + // Propagate RecordType from src onto dst (CTT does not do this for us). + // RecordDataHeader is a wrapper struct over the header pointer, so the setter + // writes through to the underlying memory even on this struct value. + var dstHeader = dstLogRecord.RecordDataHeader; + dstHeader.RecordType = RangeIndexManager.RangeIndexRecordType; + + var srcSpan = srcLogRecord.ValueSpan; + var dstSpan = dstLogRecord.ValueSpan; + ref readonly var srcStub = ref RangeIndexManager.ReadIndex(srcSpan); + var srcHandle = srcStub.TreeHandle; + + if (srcHandle != nint.Zero) + { + // Live transfer: src had an active tree; dst inherited TreeHandle via byte-copy. + // liveIndexes entry already exists for this key. Clear src.TreeHandle so a later + // OnEvict on src does not free the tree the dst now owns. + // NOTE: srcLogRecord.ValueSpan is the source's value span; the source record is + // still in the chain (TryCopyToTail does not unlink/seal the source). Mutating + // it in place is safe because the source is logically superseded by dst. + RangeIndexManager.ClearTreeHandle(srcSpan); + } + else + { + // Disk source (post-eviction or post-OnDiskRead-invalidate): pre-stage data.bftree + // from .flush.bftree, and register a pending entry so the next + // checkpoint captures dst's content. + if (srcLogicalAddress != Tsavorite.core.LogAddress.kInvalidAddress) + rangeIndexManager.PreStageAndRegisterPending(dstLogRecord.Key, srcLogicalAddress); + } + + // Mark src as transferred-out so a later OnEvict/OnFlush on the stale source does not + // remove the liveIndexes entry now owned by dst (live case) or by the pending + // registration (cold case), and does not snapshot a stale view. + RangeIndexManager.SetTransferredFlag(srcSpan); + + // Dst is a freshly copied record at the tail. Clear IsFlushed so subsequent reads + // don't loop through PromoteToTail again. + RangeIndexManager.ClearFlushedFlag(dstSpan); + } + + /// + public readonly void OnTruncate(long newBeginAddress) + { + rangeIndexManager?.OnTruncateImpl(newBeginAddress); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/LogRecordUtils.cs b/libs/server/Storage/Functions/LogRecordUtils.cs new file mode 100644 index 00000000000..b0867d30b6b --- /dev/null +++ b/libs/server/Storage/Functions/LogRecordUtils.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Garnet.server +{ + public static class LogRecordUtils + { + /// + /// Determines whether the specified log record has expired. + /// Returns true if the record has an expiration set and its expiration time is earlier than the current UTC time. + /// + /// The type of the log record, which must implement . + /// The log record to check for expiration. + /// True if the log record has expired; otherwise, false. + internal static bool CheckExpiry(in TSourceLogRecord srcLogRecord) + where TSourceLogRecord : ISourceLogRecord + => srcLogRecord.Info.HasExpiration && srcLogRecord.Expiration < DateTimeOffset.UtcNow.Ticks; + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/CallbackMethods.cs b/libs/server/Storage/Functions/MainStore/CallbackMethods.cs index 4882e2e5cbb..ee0fd2408ec 100644 --- a/libs/server/Storage/Functions/MainStore/CallbackMethods.cs +++ b/libs/server/Storage/Functions/MainStore/CallbackMethods.cs @@ -8,15 +8,15 @@ namespace Garnet.server /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly partial struct MainSessionFunctions : ISessionFunctions { /// - public void ReadCompletionCallback(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, long ctx, Status status, RecordMetadata recordMetadata) + public void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref StringInput input, ref StringOutput output, long ctx, Status status, RecordMetadata recordMetadata) { } /// - public void RMWCompletionCallback(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, long ctx, Status status, RecordMetadata recordMetadata) + public void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref StringInput input, ref StringOutput output, long ctx, Status status, RecordMetadata recordMetadata) { } } diff --git a/libs/server/Storage/Functions/MainStore/DeleteMethods.cs b/libs/server/Storage/Functions/MainStore/DeleteMethods.cs index 2b3d5cb859a..2259209c1f1 100644 --- a/libs/server/Storage/Functions/MainStore/DeleteMethods.cs +++ b/libs/server/Storage/Functions/MainStore/DeleteMethods.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using Tsavorite.core; namespace Garnet.server @@ -9,46 +8,28 @@ namespace Garnet.server /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly partial struct MainSessionFunctions : ISessionFunctions { /// - public bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - if (recordInfo.VectorSet && value.AsReadOnlySpan().ContainsAnyExcept((byte)0)) - { - // Implies this is a vector set, needs special handling - // - // Will call back in after a drop with an all 0 value - deleteInfo.Action = DeleteAction.CancelOperation; - return false; - } - - recordInfo.ClearHasETag(); + logRecord.InfoRef.ClearHasETag(); functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); return true; } /// - public void PostSingleDeleter(ref SpanByte key, ref DeleteInfo deleteInfo) + public void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { if (functionsState.appendOnlyFile != null) deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF } /// - public bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - if (recordInfo.VectorSet && value.AsReadOnlySpan().ContainsAnyExcept((byte)0)) - { - // Implies this is a vector set, needs special handling - // - // Will call back in after a drop with an all 0 value - deleteInfo.Action = DeleteAction.CancelOperation; - return false; - } - - recordInfo.ClearHasETag(); - if (!deleteInfo.RecordInfo.Modified) + logRecord.ClearOptionals(); + if (!logRecord.Info.Modified) functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); if (functionsState.appendOnlyFile != null) deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF @@ -56,13 +37,15 @@ public bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteIn } /// - public void PostDeleteOperation(ref SpanByte key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + public void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((deleteInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogDelete(ref key, deleteInfo.Version, deleteInfo.SessionID, epochAccessor); - } + WriteLogDelete(key.KeyBytes, deleteInfo.Version, deleteInfo.SessionID, epochAccessor); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/MainSessionFunctions.cs b/libs/server/Storage/Functions/MainStore/MainSessionFunctions.cs index f98c8b3166c..442721280ae 100644 --- a/libs/server/Storage/Functions/MainStore/MainSessionFunctions.cs +++ b/libs/server/Storage/Functions/MainStore/MainSessionFunctions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using Tsavorite.core; namespace Garnet.server @@ -8,25 +9,44 @@ namespace Garnet.server /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly partial struct MainSessionFunctions : ISessionFunctions { const byte NeedAofLog = 0x1; readonly FunctionsState functionsState; + readonly ReadSessionState readSessionState; /// /// Constructor /// /// - internal MainSessionFunctions(FunctionsState functionsState) + /// + internal MainSessionFunctions(FunctionsState functionsState, ReadSessionState readSessionState = null) { this.functionsState = functionsState; + this.readSessionState = readSessionState; } /// - public void ConvertOutputToHeap(ref RawStringInput input, ref SpanByteAndMemory output) + public void ConvertOutputToHeap(ref StringInput input, ref StringOutput output) { // TODO: Inspect input to determine whether we're in a context requiring ConvertToHeap. //output.ConvertToHeap(); } + + /// + public void BeforeConsistentReadCallback(long hash) + => readSessionState?.BeforeConsistentReadKeyCallback(hash); + + /// + public void AfterConsistentReadKeyCallback() + => readSessionState?.AfterConsistentReadKeyCallback(); + + /// + public void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) + => readSessionState?.BeforeConsistentReadKeyBatch(parameters); + + /// + public bool AfterConsistentReadKeyBatchCallback(int keyCount) + => readSessionState != null && readSessionState.AfterConsistentReadKeyBatch(keyCount); } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index c0168ebd614..0db74243465 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -1,89 +1,95 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Buffers; using System.Diagnostics; +using System.Runtime.CompilerServices; using Garnet.common; +using Microsoft.Extensions.Logging; using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; namespace Garnet.server { /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { - static void CopyTo(ref SpanByte src, ref SpanByteAndMemory dst, MemoryPool memoryPool) + static void CopyTo(ReadOnlySpan src, ref StringOutput dst, MemoryPool memoryPool) { - int srcLength = src.LengthWithoutMetadata; + int srcLength = src.Length; - if (dst.IsSpanByte) + if (dst.SpanByteAndMemory.IsSpanByte) { - if (dst.Length >= srcLength) + if (dst.SpanByteAndMemory.Length >= srcLength) { - dst.Length = srcLength; - src.AsReadOnlySpan().CopyTo(dst.SpanByte.AsSpan()); + dst.SpanByteAndMemory.Length = srcLength; + src.CopyTo(dst.SpanByteAndMemory.SpanByte.Span); return; } - dst.ConvertToHeap(); + dst.SpanByteAndMemory.ConvertToHeap(); } - dst.Memory = memoryPool.Rent(srcLength); - dst.Length = srcLength; - src.AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span); + dst.SpanByteAndMemory.Memory = memoryPool.Rent(srcLength); + dst.SpanByteAndMemory.Length = srcLength; + src.CopyTo(dst.SpanByteAndMemory.MemorySpan); } - void CopyRespTo(ref SpanByte src, ref SpanByteAndMemory dst, int start = 0, int end = -1) + void CopyRespTo(ReadOnlySpan src, ref StringOutput dst, int start = 0, int end = -1) { - int srcLength = end == -1 ? src.LengthWithoutMetadata : ((start < end) ? (end - start) : 0); + int srcLength = end == -1 ? src.Length : ((start < end) ? (end - start) : 0); if (srcLength == 0) { - CopyDefaultResp(CmdStrings.RESP_EMPTY, ref dst); + functionsState.CopyDefaultResp(CmdStrings.RESP_EMPTY, ref dst.SpanByteAndMemory); return; } var numLength = NumUtils.CountDigits(srcLength); - int totalSize = 1 + numLength + 2 + srcLength + 2; // $5\r\nvalue\r\n + var totalSize = 1 + numLength + 2 + srcLength + 2; // $5\r\nvalue\r\n - if (dst.IsSpanByte) + if (dst.SpanByteAndMemory.IsSpanByte) { - if (dst.Length >= totalSize) + if (dst.SpanByteAndMemory.Length >= totalSize) { - dst.Length = totalSize; + dst.SpanByteAndMemory.Length = totalSize; - byte* tmp = dst.SpanByte.ToPointer(); + var tmp = dst.SpanByteAndMemory.SpanByte.ToPointer(); *tmp++ = (byte)'$'; NumUtils.WriteInt32(srcLength, numLength, ref tmp); *tmp++ = (byte)'\r'; *tmp++ = (byte)'\n'; - src.AsReadOnlySpan().Slice(start, srcLength).CopyTo(new Span(tmp, srcLength)); + src.Slice(start, srcLength).CopyTo(new Span(tmp, srcLength)); tmp += srcLength; *tmp++ = (byte)'\r'; - *tmp++ = (byte)'\n'; + *tmp = (byte)'\n'; return; } - dst.ConvertToHeap(); + dst.SpanByteAndMemory.ConvertToHeap(); } - dst.Memory = functionsState.memoryPool.Rent(totalSize); - dst.Length = totalSize; - fixed (byte* ptr = dst.Memory.Memory.Span) + dst.SpanByteAndMemory.Memory = functionsState.memoryPool.Rent(totalSize); + dst.SpanByteAndMemory.Length = totalSize; + fixed (byte* ptr = dst.SpanByteAndMemory.MemorySpan) { - byte* tmp = ptr; + var tmp = ptr; *tmp++ = (byte)'$'; NumUtils.WriteInt32(srcLength, numLength, ref tmp); *tmp++ = (byte)'\r'; *tmp++ = (byte)'\n'; - src.AsReadOnlySpan().Slice(start, srcLength).CopyTo(new Span(tmp, srcLength)); + src.Slice(start, srcLength).CopyTo(new Span(tmp, srcLength)); tmp += srcLength; *tmp++ = (byte)'\r'; *tmp++ = (byte)'\n'; } } - void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, bool isFromPending) + void CopyRespToWithInput(in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, bool isFromPending) + where TSourceLogRecord : ISourceLogRecord { + var value = srcLogRecord.ValueSpan; + switch (input.header.cmd) { case RespCommand.ASYNC: @@ -91,31 +97,8 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB // to the network buffer in case the operation does go pending (latter is indicated by isFromPending) // This is accomplished by calling ConvertToHeap on the destination SpanByteAndMemory if (isFromPending) - dst.ConvertToHeap(); - CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); - break; - - case RespCommand.MIGRATE: - if (value.Length <= dst.Length) - { - value.CopyTo(ref dst.SpanByte); - dst.Length = value.Length; - return; - } - - dst.ConvertToHeap(); - dst.Length = value.TotalSize; - - if (dst.Memory == default) // Allocate new heap buffer - dst.Memory = functionsState.memoryPool.Rent(dst.Length); - else if (dst.Memory.Memory.Span.Length < value.TotalSize) - // Allocate new heap buffer only if existing one is smaller - // otherwise it is safe to re-use existing buffer - { - dst.Memory.Dispose(); - dst.Memory = functionsState.memoryPool.Rent(dst.Length); - } - value.CopyTo(dst.Memory.Memory.Span); + output.SpanByteAndMemory.ConvertToHeap(); + CopyRespTo(value, ref output); break; case RespCommand.VADD: @@ -126,27 +109,40 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB case RespCommand.VREM: case RespCommand.VDIM: case RespCommand.GET: + case RespCommand.RIGET: + case RespCommand.RISET: + case RespCommand.RIDEL: + case RespCommand.RISCAN: + case RespCommand.RIRANGE: + case RespCommand.RIEXISTS: + case RespCommand.RICONFIG: + case RespCommand.RIMETRICS: // Get value without RESP header; exclude expiration - if (value.LengthWithoutMetadata <= dst.Length) + if (value.Length <= output.SpanByteAndMemory.Length) { - dst.Length = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; - value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.SpanByte.AsSpan()); + output.SpanByteAndMemory.Length = value.Length; + value.CopyTo(output.SpanByteAndMemory.SpanByte.Span); return; } - dst.ConvertToHeap(); - dst.Length = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; - dst.Memory = functionsState.memoryPool.Rent(value.LengthWithoutMetadata); - value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.Memory.Memory.Span); + output.SpanByteAndMemory.ConvertToHeap(); + output.SpanByteAndMemory.Length = value.Length; + output.SpanByteAndMemory.Memory = functionsState.memoryPool.Rent(value.Length); + value.CopyTo(output.SpanByteAndMemory.MemorySpan); break; case RespCommand.GETBIT: var offset = input.arg1; - var oldValSet = BitmapManager.GetBit(offset, value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); - if (oldValSet == 0) - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref dst); + byte oldValSet; + + if (srcLogRecord.IsPinnedValue) + oldValSet = BitmapManager.GetBit(offset, srcLogRecord.PinnedValuePointer, value.Length); else - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref dst); + fixed (byte* valuePtr = value) + oldValSet = BitmapManager.GetBit(offset, valuePtr, value.Length); + + functionsState.CopyDefaultResp( + oldValSet == 0 ? CmdStrings.RESP_RETURN_VAL_0 : CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); break; case RespCommand.BITCOUNT: @@ -169,8 +165,14 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB bcOffsetType = (byte)(input.arg1 & 0x1); } - var count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, value.ToPointer() + functionsState.etagState.etagSkippedStart, value.Length - functionsState.etagState.etagSkippedStart); - CopyRespNumber(count, ref dst); + long count; + if (srcLogRecord.IsPinnedValue) + count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, srcLogRecord.PinnedValuePointer, value.Length); + else + fixed (byte* valuePtr = value) + count = BitmapManager.BitCountDriver(bcStartOffset, bcEndOffset, bcOffsetType, valuePtr, value.Length); + + functionsState.CopyRespNumber(count, ref output.SpanByteAndMemory); break; case RespCommand.BITPOS: @@ -194,89 +196,127 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } } - var pos = BitmapManager.BitPosDriver( - input: value.ToPointer() + functionsState.etagState.etagSkippedStart, - inputLen: value.Length - functionsState.etagState.etagSkippedStart, - startOffset: bpStartOffset, - endOffset: bpEndOffset, - searchFor: bpSetVal, - offsetType: bpOffsetType - ); - *(long*)dst.SpanByte.ToPointer() = pos; - CopyRespNumber(pos, ref dst); + long pos; + if (srcLogRecord.IsPinnedValue) + pos = BitmapManager.BitPosDriver(input: srcLogRecord.PinnedValuePointer, inputLen: value.Length, startOffset: bpStartOffset, + endOffset: bpEndOffset, searchFor: bpSetVal, offsetType: bpOffsetType); + else + fixed (byte* valuePtr = value) + pos = BitmapManager.BitPosDriver(input: valuePtr, inputLen: value.Length, startOffset: bpStartOffset, + endOffset: bpEndOffset, searchFor: bpSetVal, offsetType: bpOffsetType); + + *(long*)output.SpanByteAndMemory.SpanByte.ToPointer() = pos; + functionsState.CopyRespNumber(pos, ref output.SpanByteAndMemory); break; case RespCommand.BITOP: - var bitmap = (IntPtr)value.ToPointer() + functionsState.etagState.etagSkippedStart; - var output = dst.SpanByte.ToPointer(); - - *(long*)output = bitmap.ToInt64(); - *(int*)(output + 8) = value.Length; - + // Expose the value as a SpanByteAndMemory: inline values point directly at log memory + // (stable under the unsafe context); overflow values come back as a no-copy borrowed + // Memory that the caller pins for the duration of BITOP execution. For values + // sourced from a DiskLogRecord (pending completion of a disk read), the inline + // SectorAlignedMemory recordBuffer would be returned to the pool when the + // DiskLogRecord is disposed; the getter handles that by copying inline values into a + // pooled IMemoryOwner before returning, so the SpanByteAndMemory returned here is + // always safe to use beyond this callback. + output.SpanByteAndMemory = srcLogRecord.ValueSpanByteAndMemory; return; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - var (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, - value.ToPointer() + functionsState.etagState.etagSkippedStart, - value.Length - functionsState.etagState.etagSkippedStart); + + long retValue; + bool overflow; + if (srcLogRecord.IsPinnedValue) + (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, srcLogRecord.PinnedValuePointer, value.Length); + else + fixed (byte* valuePtr = value) + (retValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, valuePtr, value.Length); + if (!overflow) - CopyRespNumber(retValue, ref dst); + functionsState.CopyRespNumber(retValue, ref output.SpanByteAndMemory); else - CopyDefaultResp(functionsState.nilResp, ref dst); + functionsState.CopyDefaultResp(functionsState.nilResp, ref output.SpanByteAndMemory); return; case RespCommand.BITFIELD_RO: var bitFieldArgs_RO = GetBitFieldArguments(ref input); - var retValue_RO = BitmapManager.BitFieldExecute_RO(bitFieldArgs_RO, - value.ToPointer() + functionsState.etagState.etagSkippedStart, - value.Length - functionsState.etagState.etagSkippedStart); - CopyRespNumber(retValue_RO, ref dst); + + long retValue_RO; + if (srcLogRecord.IsPinnedValue) + retValue_RO = BitmapManager.BitFieldExecute_RO(bitFieldArgs_RO, srcLogRecord.PinnedValuePointer, value.Length); + else + fixed (byte* valuePtr = value) + retValue_RO = BitmapManager.BitFieldExecute_RO(bitFieldArgs_RO, valuePtr, value.Length); + + functionsState.CopyRespNumber(retValue_RO, ref output.SpanByteAndMemory); return; case RespCommand.PFCOUNT: case RespCommand.PFMERGE: - if (!HyperLogLog.DefaultHLL.IsValidHYLL(value.ToPointer(), value.Length)) + // Caller (HyperLogLogOps) provides a sector-aligned destination buffer sized to + // HyperLogLog.DefaultHLL.DenseBytes. Validate signature AND size before the copy + // so a corrupted/oversized value cannot overflow the destination. + var pfDstPtr = output.SpanByteAndMemory.SpanByte.ToPointer(); + var pfDstCapacity = output.SpanByteAndMemory.SpanByte.Length; + + bool isValid; + if (srcLogRecord.IsPinnedValue) { - *(long*)dst.SpanByte.ToPointer() = -1; - return; + isValid = HyperLogLog.DefaultHLL.IsValidHYLL(srcLogRecord.PinnedValuePointer, value.Length); + } + else + { + fixed (byte* valuePtr = value) + isValid = HyperLogLog.DefaultHLL.IsValidHYLL(valuePtr, value.Length); } - if (value.Length <= dst.Length) + // Surface invalid OR oversized as the -1 sentinel; the caller already checks for it. + if (!isValid || value.Length > pfDstCapacity) { - Buffer.MemoryCopy(value.ToPointer(), dst.SpanByte.ToPointer(), value.Length, value.Length); - dst.SpanByte.Length = value.Length; + *(long*)pfDstPtr = -1; return; } - throw new GarnetException($"Not enough space in {input.header.cmd} buffer"); + // Pass the actual destination capacity to MemoryCopy so the bounds check is meaningful. + if (srcLogRecord.IsPinnedValue) + { + Buffer.MemoryCopy(srcLogRecord.PinnedValuePointer, pfDstPtr, pfDstCapacity, value.Length); + } + else + { + fixed (byte* valuePtr = value) + Buffer.MemoryCopy(valuePtr, pfDstPtr, pfDstCapacity, value.Length); + } + + output.SpanByteAndMemory.SpanByte.Length = value.Length; + return; case RespCommand.TTL: - var ttlValue = ConvertUtils.SecondsFromDiffUtcNowTicks(value.MetadataSize == 8 ? value.ExtraMetadata : -1); - CopyRespNumber(ttlValue, ref dst); + var ttlValue = ConvertUtils.SecondsFromDiffUtcNowTicks(srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1); + functionsState.CopyRespNumber(ttlValue, ref output.SpanByteAndMemory); return; case RespCommand.PTTL: - var pttlValue = ConvertUtils.MillisecondsFromDiffUtcNowTicks(value.MetadataSize == 8 ? value.ExtraMetadata : -1); - CopyRespNumber(pttlValue, ref dst); + var pttlValue = ConvertUtils.MillisecondsFromDiffUtcNowTicks(srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1); + functionsState.CopyRespNumber(pttlValue, ref output.SpanByteAndMemory); return; case RespCommand.GETRANGE: - var len = value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart; + var len = value.Length; var start = input.parseState.GetInt(0); var end = input.parseState.GetInt(1); (start, end) = NormalizeRange(start, end, len); - CopyRespTo(ref value, ref dst, start + functionsState.etagState.etagSkippedStart, end + functionsState.etagState.etagSkippedStart); + CopyRespTo(value, ref output, start, end); return; case RespCommand.EXPIRETIME: - var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(value.MetadataSize == 8 ? value.ExtraMetadata : -1); - CopyRespNumber(expireTime, ref dst); + var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1); + functionsState.CopyRespNumber(expireTime, ref output.SpanByteAndMemory); return; case RespCommand.PEXPIRETIME: - var pexpireTime = ConvertUtils.UnixTimeInMillisecondsFromTicks(value.MetadataSize == 8 ? value.ExtraMetadata : -1); - CopyRespNumber(pexpireTime, ref dst); + var pexpireTime = ConvertUtils.UnixTimeInMillisecondsFromTicks(srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1); + functionsState.CopyRespNumber(pexpireTime, ref output.SpanByteAndMemory); return; default: @@ -284,122 +324,40 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB } } - IPUResult EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, long newExpiry, ref SpanByte value, ref SpanByteAndMemory output) + IPUResult EvaluateExpireInPlace(ref LogRecord logRecord, ExpireOption optionType, long newExpiry, ref StringOutput output) { - ObjectOutputHeader* o = (ObjectOutputHeader*)output.SpanByte.ToPointer(); - if (expiryExists) - { - switch (optionType) - { - case ExpireOption.NX: - o->result1 = 0; - break; - case ExpireOption.XX: - case ExpireOption.None: - value.ExtraMetadata = newExpiry; - o->result1 = 1; - break; - case ExpireOption.GT: - case ExpireOption.XXGT: - var replace = newExpiry > value.ExtraMetadata; - if (replace) value.ExtraMetadata = newExpiry; - if (replace) - o->result1 = 1; - else - o->result1 = 0; - break; - case ExpireOption.LT: - case ExpireOption.XXLT: - replace = newExpiry < value.ExtraMetadata; - if (replace) value.ExtraMetadata = newExpiry; - if (replace) - o->result1 = 1; - else - o->result1 = 0; - break; - default: - throw new GarnetException($"EvaluateExpireInPlace exception expiryExists:{expiryExists}, optionType{optionType}"); - } - return o->result1 == 1 ? IPUResult.Succeeded : IPUResult.NotUpdated; - } - else - { - switch (optionType) - { - case ExpireOption.NX: - case ExpireOption.None: - case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite - return IPUResult.Failed; - case ExpireOption.XX: - case ExpireOption.GT: - case ExpireOption.XXGT: - case ExpireOption.XXLT: - o->result1 = 0; - return IPUResult.NotUpdated; - default: - throw new GarnetException($"EvaluateExpireInPlace exception expiryExists:{expiryExists}, optionType{optionType}"); - } - } + ref var result = ref output.AsRef(); + result = 0; + + if (!EvaluateExpire(ref logRecord, optionType, newExpiry, logRecord.Info.HasExpiration, logErrorOnFail: false, functionsState.logger, out var expirationChanged)) + return IPUResult.Failed; + + if (!expirationChanged) + return IPUResult.NotUpdated; + + result = 1; + return IPUResult.Succeeded; } - void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, long newExpiry, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output) + bool EvaluateExpireCopyUpdate(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ExpireOption optionType, long newExpiry, ReadOnlySpan newValue, ref StringOutput output) { - ObjectOutputHeader* o = (ObjectOutputHeader*)output.SpanByte.ToPointer(); - if (expiryExists) - { - switch (optionType) - { - case ExpireOption.NX: - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - break; - case ExpireOption.XX: - case ExpireOption.None: - newValue.ExtraMetadata = newExpiry; - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - o->result1 = 1; - break; - case ExpireOption.GT: - case ExpireOption.XXGT: - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - bool replace = newExpiry < oldValue.ExtraMetadata; - newValue.ExtraMetadata = replace ? oldValue.ExtraMetadata : newExpiry; - if (replace) - o->result1 = 0; - else - o->result1 = 1; - break; - case ExpireOption.LT: - case ExpireOption.XXLT: - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - replace = newExpiry > oldValue.ExtraMetadata; - newValue.ExtraMetadata = replace ? oldValue.ExtraMetadata : newExpiry; - if (replace) - o->result1 = 0; - else - o->result1 = 1; - break; - } - } - else + var hasExpiration = logRecord.Info.HasExpiration; + + ref var result = ref output.AsRef(); + result = 0; + + // TODO ETag? + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newValue, in sizeInfo)) { - switch (optionType) - { - case ExpireOption.NX: - case ExpireOption.None: - case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite - newValue.ExtraMetadata = newExpiry; - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - o->result1 = 1; - break; - case ExpireOption.XX: - case ExpireOption.GT: - case ExpireOption.XXGT: - case ExpireOption.XXLT: - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - o->result1 = 0; - break; - } + functionsState.logger?.LogError("Failed to set value in {methodName}", nameof(EvaluateExpireCopyUpdate)); + return false; } + + var isSuccessful = EvaluateExpire(ref logRecord, optionType, newExpiry, hasExpiration, logErrorOnFail: true, + functionsState.logger, out var expirationChanged); + + result = expirationChanged ? 1 : 0; + return isSuccessful; } static (int, int) NormalizeRange(int start, int end, int len) @@ -425,125 +383,192 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, long n return (0, 0); } - internal static bool CheckExpiry(ref SpanByte src) => src.ExtraMetadata < DateTimeOffset.UtcNow.Ticks; - - static bool InPlaceUpdateNumber(long val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) + static bool TryInPlaceUpdateNumber(ref LogRecord logRecord, ref StringOutput output, ref RMWInfo rmwInfo, long input) { - var ndigits = NumUtils.CountDigits(val, out var isNegative); - ndigits += isNegative ? 1 : 0; + Debug.Assert(output.SpanByteAndMemory.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - if (ndigits > value.LengthWithoutMetadata - valueOffset) - return false; + // Check if the current value in the logRecord contains a valid number and if so, add the input to it. + try + { + if (logRecord.IsPinnedValue) + { + // Using the pinned pointer directly is faster than pinning 'value'. + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!IsValidNumber(valueLength, (byte*)valueAddress, ref output, out var val)) + return true; - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); - _ = NumUtils.WriteInt64(val, value.AsSpan(valueOffset)); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + checked { val += input; } + var ndigits = NumUtils.CountDigits(val, out var isNegative); - Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; - return true; - } + // Set the logRecord's length to the full length of the new value, including negative sign, and get the updated valueLength + if (!logRecord.TrySetPinnedValueLength(ndigits + (isNegative ? 1 : 0), valueAddress, ref valueLength)) + return false; - static bool InPlaceUpdateNumber(double val, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, int valueOffset) - { - var ndigits = NumUtils.CountCharsInDouble(val, out var _, out var _, out var _); + // Call the pinned form of the number-writer. Don't include space for the negative sign; that's added in the callee. + { + var valuePtr = (byte*)valueAddress; // Use ONLY here; it is updated by WriteInt64, so do not use as the pointer for the subsequent copy + NumUtils.WriteInt64(val, ndigits, ref valuePtr); + } - if (ndigits > value.LengthWithoutMetadata - valueOffset) - return false; + new ReadOnlySpan((byte*)valueAddress, valueLength).CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = valueLength; + } + else + { + // The value is not inline, so LogRecord will probably change it to inline because the update is to a very short (# chars in number) length. + var value = logRecord.ValueSpan; - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(ndigits + value.MetadataSize + valueOffset); - _ = NumUtils.WriteDouble(val, value.AsSpan(valueOffset)); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + // TODO: Create sizeInfo + RecordSizeInfo sizeInfo = default; - Debug.Assert(output.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - value.AsReadOnlySpan(valueOffset).CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = value.LengthWithoutMetadata - valueOffset; - return true; - } + fixed (byte* valuePtr = value) + { + if (!IsValidNumber(value.Length, valuePtr, ref output, out var val)) + return true; + checked { val += input; } + var ndigits = NumUtils.CountDigits(val, out var isNegative); - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, long input, int valueOffset) - { - // Check if value contains a valid number - int valLen = value.LengthWithoutMetadata - valueOffset; - byte* valPtr = value.ToPointer() + valueOffset; - if (!IsValidNumber(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) - return true; + // Set the logRecord's length to the full length of the new value, including negative sign. + if (!logRecord.TrySetContentLengths(ndigits + (isNegative ? 1 : 0), in sizeInfo)) + return false; + value = logRecord.ValueSpan; // Re-get since length (and possibly inline-ness) has changed - try - { - checked { val += input; } + // This call will pin the destination. + _ = NumUtils.WriteInt64(val, value); + + value.CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = value.Length; + } + } } catch { - output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; - return true; + output.OutputFlags |= StringOutputFlags.InvalidTypeError; } - - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); + return true; } - static bool TryInPlaceUpdateNumber(ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo, double input, int valueOffset) + static bool TryInPlaceUpdateNumber(ref LogRecord logRecord, ref StringOutput output, ref RMWInfo rmwInfo, double input) { - // Check if value contains a valid number - int valLen = value.LengthWithoutMetadata - valueOffset; - byte* valPtr = value.ToPointer() + valueOffset; - if (!IsValidDouble(valLen, valPtr, output.SpanByte.AsSpan(), out var val)) - return true; + Debug.Assert(output.SpanByteAndMemory.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - val += input; - - if (!double.IsFinite(val)) + // Check if the current value in the logRecord contains a valid double and if so, add the input to it. + if (logRecord.IsPinnedValue) { - output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; - return true; + // Using the pinned pointer directly is faster than pinning 'value'. + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!IsValidDouble(valueLength, (byte*)valueAddress, ref output, out var val)) + return true; + + val += input; + if (!double.IsFinite(val)) + { + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return true; + } + var ndigits = NumUtils.CountCharsInDouble(val, out var _, out var _, out var _); + + if (!logRecord.TrySetPinnedValueLength(ndigits, valueAddress, ref valueLength)) + return false; + + // Call the pinned form of the number-writer. ndigits includes space for the negative sign if present. + var ptr = (byte*)valueAddress; + NumUtils.WriteDouble(val, ndigits, ref ptr); + + new ReadOnlySpan((byte*)valueAddress, valueLength).CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = valueLength; } + else + { + // The value is not inline, so LogRecord will probably change it to inline because the update is to a very short (# chars in number) length. - return InPlaceUpdateNumber(val, ref value, ref output, ref rmwInfo, ref recordInfo, valueOffset); - } + // TODO: Create sizeInfo + RecordSizeInfo sizeInfo = default; - static void CopyUpdateNumber(long next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) - { - NumUtils.WriteInt64(next, newValue.AsSpan(etagIgnoredOffset)); - newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; + var value = logRecord.ValueSpan; + fixed (byte* valuePtr = value) + { + if (!IsValidDouble(value.Length, valuePtr, ref output, out var val)) + return true; + + val += input; + if (!double.IsFinite(val)) + { + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return true; + } + var ndigits = NumUtils.CountCharsInDouble(val, out var _, out var _, out var _); + + // Set the logRecord's length to the length of the new value + if (!logRecord.TrySetContentLengths(ndigits, in sizeInfo)) + return false; + value = logRecord.ValueSpan; + _ = NumUtils.WriteDouble(val, value); + + value.CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = value.Length; + } + } + return true; } - static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output, int etagIgnoredOffset) + static bool TryCopyUpdateNumber(long next, Span newValue, ref StringOutput output) { - NumUtils.WriteDouble(next, newValue.AsSpan(etagIgnoredOffset)); - newValue.AsReadOnlySpan(etagIgnoredOffset).CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = newValue.LengthWithoutMetadata - etagIgnoredOffset; + if (NumUtils.WriteInt64(next, newValue) == 0) + return false; + newValue.CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = newValue.Length; + return true; } - static void CopyUpdateNumber(double next, ref SpanByte newValue, ref SpanByteAndMemory output) + static bool TryCopyUpdateNumber(double next, Span newValue, ref StringOutput output) { - NumUtils.WriteDouble(next, newValue.AsSpan()); - newValue.AsReadOnlySpan().CopyTo(output.SpanByte.AsSpan()); - output.SpanByte.Length = newValue.LengthWithoutMetadata; + if (NumUtils.WriteDouble(next, newValue) == 0) + return false; + newValue.CopyTo(output.SpanByteAndMemory.SpanByte.Span); + output.SpanByteAndMemory.SpanByte.Length = newValue.Length; + return true; } /// - /// Copy update from old value to new value while also validating whether oldValue is a numerical value. + /// Copy update from old 'long' value to new value while also validating whether oldValue is a numerical value. /// - /// Old value copying from - /// New value copying to + /// The source log record, either in-memory or from disk + /// The destination log record + /// Size info for record fields /// Output value /// Parsed input value - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, long input, int etagIgnoredOffset) + static bool TryCopyUpdateNumber(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringOutput output, long input) + where TSourceLogRecord : ISourceLogRecord { - newValue.ExtraMetadata = oldValue.ExtraMetadata; + if (!dstLogRecord.TryCopyOptionals(in srcLogRecord, in sizeInfo)) + return false; - // Check if value contains a valid number - if (!IsValidNumber(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) + var srcValue = srcLogRecord.ValueSpan; // To reduce redundant length calculations getting to ValueSpan + + long val; + if (srcLogRecord.IsPinnedValue) { - // Move to tail of the log even when oldValue is alphanumeric - // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory - oldValue.CopyTo(ref newValue); - output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; - return; + if (!IsValidNumber(srcValue.Length, srcLogRecord.PinnedValuePointer, ref output, out val)) + { + // Move to tail of the log even when oldValue is alphanumeric + // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcLogRecord.ValueSpan, in sizeInfo); + } + } + else + { + fixed (byte* valuePtr = srcValue) + { + if (!IsValidNumber(srcValue.Length, valuePtr, ref output, out val)) + { + // Move to tail of the log even when oldValue is alphanumeric + // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcLogRecord.ValueSpan, in sizeInfo); + } + } } // Check operation overflow @@ -553,44 +578,64 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re } catch { - output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; - return; + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return false; } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); + return TryCopyUpdateNumber(val, dstLogRecord.ValueSpan, ref output); } /// - /// Copy update from old value to new value while also validating whether oldValue is a numerical value. + /// Copy update from old 'double' value to new value while also validating whether oldValue is a numerical value. /// - /// Old value copying from - /// New value copying to + /// The source log record, either in-memory or from disk + /// The destination log record + /// Size information for record fields /// Output value /// Parsed input value - /// Number of bytes to skip for ignoring etag in value payload - static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, double input, int etagIgnoredOffset) + static bool TryCopyUpdateNumber(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringOutput output, double input) + where TSourceLogRecord : ISourceLogRecord { - newValue.ExtraMetadata = oldValue.ExtraMetadata; + if (!dstLogRecord.TryCopyOptionals(in srcLogRecord, in sizeInfo)) + return false; - // Check if value contains a valid number - if (!IsValidDouble(oldValue.LengthWithoutMetadata - etagIgnoredOffset, oldValue.ToPointer() + etagIgnoredOffset, output.SpanByte.AsSpan(), out var val)) + var srcValue = srcLogRecord.ValueSpan; // To reduce redundant length calculations getting to ValueSpan + + double val; + if (srcLogRecord.IsPinnedValue) { - // Move to tail of the log even when oldValue is alphanumeric - // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory - oldValue.CopyTo(ref newValue); - return; + if (!IsValidDouble(srcValue.Length, srcLogRecord.PinnedValuePointer, ref output, out val)) + { + // Move to tail of the log even when oldValue is alphanumeric + // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcLogRecord.ValueSpan, in sizeInfo); + } + } + else + { + fixed (byte* valuePtr = srcValue) + { + if (!IsValidDouble(srcValue.Length, valuePtr, ref output, out val)) + { + // Move to tail of the log even when oldValue is alphanumeric + // We have already paid the cost of bringing from disk so we are treating as a regular access and bring it into memory + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcLogRecord.ValueSpan, in sizeInfo); + } + } } val += input; if (!double.IsFinite(val)) { - output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; - return; + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return false; } // Move to tail of the log and update - CopyUpdateNumber(val, ref newValue, ref output, etagIgnoredOffset); + return TryCopyUpdateNumber(val, dstLogRecord.ValueSpan, ref output); } /// @@ -601,128 +646,89 @@ static void TryCopyUpdateNumber(ref SpanByte oldValue, ref SpanByte newValue, re /// Output error flag /// Parsed long value /// True if input contained only ASCII decimal characters, otherwise false - static bool IsValidNumber(int length, byte* source, Span output, out long val) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool IsValidNumber(int length, byte* source, ref StringOutput output, out long val) { // Check for valid number - if (!NumUtils.TryReadInt64(length, source, out val)) - { - // Signal value is not a valid number - output[0] = (byte)OperationError.INVALID_TYPE; - return false; - } - return true; + if (NumUtils.TryReadInt64(length, source, out val)) + return true; + + // Signal value is not a valid number + output.OutputFlags |= StringOutputFlags.InvalidTypeError; + return false; } - static bool IsValidDouble(int length, byte* source, Span output, out double val) + static bool IsValidDouble(int length, byte* source, ref StringOutput output, out double val) { // Check for valid number if (!NumUtils.TryParseWithInfinity(new ReadOnlySpan(source, length), out val)) { // Signal value is not a valid number - output[0] = (byte)OperationError.INVALID_TYPE; + output.OutputFlags |= StringOutputFlags.InvalidTypeError; return false; } if (!double.IsFinite(val)) { // Signal value is not a Nan/Infinity - output[0] = (byte)OperationError.NAN_OR_INFINITY; + output.OutputFlags |= StringOutputFlags.NaNOrInfinityError; return false; } return true; } - void CopyDefaultResp(ReadOnlySpan resp, ref SpanByteAndMemory dst) - { - if (resp.Length < dst.SpanByte.Length) - { - resp.CopyTo(dst.SpanByte.AsSpan()); - dst.SpanByte.Length = resp.Length; - return; - } - - dst.ConvertToHeap(); - dst.Length = resp.Length; - dst.Memory = functionsState.memoryPool.Rent(resp.Length); - resp.CopyTo(dst.Memory.Memory.Span); - } - - void CopyRespNumber(long number, ref SpanByteAndMemory dst) - { - byte* curr = dst.SpanByte.ToPointer(); - byte* end = curr + dst.SpanByte.Length; - if (RespWriteUtils.TryWriteInt64(number, ref curr, end, out int integerLen, out int totalLen)) - { - dst.SpanByte.Length = (int)(curr - dst.SpanByte.ToPointer()); - return; - } - - //handle resp buffer overflow here - dst.ConvertToHeap(); - dst.Length = totalLen; - dst.Memory = functionsState.memoryPool.Rent(totalLen); - fixed (byte* ptr = dst.Memory.Memory.Span) - { - byte* cc = ptr; - *cc++ = (byte)':'; - NumUtils.WriteInt64(number, integerLen, ref cc); - *cc++ = (byte)'\r'; - *cc++ = (byte)'\n'; - } - } - /// /// Copy length of value to output (as ASCII bytes) /// - static void CopyValueLengthToOutput(ref SpanByte value, ref SpanByteAndMemory output, int eTagIgnoredOffset) + static bool TryCopyValueLengthToOutput(ReadOnlySpan value, ref StringOutput output) { - int numDigits = NumUtils.CountDigits(value.LengthWithoutMetadata - eTagIgnoredOffset); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte, "This code assumes it is called in a non-pending context or in a pending context where dst.SpanByte's pointer remains valid"); + + var numDigits = NumUtils.CountDigits(value.Length); + if (numDigits > output.SpanByteAndMemory.SpanByte.Length) + { + Debug.Fail("Output length overflow in TryCopyValueLengthToOutput"); + return false; + } - Debug.Assert(output.IsSpanByte, "This code assumes it is called in a non-pending context or in a pending context where dst.SpanByte's pointer remains valid"); - var outputPtr = output.SpanByte.ToPointer(); - NumUtils.WriteInt32(value.LengthWithoutMetadata - eTagIgnoredOffset, numDigits, ref outputPtr); - output.SpanByte.Length = numDigits; + var outputPtr = output.SpanByteAndMemory.SpanByte.ToPointer(); + NumUtils.WriteInt32(value.Length, numDigits, ref outputPtr); + output.SpanByteAndMemory.SpanByte.Length = numDigits; + return true; } - static void CopyRespWithEtagData(ref SpanByte value, ref SpanByteAndMemory dst, bool hasEtagInVal, int etagSkippedStart, MemoryPool memoryPool) + void CopyRespWithEtagData(ReadOnlySpan value, ref StringOutput dst, bool hasETag, long etag, MemoryPool memoryPool) { - int valueLength = value.LengthWithoutMetadata; + int valueLength = value.Length; // always writing an array of size 2 => *2\r\n int desiredLength = 4; - ReadOnlySpan etagTruncatedVal; - // get etag to write, default etag 0 for when no etag - long etag = hasEtagInVal ? value.GetEtagInPayload() : EtagConstants.NoETag; - // remove the length of the ETAG - var etagAccountedValueLength = valueLength - etagSkippedStart; - if (hasEtagInVal) - { - etagAccountedValueLength = valueLength - EtagConstants.EtagSize; - } - // here we know the value span has first bytes set to etag so we hardcode skipping past the bytes for the etag below - etagTruncatedVal = value.AsReadOnlySpan(etagSkippedStart); - // *2\r\n :(etag digits)\r\n $(val Len digits)\r\n (value len)\r\n - desiredLength += 1 + NumUtils.CountDigits(etag) + 2 + 1 + NumUtils.CountDigits(etagAccountedValueLength) + 2 + etagAccountedValueLength + 2; + // use provided etag, default etag 0 for when no etag + long etagToWrite = hasETag ? etag : LogRecord.NoETag; - WriteValAndEtagToDst(desiredLength, ref etagTruncatedVal, etag, ref dst, memoryPool); + // Account for the two RESP array elements written separately below: + // *2\r\n :(etag digits)\r\n $(value length digits)\r\n (value bytes)\r\n + desiredLength += 1 + NumUtils.CountDigits(etagToWrite) + 2 + 1 + NumUtils.CountDigits(valueLength) + 2 + valueLength + 2; + + WriteValAndEtagToDst(desiredLength, value, etagToWrite, ref dst, memoryPool); } - static void WriteValAndEtagToDst(int desiredLength, ref ReadOnlySpan value, long etag, ref SpanByteAndMemory dst, MemoryPool memoryPool, bool writeDirect = false) + static void WriteValAndEtagToDst(int desiredLength, ReadOnlySpan value, long etag, ref StringOutput dst, MemoryPool memoryPool, bool writeDirect = false) { - if (desiredLength <= dst.Length) + if (desiredLength <= dst.SpanByteAndMemory.Length) { - dst.Length = desiredLength; - byte* curr = dst.SpanByte.ToPointer(); - byte* end = curr + dst.SpanByte.Length; + dst.SpanByteAndMemory.Length = desiredLength; + byte* curr = dst.SpanByteAndMemory.SpanByte.ToPointer(); + byte* end = curr + dst.SpanByteAndMemory.SpanByte.Length; RespWriteUtils.WriteEtagValArray(etag, ref value, ref curr, end, writeDirect); return; } - dst.ConvertToHeap(); - dst.Length = desiredLength; - dst.Memory = memoryPool.Rent(desiredLength); - fixed (byte* ptr = dst.Memory.Memory.Span) + dst.SpanByteAndMemory.ConvertToHeap(); + dst.SpanByteAndMemory.Length = desiredLength; + dst.SpanByteAndMemory.Memory = memoryPool.Rent(desiredLength); + fixed (byte* ptr = dst.SpanByteAndMemory.MemorySpan) { byte* curr = ptr; byte* end = ptr + desiredLength; @@ -732,10 +738,10 @@ static void WriteValAndEtagToDst(int desiredLength, ref ReadOnlySpan value /// /// Logging upsert from - /// a. ConcurrentWriter - /// b. PostSingleWriter + /// a. InPlaceWriter + /// b. PostInitialWriter /// - void WriteLogUpsert(ref SpanByte key, ref RawStringInput input, ref SpanByte value, long version, int sessionId, TEpochAccessor epochAccessor) + void WriteLogUpsert(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, long version, int sessionId, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { if (functionsState.StoredProcMode) return; @@ -751,9 +757,15 @@ void WriteLogUpsert(ref SpanByte key, ref RawStringInput input, if (input.SerializedLength > 0) input.header.flags |= RespInputFlags.Deterministic; - functionsState.appendOnlyFile.Enqueue( - new AofHeader { opType = AofEntryType.StoreUpsert, storeVersion = version, sessionID = sessionId }, - ref key, ref value, ref input, epochAccessor, out _); + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.StoreUpsert, + version, + sessionId, + key, + value, + ref input, + epochAccessor, + out _); } /// @@ -762,7 +774,7 @@ void WriteLogUpsert(ref SpanByte key, ref RawStringInput input, /// b. InPlaceUpdater /// c. PostCopyUpdater /// - void WriteLogRMW(ref SpanByte key, ref RawStringInput input, long version, int sessionId, TEpochAccessor epochAccessor) + void WriteLogRMW(ReadOnlySpan key, ref StringInput input, long version, int sessionId, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { if (functionsState.StoredProcMode) return; @@ -774,25 +786,37 @@ void WriteLogRMW(ref SpanByte key, ref RawStringInput input, lon input.header.flags |= RespInputFlags.Deterministic; - functionsState.appendOnlyFile.Enqueue( - new AofHeader { opType = AofEntryType.StoreRMW, storeVersion = version, sessionID = sessionId }, - ref key, ref input, epochAccessor, out _); + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.StoreRMW, + version, + sessionId, + key, + ref input, + epochAccessor, + out _); } /// /// Logging Delete from - /// a. ConcurrentDeleter - /// b. PostSingleDeleter + /// a. InPlaceDeleter + /// b. PostInitialDeleter /// - void WriteLogDelete(ref SpanByte key, long version, int sessionID, TEpochAccessor epochAccessor) + void WriteLogDelete(ReadOnlySpan key, long version, int sessionID, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { if (functionsState.StoredProcMode) return; - SpanByte def = default; - functionsState.appendOnlyFile.Enqueue(new AofHeader { opType = AofEntryType.StoreDelete, storeVersion = version, sessionID = sessionID }, ref key, ref def, epochAccessor, out _); + + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.StoreDelete, + version, + sessionID, + key, + value: default, + epochAccessor, + out _); } - BitFieldCmdArgs GetBitFieldArguments(ref RawStringInput input) + BitFieldCmdArgs GetBitFieldArguments(ref StringInput input) { var currTokenIdx = 0; diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.Etags.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.Etags.cs new file mode 100644 index 00000000000..d7d1ba32ad0 --- /dev/null +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.Etags.cs @@ -0,0 +1,385 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; + +namespace Garnet.server +{ + /// + /// ETag-specific RMW callback methods for main store, kept in a separate file + /// with NoInlining to minimize hot-path method footprint. + /// All helpers are stateless with respect to ETag — they receive existingEtag as a parameter + /// and write new ETag values directly to the record, without using any shared mutable state. + /// + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + { + #region InPlaceUpdater dispatcher + + /// + /// Single dispatcher for all ETag commands in InPlaceUpdaterWorker. + /// Reads the existing ETag from the record, delegates to the appropriate helper, + /// and returns the result directly — no bottom-level ETag update needed. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly IPUResult HandleEtagInPlaceUpdateWorker(RespCommand cmd, ref LogRecord logRecord, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + { + long existingEtag = logRecord.Info.HasETag ? logRecord.ETag : LogRecord.NoETag; + + return cmd switch + { + RespCommand.DELIFGREATER => HandleDelIfGreaterInPlaceUpdate(existingEtag, ref input, ref rmwInfo), + RespCommand.SETIFMATCH or RespCommand.SETIFGREATER => HandleSetIfMatchInPlaceUpdate(cmd, existingEtag, ref logRecord, ref input, ref output), + RespCommand.SETWITHETAG => HandleSetWithEtagInPlaceUpdate(existingEtag, ref logRecord, ref input, ref output), + _ => throw new GarnetException("Unexpected ETag command") + }; + } + + #endregion + + #region InPlaceUpdater helpers + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly IPUResult HandleDelIfGreaterInPlaceUpdate(long existingEtag, ref StringInput input, ref RMWInfo rmwInfo) + { + var etagFromClient = input.parseState.GetLong(0); + rmwInfo.Action = etagFromClient > existingEtag ? RMWAction.ExpireAndStop : RMWAction.CancelOperation; + return IPUResult.Failed; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly IPUResult HandleSetIfMatchInPlaceUpdate(RespCommand cmd, long existingEtag, ref LogRecord logRecord, ref StringInput input, ref StringOutput output) + { + var etagFromClient = input.parseState.GetLong(1); + // in IFMATCH we check for equality, in IFGREATER we are checking for sent etag being strictly greater + var comparisonResult = etagFromClient.CompareTo(existingEtag); + var expectedResult = cmd is RespCommand.SETIFMATCH ? 0 : 1; + + if (comparisonResult != expectedResult) + { + if (input.header.CheckSetGetFlag()) + CopyRespWithEtagData(logRecord.ValueSpan, ref output, logRecord.Info.HasETag, existingEtag, functionsState.memoryPool); + else + { + // write back array of the format [etag, nil] + var nilResponse = functionsState.nilResp; + // *2\r\n: + + \r\n + + WriteValAndEtagToDst( + 4 + 1 + NumUtils.CountDigits(existingEtag) + 2 + nilResponse.Length, + nilResponse, + existingEtag, + ref output, + functionsState.memoryPool, + writeDirect: true + ); + } + return IPUResult.NotUpdated; + } + + // If we're here we know we have a valid ETag for update. Get the value to update. We'll need to return false for CopyUpdate if no space for new value. + var inputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (logRecord.Info.ValueIsInline) + { + // We are going to set ETag and possibly Expiration--but we won't remove either. Precheck adequate length before making any changes. + if (!logRecord.CanGrowPinnedValue(inputValue.Length, newETagLen: LogRecord.ETagSize, + newExpirationLen: input.arg1 != 0 ? LogRecord.ExpirationSize : logRecord.ExpirationLen, out var valueAddress, out var valueLength)) + return IPUResult.Failed; + if (!logRecord.TrySetPinnedValueSpan(inputValue, valueAddress, ref valueLength)) + { + Debug.Fail("Should have succeeded in growing the value as we have ensured there was space there already"); + return IPUResult.Failed; + } + } + else + { + // Create local sizeInfo + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(inputValue, in sizeInfo)) + return IPUResult.Failed; + } + + var newEtag = cmd is RespCommand.SETIFMATCH ? (existingEtag + 1) : etagFromClient; + if (!logRecord.TrySetETag(newEtag)) + { + Debug.Fail("Should have succeeded in setting ETag as we should have ensured there was space there already"); + return IPUResult.Failed; + } + + // Need to check for input.arg1 != 0 because GetRMWModifiedFieldInfo shares its logic with CopyUpdater and thus may set sizeInfo.FieldInfo.Expiration true + // due to srcRecordInfo having expiration set; here, that srcRecordInfo is us, so we should do nothing if input.arg1 == 0. + if (input.arg1 != 0 && !logRecord.TrySetExpiration(input.arg1)) + return IPUResult.Failed; + + // Write Etag and Val back to Client as an array of the format [etag, nil] + var nilResp = functionsState.nilResp; + // *2\r\n: + + \r\n + + var numDigitsInEtag = NumUtils.CountDigits(newEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, nilResp, newEtag, ref output, functionsState.memoryPool, writeDirect: true); + return IPUResult.Succeeded; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly IPUResult HandleSetWithEtagInPlaceUpdate(long existingEtag, ref LogRecord logRecord, ref StringInput input, ref StringOutput output) + { + // Update value and increment ETag + var inputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (logRecord.Info.ValueIsInline) + { + if (!logRecord.CanGrowPinnedValue(inputValue.Length, newETagLen: LogRecord.ETagSize, + newExpirationLen: input.arg1 != 0 ? LogRecord.ExpirationSize : logRecord.ExpirationLen, out var valueAddress, out var valueLength)) + return IPUResult.Failed; + if (!logRecord.TrySetPinnedValueSpan(inputValue, valueAddress, ref valueLength)) + { + Debug.Fail("Should have succeeded in growing the value as we have ensured there was space there already"); + return IPUResult.Failed; + } + } + else + { + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(inputValue, in sizeInfo)) + return IPUResult.Failed; + } + + var newEtag = existingEtag + 1; + if (!logRecord.TrySetETag(newEtag)) + { + Debug.Fail("Should have succeeded in setting ETag"); + return IPUResult.Failed; + } + + // Set or clear expiration to match SET semantics + if (input.arg1 != 0) + { + if (!logRecord.TrySetExpiration(input.arg1)) + return IPUResult.Failed; + } + else if (logRecord.Info.HasExpiration) + _ = logRecord.RemoveExpiration(); + + // Return the new ETag as integer + functionsState.CopyRespNumber(newEtag, ref output.SpanByteAndMemory); + return IPUResult.Succeeded; + } + + #endregion + + #region NeedCopyUpdate dispatcher + + /// + /// Single dispatcher for all ETag commands in NeedCopyUpdate. + /// Reads the existing ETag from the source record and delegates to the appropriate helper. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleEtagNeedCopyUpdate(RespCommand cmd, in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + { + long existingEtag = srcLogRecord.Info.HasETag ? srcLogRecord.ETag : LogRecord.NoETag; + + return cmd switch + { + RespCommand.DELIFGREATER => HandleDelIfGreaterNeedCopyUpdate(existingEtag, ref input, ref rmwInfo), + RespCommand.SETIFMATCH or RespCommand.SETIFGREATER => HandleSetIfMatchNeedCopyUpdate(cmd, existingEtag, in srcLogRecord, ref input, ref output), + RespCommand.SETWITHETAG => true, + _ => throw new GarnetException("Unexpected ETag command") + }; + } + + #endregion + + #region NeedCopyUpdate helpers + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleDelIfGreaterNeedCopyUpdate(long existingEtag, ref StringInput input, ref RMWInfo rmwInfo) + { + long etagFromClient = input.parseState.GetLong(0); + if (etagFromClient > existingEtag) + rmwInfo.Action = RMWAction.ExpireAndStop; + + // We always return false because we would rather not create a new record in hybrid log if we don't need to delete the object. + // Setting no Action and returning false for non-delete case will shortcircuit the InternalRMW code to not run CU, and return SUCCESS. + // If we want to delete the object setting the Action to ExpireAndStop will add the tombstone in hybrid log for us. + return false; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleSetIfMatchNeedCopyUpdate(RespCommand cmd, long existingEtag, in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output) + where TSourceLogRecord : ISourceLogRecord + { + long etagToCheckWith = input.parseState.GetLong(1); + + // in IFMATCH we check for equality, in IFGREATER we are checking for sent etag being strictly greater + int comparisonResult = etagToCheckWith.CompareTo(existingEtag); + int expectedResult = cmd is RespCommand.SETIFMATCH ? 0 : 1; + + if (comparisonResult == expectedResult) + return true; + + if (input.header.CheckSetGetFlag()) + { + // Copy value to output for the GET part of the command. + CopyRespWithEtagData(srcLogRecord.ValueSpan, ref output, srcLogRecord.Info.HasETag, existingEtag, functionsState.memoryPool); + } + else + { + // write back array of the format [etag, nil] + var nilResponse = functionsState.nilResp; + // *2\r\n: + + \r\n + + WriteValAndEtagToDst( + 4 + 1 + NumUtils.CountDigits(existingEtag) + 2 + nilResponse.Length, + nilResponse, + existingEtag, + ref output, + functionsState.memoryPool, + writeDirect: true + ); + } + + return false; + } + + #endregion + + #region InitialUpdater helpers + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleSetIfMatchInitialUpdate(RespCommand cmd, ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output) + { + // Copy input to value + var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newInputValue, in sizeInfo)) + return false; + if (sizeInfo.FieldInfo.HasExpiration) + _ = logRecord.TrySetExpiration(input.arg1); + + // the increment on initial etag is for satisfying the variant that any key with no etag is the same as a zero'd etag + Debug.Assert(sizeInfo.FieldInfo.HasETag, "Expected sizeInfo.FieldInfo.HasETag to be true"); + var newEtag = input.parseState.GetLong(1) + (cmd == RespCommand.SETIFMATCH ? 1 : 0); + _ = logRecord.TrySetETag(newEtag); + + // write back array of the format [etag, nil] + var nilResponse = functionsState.nilResp; + // *2\r\n: + + \r\n + + WriteValAndEtagToDst( + 4 + 1 + NumUtils.CountDigits(newEtag) + 2 + nilResponse.Length, + nilResponse, + newEtag, + ref output, + functionsState.memoryPool, + writeDirect: true + ); + + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleSetWithEtagInitialUpdate(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output) + { + // Copy input to value + var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newInputValue, in sizeInfo)) + return false; + + Debug.Assert(sizeInfo.FieldInfo.HasETag, "Expected sizeInfo.FieldInfo.HasETag to be true"); + _ = logRecord.TrySetETag(LogRecord.NoETag + 1); + + // Set expiration if provided + if (sizeInfo.FieldInfo.HasExpiration && !logRecord.TrySetExpiration(input.arg1)) + { + functionsState.logger?.LogError("Could not set expiration in {methodName}.{caseName}", "InitialUpdater", "SETWITHETAG"); + return false; + } + + // Return the initial ETag + functionsState.CopyRespNumber(LogRecord.NoETag + 1, ref output.SpanByteAndMemory); + + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + return true; + } + + #endregion + + #region CopyUpdater dispatcher + + /// + /// Single dispatcher for all ETag commands in CopyUpdater. + /// Reads the existing ETag from the source record and delegates to the appropriate helper. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleEtagCopyUpdateWorker(RespCommand cmd, in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output) + where TSourceLogRecord : ISourceLogRecord + { + long existingEtag = srcLogRecord.Info.HasETag ? srcLogRecord.ETag : LogRecord.NoETag; + + return cmd switch + { + RespCommand.SETIFMATCH or RespCommand.SETIFGREATER => HandleSetIfMatchCopyUpdate(cmd, existingEtag, in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output), + RespCommand.SETWITHETAG => HandleSetWithEtagCopyUpdate(existingEtag, in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output), + _ => throw new GarnetException("Unexpected ETag command") + }; + } + + #endregion + + #region CopyUpdater helpers + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleSetIfMatchCopyUpdate(RespCommand cmd, long existingEtag, in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output) + where TSourceLogRecord : ISourceLogRecord + { + var inputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(inputValue, in sizeInfo)) + return false; + + // Use the etag sent from client as the base + var etagFromClient = input.parseState.GetLong(1); + var newEtag = etagFromClient + (cmd == RespCommand.SETIFMATCH ? 1 : 0); + if (!dstLogRecord.TrySetETag(newEtag)) + return false; + + if (sizeInfo.FieldInfo.HasExpiration && !dstLogRecord.TrySetExpiration(input.arg1 != 0 ? input.arg1 : srcLogRecord.Expiration)) + return false; + + // Write Etag and Val back to Client as an array of the format [etag, nil] + // *2\r\n: + + \r\n + + var numDigitsInEtag = NumUtils.CountDigits(newEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + functionsState.nilResp.Length, functionsState.nilResp, newEtag, ref output, functionsState.memoryPool, writeDirect: true); + + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private readonly bool HandleSetWithEtagCopyUpdate(long existingEtag, in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output) + where TSourceLogRecord : ISourceLogRecord + { + var inputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(inputValue, in sizeInfo)) + return false; + + // Increment existing ETag or set to 1 for non-ETag keys + var newEtag = existingEtag + 1; + if (!dstLogRecord.TrySetETag(newEtag)) + return false; + + // Set or clear expiration to match SET semantics + if (sizeInfo.FieldInfo.HasExpiration && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + + // Return the new ETag as integer + functionsState.CopyRespNumber(newEtag, ref output.SpanByteAndMemory); + + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + #endregion + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index b909f37f94a..38e9619640c 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -1,68 +1,61 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Garnet.common; +using Microsoft.Extensions.Logging; using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; namespace Garnet.server { /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { - enum IPUResult : byte - { - Failed = 0, - Succeeded, - NotUpdated, - } - /// - public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public readonly bool NeedInitialUpdate(TKey key, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { switch (input.header.cmd) { case RespCommand.SETKEEPTTLXX: - case RespCommand.PERSIST: - case RespCommand.EXPIRE: case RespCommand.GETDEL: - case RespCommand.DELIFEXPIM: case RespCommand.GETEX: case RespCommand.DELIFGREATER: return false; case RespCommand.SETEXXX: - // when called withetag all output needs to be placed on the buffer - if (input.header.CheckWithEtagFlag()) - { - // XX when unsuccesful will write back NIL - CopyDefaultResp(functionsState.nilResp, ref output); - } return false; case RespCommand.SETIFGREATER: case RespCommand.SETIFMATCH: - // add etag on first insertion - this.functionsState.etagState.etagOffsetForVarlen = EtagConstants.EtagSize; - return true; + case RespCommand.SETWITHETAG: + // add etag on first insertion, already tracked by header.CheckWithEtagFlag() case RespCommand.SET: case RespCommand.SETEXNX: case RespCommand.SETKEEPTTL: - if (input.header.CheckWithEtagFlag()) - { - this.functionsState.etagState.etagOffsetForVarlen = EtagConstants.EtagSize; - } return true; + case RespCommand.RICREATE: + return true; + case RespCommand.RIPROMOTE: + return false; // Key must already exist; don't create new + case RespCommand.RIRESTORE: + return false; // Key must already exist default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { + // For custom functions, deliberately hiding the complexity of key types var ret = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd) - .NeedInitialUpdate(key.AsReadOnlySpan(), ref input, ref writer); + .NeedInitialUpdate(key.KeyBytes, ref input, ref writer); return ret; } finally @@ -76,104 +69,81 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp } /// - public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public readonly bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) { - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); + Debug.Assert(!logRecord.Info.HasETag && !logRecord.Info.HasExpiration, "Should not have Expiration or ETag on InitialUpdater log records"); - RespCommand cmd = input.header.cmd; + // Because this is InitialUpdater, the destination length should be set correctly, but test and log failures to be safe. + var cmd = input.header.cmd; switch (cmd) { case RespCommand.PFADD: - var v = value.ToPointer(); - value.ShrinkSerializedLength(HyperLogLog.DefaultHLL.SparseInitialLength(ref input)); - HyperLogLog.DefaultHLL.Init(ref input, v, value.Length); - *output.SpanByte.ToPointer() = 1; + RecordSizeInfo.AssertValueDataLength(HyperLogLog.DefaultHLL.SparseInitialLength(ref input), in sizeInfo); + if (!logRecord.TrySetContentLengths(in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "PFADD"); + return false; + } + + var value = logRecord.ValueSpan; + if (logRecord.IsPinnedValue) + HyperLogLog.DefaultHLL.Init(ref input, logRecord.PinnedValuePointer, value.Length); + else + fixed (byte* valuePtr = value) + HyperLogLog.DefaultHLL.Init(ref input, valuePtr, value.Length); + + *output.SpanByteAndMemory.SpanByte.ToPointer() = 1; break; case RespCommand.PFMERGE: //srcHLL offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy + 4 (skip len size) - var sbSrcHLL = input.parseState.GetArgSliceByRef(0).SpanByte; - var length = sbSrcHLL.Length; - var srcHLL = sbSrcHLL.ToPointer(); - var dstHLL = value.ToPointer(); - value.ShrinkSerializedLength(length); - Buffer.MemoryCopy(srcHLL, dstHLL, value.Length, value.Length); + var sbSrcHLL = input.parseState.GetArgSliceByRef(0); + + if (!logRecord.TrySetContentLengths(sbSrcHLL.Length, in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "PFMERGE"); + return false; + } + + value = logRecord.ValueSpan; + + if (logRecord.IsPinnedValue) + Buffer.MemoryCopy(sbSrcHLL.ToPointer(), logRecord.PinnedValuePointer, value.Length, value.Length); + else + fixed (byte* valuePtr = value) + Buffer.MemoryCopy(sbSrcHLL.ToPointer(), valuePtr, value.Length, value.Length); + break; case RespCommand.SETIFGREATER: case RespCommand.SETIFMATCH: - int spaceForEtag = this.functionsState.etagState.etagOffsetForVarlen; - var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - value.ShrinkSerializedLength(newInputValue.Length + metadataSize + spaceForEtag); - value.ExtraMetadata = input.arg1; - newInputValue.CopyTo(value.AsSpan(spaceForEtag)); - long clientSentEtag = input.parseState.GetLong(1); - if (cmd == RespCommand.SETIFMATCH) - clientSentEtag++; - - recordInfo.SetHasETag(); - // the increment on initial etag is for satisfying the variant that any key with no etag is the same as a zero'd etag - value.SetEtagInPayload(clientSentEtag); - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); - - // write back array of the format [etag, nil] - var nilResponse = functionsState.nilResp; - // *2\r\n: + + \r\n + - WriteValAndEtagToDst( - 4 + 1 + NumUtils.CountDigits(functionsState.etagState.etag) + 2 + nilResponse.Length, - ref nilResponse, - functionsState.etagState.etag, - ref output, - functionsState.memoryPool, - writeDirect: true - ); - - break; + return HandleSetIfMatchInitialUpdate(cmd, ref logRecord, in sizeInfo, ref input, ref output); + case RespCommand.SETWITHETAG: + return HandleSetWithEtagInitialUpdate(ref logRecord, in sizeInfo, ref input, ref output); case RespCommand.SET: case RespCommand.SETEXNX: - spaceForEtag = this.functionsState.etagState.etagOffsetForVarlen; - // Copy input to value - newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - value.ShrinkSerializedLength(newInputValue.Length + metadataSize + spaceForEtag); - value.ExtraMetadata = input.arg1; - newInputValue.CopyTo(value.AsSpan(spaceForEtag)); - - if (spaceForEtag != 0) + var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newInputValue, in sizeInfo)) { - recordInfo.SetHasETag(); - // the increment on initial etag is for satisfying the variant that any key with no etag is the same as a zero'd etag - value.SetEtagInPayload(EtagConstants.NoETag + 1); - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); - // Copy initial etag to output only for SET + WITHETAG and not SET NX or XX - CopyRespNumber(EtagConstants.NoETag + 1, ref output); + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "SETEXNX"); + return false; } - break; - case RespCommand.SETKEEPTTL: - spaceForEtag = this.functionsState.etagState.etagOffsetForVarlen; - // Copy input to value - var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - value.ShrinkSerializedLength(value.MetadataSize + setValue.Length + spaceForEtag); - setValue.CopyTo(value.AsSpan(spaceForEtag)); - - if (spaceForEtag != 0) + // Set or remove expiration + if (sizeInfo.FieldInfo.HasExpiration && !logRecord.TrySetExpiration(input.arg1)) { - recordInfo.SetHasETag(); - value.SetEtagInPayload(EtagConstants.NoETag + 1); - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); - // Copy initial etag to output - CopyRespNumber(EtagConstants.NoETag + 1, ref output); + functionsState.logger?.LogError("Could not set expiration in {methodName}.{caseName}", "InitialUpdater", "SETEXNX"); + return false; } break; + case RespCommand.SETKEEPTTL: + // Copy input to value; do not change expiration + _ = logRecord.TrySetValueSpanAndPrepareOptionals(input.parseState.GetArgSliceByRef(0).ReadOnlySpan, in sizeInfo); + break; case RespCommand.SETKEEPTTLXX: case RespCommand.SETEXXX: - case RespCommand.EXPIRE: - case RespCommand.PERSIST: case RespCommand.GETDEL: case RespCommand.GETEX: throw new Exception(); @@ -181,69 +151,143 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.SETBIT: var bOffset = input.arg1; var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); - value.ShrinkSerializedLength(BitmapManager.Length(bOffset)); - BitmapManager.UpdateBitmap(value.ToPointer(), bOffset, bSetVal); + + if (!logRecord.TrySetContentLengths(BitmapManager.Length(bOffset), in sizeInfo, zeroInit: true)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "SETBIT"); + return false; + } + // Always return 0 at initial updater because previous value was 0 - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); + value = logRecord.ValueSpan; + + if (logRecord.IsPinnedValue) + _ = BitmapManager.UpdateBitmap(logRecord.PinnedValuePointer, bOffset, bSetVal); + else + fixed (byte* valuePtr = value) + _ = BitmapManager.UpdateBitmap(valuePtr, bOffset, bSetVal); + + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); break; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - value.ShrinkSerializedLength(BitmapManager.LengthFromType(bitFieldArgs)); + + if (!logRecord.TrySetContentLengths(BitmapManager.LengthFromType(bitFieldArgs), in sizeInfo, zeroInit: true)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "BitField"); + return false; + } + // Ensure new-record space is zero-init'd before we do any bit operations (e.g. it may have been revivified, which for efficiency does not clear old data) - value.AsSpan().Clear(); - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, value.ToPointer(), value.Length); + value = logRecord.ValueSpan; + value.Clear(); + + long bitfieldReturnValue; + bool overflow; + if (logRecord.IsPinnedValue) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, logRecord.PinnedValuePointer, value.Length); + else + fixed (byte* valuePtr = value) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, valuePtr, value.Length); + if (!overflow) - CopyRespNumber(bitfieldReturnValue, ref output); + functionsState.CopyRespNumber(bitfieldReturnValue, ref output.SpanByteAndMemory); else - CopyDefaultResp(functionsState.nilResp, ref output); + functionsState.CopyDefaultResp(functionsState.nilResp, ref output.SpanByteAndMemory); break; case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; + + // If the offset is greater than 0, we need to zero-fill the gap (e.g. new record might have been revivified). + value = logRecord.ValueSpan; if (offset > 0) - { - // If the offset is greater than 0, we need to zero-fill the gap (e.g. new record might have been revivified). - value.AsSpan().Slice(0, offset).Clear(); - } - newValue.CopyTo(value.AsSpan().Slice(offset)); + value.Slice(0, offset).Clear(); + newValue.CopyTo(value.Slice(offset)); - CopyValueLengthToOutput(ref value, ref output, 0); + if (!TryCopyValueLengthToOutput(value, ref output)) + return false; break; case RespCommand.APPEND: var appendValue = input.parseState.GetArgSliceByRef(0); - value.ShrinkSerializedLength(appendValue.Length); - appendValue.ReadOnlySpan.CopyTo(value.AsSpan()); - CopyValueLengthToOutput(ref value, ref output, 0); + // Copy value to be appended to the newly allocated value buffer + value = logRecord.ValueSpan; + appendValue.ReadOnlySpan.CopyTo(value); + + if (!TryCopyValueLengthToOutput(value, ref output)) + return false; break; case RespCommand.INCR: - value.ShrinkSerializedLength(1); // # of digits in "1" - CopyUpdateNumber(1, ref value, ref output); + // This is InitialUpdater so set the value to 1 and the length to the # of digits in "1" + if (!logRecord.TrySetContentLengths(1, in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "INCR"); + return false; + } + + value = logRecord.ValueSpan; + _ = TryCopyUpdateNumber(1L, value, ref output); break; case RespCommand.INCRBY: var incrBy = input.arg1; + var ndigits = NumUtils.CountDigits(incrBy, out var isNegative); - value.ShrinkSerializedLength(ndigits + (isNegative ? 1 : 0)); - CopyUpdateNumber(incrBy, ref value, ref output); + if (!logRecord.TrySetContentLengths(ndigits + (isNegative ? 1 : 0), in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "INCRBY"); + return false; + } + + _ = TryCopyUpdateNumber(incrBy, logRecord.ValueSpan, ref output); break; case RespCommand.DECR: - value.ShrinkSerializedLength(2); // # of digits in "-1" - CopyUpdateNumber(-1, ref value, ref output); + // This is InitialUpdater so set the value to -1 and the length to the # of digits in "-1" + if (!logRecord.TrySetContentLengths(2, in sizeInfo)) + { + Debug.Assert(logRecord.ValueSpan.Length >= 2, "Length overflow in DECR"); + return false; + } + value = logRecord.ValueSpan; + _ = TryCopyUpdateNumber(-1, value, ref output); break; case RespCommand.DECRBY: - isNegative = false; var decrBy = -input.arg1; + ndigits = NumUtils.CountDigits(decrBy, out isNegative); - value.ShrinkSerializedLength(ndigits + (isNegative ? 1 : 0)); - CopyUpdateNumber(decrBy, ref value, ref output); + if (!logRecord.TrySetContentLengths(ndigits + (isNegative ? 1 : 0), in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "DECRBY"); + return false; + } + + _ = TryCopyUpdateNumber(decrBy, logRecord.ValueSpan, ref output); break; case RespCommand.INCRBYFLOAT: var incrByFloat = BitConverter.Int64BitsToDouble(input.arg1); - CopyUpdateNumber(incrByFloat, ref value, ref output); + if (!TryCopyUpdateNumber(incrByFloat, logRecord.ValueSpan, ref output)) + return false; break; + case RespCommand.RICREATE: + { + // The stub bytes (including TreeHandle) are passed as parseState arg 0. + // On AOF replay, HandleRangeIndexCreateReplay intercepts this and replaces + // the stale TreeHandle with a fresh one before the RMW reaches here. + var stubSpan = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!logRecord.TrySetContentLengths(RangeIndexManager.IndexSizeBytes, in sizeInfo)) + { + functionsState.logger?.LogError("Length overflow in {methodName}.{caseName}", "InitialUpdater", "RICREATE"); + return false; + } + stubSpan.CopyTo(logRecord.ValueSpan); + + var dataHeader = logRecord.RecordDataHeader; + dataHeader.RecordType = RangeIndexManager.RangeIndexRecordType; + } + break; case RespCommand.VADD: { if (input.arg1 is VectorManager.VADDAppendLogArg or VectorManager.MigrateElementKeyLogArg or VectorManager.MigrateIndexKeyLogArg) @@ -268,9 +312,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB var context = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(10).Span); var index = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(11).Span); - recordInfo.VectorSet = true; - - functionsState.vectorManager.CreateIndex(dims, reduceDims, quantizer, buildExplorationFactor, numLinks, distanceMetric, context, index, ref value); + functionsState.vectorManager.CreateIndex(dims, reduceDims, quantizer, buildExplorationFactor, numLinks, distanceMetric, context, index, logRecord.ValueSpan); } break; case RespCommand.VREM: @@ -280,23 +322,22 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB if (input.header.cmd > RespCommandExtensions.LastValidCommand) { var functions = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd); - // compute metadata size for result - var expiration = input.arg1; - metadataSize = expiration switch + if (!logRecord.TrySetContentLengths(functions.GetInitialLength(ref input), in sizeInfo, zeroInit: true)) // ZeroInit to be safe { - -1 => 0, - 0 => 0, - _ => 8, - }; - - value.ShrinkSerializedLength(metadataSize + functions.GetInitialLength(ref input)); - if (expiration > 0) - value.ExtraMetadata = expiration; + functionsState.logger?.LogError("Length overflow in 'default' > StartOffset: {methodName}.{caseName}", "InitialUpdater", "default"); + return false; + } + if (input.arg1 > 0 && !logRecord.TrySetExpiration(input.arg1)) + { + functionsState.logger?.LogError("Could not set expiration in 'default' > StartOffset: {methodName}.{caseName}", "InitialUpdater", "default"); + return false; + } - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { - functions.InitialUpdater(key.AsReadOnlySpan(), ref input, value.AsSpan(), ref writer, ref rmwInfo); + functions.InitialUpdater(logRecord.Key, ref input, logRecord.ValueSpan, ref writer, ref rmwInfo); + Debug.Assert(sizeInfo.FieldInfo.ValueSize == logRecord.ValueSpan.Length, $"Inconsistency in initial updater value length: expected {sizeInfo.FieldInfo.ValueSize}, actual {logRecord.ValueSpan.Length}"); } finally { @@ -306,27 +347,25 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB } // Copy input to value - var inputValue = input.parseState.GetArgSliceByRef(0); - value.ShrinkSerializedLength(inputValue.Length); - value.ExtraMetadata = input.arg1; - inputValue.ReadOnlySpan.CopyTo(value.AsSpan()); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(input.parseState.GetArgSliceByRef(0).ReadOnlySpan, in sizeInfo)) + { + functionsState.logger?.LogError("Failed to set value in {methodName}.{caseName}", "InitialUpdater", "default"); + return false; + } // Copy value to output - CopyTo(ref value, ref output, functionsState.memoryPool); + CopyTo(logRecord.ValueSpan, ref output, functionsState.memoryPool); break; } - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + // Success if we made it here + sizeInfo.AssertOptionalsIfSet(logRecord.Info); return true; } /// - public void PostInitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public readonly void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) { - // reset etag state set at need initial update - if (input.header.cmd is (RespCommand.SET or RespCommand.SETEXNX or RespCommand.SETKEEPTTL or RespCommand.SETIFMATCH or RespCommand.SETIFGREATER)) - EtagState.ResetState(ref functionsState.etagState); - functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); if (functionsState.appendOnlyFile != null) { @@ -336,16 +375,38 @@ public void PostInitialUpdater(ref SpanByte key, ref RawStringInput input, ref S } /// - public bool InPlaceUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public readonly bool InPlaceUpdater(ref LogRecord logRecord, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) { - var ipuResult = InPlaceUpdaterWorker(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + if (logRecord.Info.ValueIsObject) + { + rmwInfo.Action = RMWAction.WrongType; + return false; + } + + // RangeIndex type safety – normal string records have RecordType 0; skip all checks in that common case. + if (logRecord.RecordType == RangeIndexManager.RangeIndexRecordType) + { + // Reject non-RI commands on RI keys + if (!input.header.cmd.IsLegalOnRangeIndex()) + { + rmwInfo.Action = RMWAction.WrongType; + return false; + } + } + else if (input.header.cmd.IsRangeIndexCommand()) + { + // Reject RI-specific commands on non-RI keys + rmwInfo.Action = RMWAction.WrongType; + return false; + } + + var ipuResult = InPlaceUpdaterWorker(ref logRecord, ref input, ref output, ref rmwInfo); switch (ipuResult) { case IPUResult.Failed: return false; case IPUResult.Succeeded: - rmwInfo.UsedValueLength = value.TotalSize; - if (!rmwInfo.RecordInfo.Modified) + if (!logRecord.Info.Modified) functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); if (functionsState.appendOnlyFile != null) rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF @@ -356,478 +417,397 @@ public bool InPlaceUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB } } - // NOTE: In the below control flow if you decide to add a new command or modify a command such that it will now do an early return with TRUE, you must make sure you must reset etagState in FunctionState - private IPUResult InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + private readonly IPUResult InPlaceUpdaterWorker(ref LogRecord logRecord, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) { - RespCommand cmd = input.header.cmd; + var cmd = input.header.cmd; // Expired data - if (value.MetadataSize == 8 && input.header.CheckExpiry(value.ExtraMetadata)) + if (logRecord.Info.HasExpiration && input.header.CheckExpiry(logRecord.Expiration)) { - rmwInfo.Action = cmd is RespCommand.DELIFEXPIM ? RMWAction.ExpireAndStop : RMWAction.ExpireAndResume; - recordInfo.ClearHasETag(); + rmwInfo.Action = RMWAction.ExpireAndResume; + _ = logRecord.RemoveETag(); return IPUResult.Failed; } - bool hadRecordPreMutation = recordInfo.ETag; - bool shouldUpdateEtag = hadRecordPreMutation; - if (shouldUpdateEtag) - { - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); - } + var shouldCheckExpiration = true; + + RecordSizeInfo sizeInfo2 = new(); switch (cmd) { + case RespCommand.DELIFGREATER: + case RespCommand.SETIFMATCH: + case RespCommand.SETIFGREATER: + case RespCommand.SETWITHETAG: + return HandleEtagInPlaceUpdateWorker(cmd, ref logRecord, ref input, ref output, ref rmwInfo); case RespCommand.SETEXNX: + // Note: SETEXNX may or may not actually have an expiration. if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output); - } - else if (input.header.CheckWithEtagFlag()) - { - // when called withetag all output needs to be placed on the buffer - // EXX when unsuccesful will write back NIL - CopyDefaultResp(functionsState.nilResp, ref output); + CopyRespTo(logRecord.ValueSpan, ref output); } - // reset etag state after done using - EtagState.ResetState(ref functionsState.etagState); // Nothing is set because being in this block means NX was already violated return IPUResult.NotUpdated; - - case RespCommand.DELIFGREATER: - long etagFromClient = input.parseState.GetLong(0); - rmwInfo.Action = etagFromClient > functionsState.etagState.etag ? RMWAction.ExpireAndStop : RMWAction.CancelOperation; - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.Failed; - - case RespCommand.SETIFGREATER: - case RespCommand.SETIFMATCH: - etagFromClient = input.parseState.GetLong(1); - // in IFMATCH we check for equality, in IFGREATER we are checking for sent etag being strictly greater - int comparisonResult = etagFromClient.CompareTo(functionsState.etagState.etag); - int expectedResult = cmd is RespCommand.SETIFMATCH ? 0 : 1; - - if (comparisonResult != expectedResult) - { - if (input.header.CheckSetGetFlag()) - { - CopyRespWithEtagData(ref value, ref output, shouldUpdateEtag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); - } - else - { - // write back array of the format [etag, nil] - var nilResponse = functionsState.nilResp; - // *2\r\n: + + \r\n + - WriteValAndEtagToDst( - 4 + 1 + NumUtils.CountDigits(functionsState.etagState.etag) + 2 + nilResponse.Length, - ref nilResponse, - functionsState.etagState.etag, - ref output, - functionsState.memoryPool, - writeDirect: true - ); - } - // reset etag state after done using - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.NotUpdated; - } - - // Need Copy update if no space for new value - var inputValue = input.parseState.GetArgSliceByRef(0); - - // retain metadata unless metadata sent - int metadataSize = input.arg1 != 0 ? sizeof(long) : value.MetadataSize; - - if (value.Length < inputValue.length + EtagConstants.EtagSize + metadataSize) - return IPUResult.Failed; - - recordInfo.SetHasETag(); - - long newEtag = cmd is RespCommand.SETIFMATCH ? (functionsState.etagState.etag + 1) : etagFromClient; - - long oldExtraMetadata = value.ExtraMetadata; - - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(metadataSize + inputValue.Length + EtagConstants.EtagSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - - if (input.arg1 != 0) - { - value.ExtraMetadata = input.arg1; - } - else if (oldExtraMetadata != 0) - { - value.ExtraMetadata = oldExtraMetadata; - } - - value.SetEtagInPayload(newEtag); - - inputValue.ReadOnlySpan.CopyTo(value.AsSpan(EtagConstants.EtagSize)); - - // write back array of the format [etag, nil] - var nilResp = functionsState.nilResp; - // *2\r\n: + + \r\n + - var numDigitsInEtag = NumUtils.CountDigits(newEtag); - WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, newEtag, ref output, functionsState.memoryPool, writeDirect: true); - // reset etag state after done using - EtagState.ResetState(ref functionsState.etagState); - // early return since we already updated the ETag - return IPUResult.Succeeded; - case RespCommand.SET: case RespCommand.SETEXXX: - // If the user calls withetag then we need to either update an existing etag and set the value or set the value with an etag and increment it. - bool inputHeaderHasEtag = input.header.CheckWithEtagFlag(); - - int nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; - - // only when both are not false && false or true and true, do we need to readjust - if (inputHeaderHasEtag != shouldUpdateEtag) - { - // in the common path the above condition is skipped - if (inputHeaderHasEtag) - { - // nextUpdate will add etag but currently there is no etag - nextUpdateEtagOffset = EtagConstants.EtagSize; - shouldUpdateEtag = true; - // if something is going to go past this into copy we need to provide offset management for its varlen during allocation - this.functionsState.etagState.etagOffsetForVarlen = EtagConstants.EtagSize; - } - else - { - shouldUpdateEtag = false; - // nextUpdate will remove etag but currently there is an etag - nextUpdateEtagOffset = 0; - this.functionsState.etagState.etagOffsetForVarlen = 0; - } - } - - ArgSlice setValue = input.parseState.GetArgSliceByRef(0); - - // Need CU if no space for new value - metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - if (setValue.Length + metadataSize > value.Length - nextUpdateEtagOffset) - return IPUResult.Failed; - + // Note: SETEXXX may or may not actually have an expiration. // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { - Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(logRecord.ValueSpan, ref output); } - // Adjust value length - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(setValue.Length + metadataSize + nextUpdateEtagOffset); - - // Copy input to value - value.ExtraMetadata = input.arg1; - setValue.ReadOnlySpan.CopyTo(value.AsSpan(nextUpdateEtagOffset)); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - - // If withEtag is called we return the etag back in the response - if (inputHeaderHasEtag) + // If we are not adding an ETag or Expiration we don't need to grow the record size for any reason other than value growth, + // so we can optimize this to just set the length and span. Note: SETEXXX may or may not actually have an expiration. + // TODO: Convert this and similar to use LogRecord.CanGrowPinnedValue. + if (logRecord.Info.ValueIsInline && (input.arg1 == 0 || logRecord.Info.HasExpiration)) { - recordInfo.SetHasETag(); - value.SetEtagInPayload(functionsState.etagState.etag + 1); - // withetag flag means we need to write etag back to the output buffer - CopyRespNumber(functionsState.etagState.etag + 1, ref output); - // reset etag state after done using - EtagState.ResetState(ref functionsState.etagState); - // early return since we already updated etag - return IPUResult.Succeeded; + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!logRecord.TrySetPinnedValueSpan(input.parseState.GetArgSliceByRef(0), valueAddress, ref valueLength)) + return IPUResult.Failed; } else { - recordInfo.ClearHasETag(); + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(in logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(input.parseState.GetArgSliceByRef(0), in sizeInfo2)) + return IPUResult.Failed; } - break; - case RespCommand.SETKEEPTTLXX: - case RespCommand.SETKEEPTTL: - // If the user calls withetag then we need to either update an existing etag and set the value - // or set the value with an initial etag and increment it. - // If withEtag is called we return the etag back to the user - inputHeaderHasEtag = input.header.CheckWithEtagFlag(); - - nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; - - // only when both are not false && false or true and true, do we need to readjust - if (inputHeaderHasEtag != shouldUpdateEtag) + // Update expiration + if (input.arg1 != 0) { - // in the common path the above condition is skipped - if (inputHeaderHasEtag) - { - // nextUpdate will add etag but currently there is no etag - nextUpdateEtagOffset = EtagConstants.EtagSize; - shouldUpdateEtag = true; - // if something is going to go past this into copy we need to provide offset management for its varlen during allocation - this.functionsState.etagState.etagOffsetForVarlen = EtagConstants.EtagSize; - } - else - { - shouldUpdateEtag = false; - // nextUpdate will remove etag but currentyly there is an etag - nextUpdateEtagOffset = 0; - this.functionsState.etagState.etagOffsetForVarlen = 0; - } + if (!logRecord.TrySetExpiration(input.arg1)) + Debug.Fail("Should have succeeded in setting Expiration as we should have ensured there was space there already"); } + else if (logRecord.Info.HasExpiration) + _ = logRecord.RemoveExpiration(); - setValue = input.parseState.GetArgSliceByRef(0); - // Need CU if no space for new value - if (setValue.Length + value.MetadataSize > value.Length - nextUpdateEtagOffset) - return IPUResult.Failed; - - // Check if SetGet flag is set + break; + case RespCommand.SETKEEPTTLXX: + case RespCommand.SETKEEPTTL: + // If the SetGet flag is set, copy the current value to output for the GET part of the command. if (input.header.CheckSetGetFlag()) { - Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(logRecord.ValueSpan, ref output); } - // Adjust value length - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(setValue.Length + value.MetadataSize + functionsState.etagState.etagSkippedStart); - - // Copy input to value - setValue.ReadOnlySpan.CopyTo(value.AsSpan(functionsState.etagState.etagSkippedStart)); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - if (inputHeaderHasEtag) + if (logRecord.Info.ValueIsInline) { - recordInfo.SetHasETag(); - value.SetEtagInPayload(functionsState.etagState.etag + 1); - // withetag flag means we need to write etag back to the output buffer - CopyRespNumber(functionsState.etagState.etag + 1, ref output); - // reset etag state after done using - EtagState.ResetState(ref functionsState.etagState); - // early return since we already updated etag - return IPUResult.Succeeded; + // We won't change ETag or Expiration. Precheck adequate length before making any changes. + if (!logRecord.CanGrowPinnedValue(setValue.Length, newETagLen: logRecord.ETagLen, newExpirationLen: logRecord.ExpirationLen, out var valueAddress, out var valueLength)) + return IPUResult.Failed; + if (!logRecord.TrySetPinnedValueLength(setValue.Length, valueAddress, ref valueLength)) + { + Debug.Fail("Should have succeeded in growing the value as we have ensured there was space there already"); + return IPUResult.Failed; + } } else { - recordInfo.ClearHasETag(); + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(setValue, in sizeInfo2)) + return IPUResult.Failed; } + setValue.CopyTo(logRecord.ValueSpan); break; - case RespCommand.EXPIRE: - var expiryExists = value.MetadataSize == 8; - - var expirationWithOption = new ExpirationWithOption(input.arg1); - - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - - return EvaluateExpireInPlace(expirationWithOption.ExpireOption, expiryExists, expirationWithOption.ExpirationTimeInTicks, ref value, ref output); - - case RespCommand.PERSIST: - if (value.MetadataSize == 8) - { - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.AsSpan().CopyTo(value.AsSpanWithMetadata()); - value.ShrinkSerializedLength(value.Length - value.MetadataSize); - value.UnmarkExtraMetadata(); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - output.SpanByte.AsSpan()[0] = 1; - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.Succeeded; - } - else - { - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.NotUpdated; - } - case RespCommand.INCR: - if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: 1, functionsState.etagState.etagSkippedStart)) + if (!TryInPlaceUpdateNumber(ref logRecord, ref output, ref rmwInfo, input: 1)) return IPUResult.Failed; break; - case RespCommand.DECR: - if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -1, functionsState.etagState.etagSkippedStart)) - { - + if (!TryInPlaceUpdateNumber(ref logRecord, ref output, ref rmwInfo, input: -1)) return IPUResult.Failed; - } break; - case RespCommand.INCRBY: // Check if input contains a valid number var incrBy = input.arg1; - if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: incrBy, functionsState.etagState.etagSkippedStart)) + if (!TryInPlaceUpdateNumber(ref logRecord, ref output, ref rmwInfo, input: incrBy)) return IPUResult.Failed; break; - case RespCommand.DECRBY: var decrBy = input.arg1; - if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, input: -decrBy, functionsState.etagState.etagSkippedStart)) + if (!TryInPlaceUpdateNumber(ref logRecord, ref output, ref rmwInfo, input: -decrBy)) return IPUResult.Failed; break; - case RespCommand.INCRBYFLOAT: var incrByFloat = BitConverter.Int64BitsToDouble(input.arg1); - if (!TryInPlaceUpdateNumber(ref value, ref output, ref rmwInfo, ref recordInfo, incrByFloat, functionsState.etagState.etagSkippedStart)) + if (!TryInPlaceUpdateNumber(ref logRecord, ref output, ref rmwInfo, incrByFloat)) return IPUResult.Failed; break; case RespCommand.SETBIT: - var v = value.ToPointer() + functionsState.etagState.etagSkippedStart; var bOffset = input.arg1; var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); - if (!BitmapManager.IsLargeEnough(functionsState.etagState.etagAccountedLength, bOffset)) return IPUResult.Failed; - - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(value.Length + value.MetadataSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + if (!BitmapManager.IsLargeEnough(logRecord.ValueSpan.Length, bOffset)) + { + var newLength = BitmapManager.Length(bOffset); + if (logRecord.Info.ValueIsInline) + { + // We are removing expiration and not changing ETag presence. Precheck adequate length before making any changes. + if (!logRecord.CanGrowPinnedValue(newLength, newETagLen: logRecord.ETagLen, newExpirationLen: 0, out var valueAddress, out var valueLength)) + return IPUResult.Failed; + // Remove Expiration first to free up the space for value growth. + _ = logRecord.RemoveExpiration(); + if (!logRecord.TrySetPinnedValueLength(newLength, valueAddress, ref valueLength, zeroInit: true)) + { + Debug.Fail("Should have succeeded in growing the value as we have ensured there was space there already"); + return IPUResult.Failed; + } + } + else + { + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetContentLengths(newLength, in sizeInfo2, zeroInit: true)) + return IPUResult.Failed; + _ = logRecord.RemoveExpiration(); + } + } - var oldValSet = BitmapManager.UpdateBitmap(v, bOffset, bSetVal); - if (oldValSet == 0) - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); + byte oldValSet; + if (logRecord.IsPinnedValue) + oldValSet = BitmapManager.UpdateBitmap(logRecord.PinnedValuePointer, bOffset, bSetVal); else - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output); + fixed (byte* valuePtr = logRecord.ValueSpan) + oldValSet = BitmapManager.UpdateBitmap(valuePtr, bOffset, bSetVal); + + functionsState.CopyDefaultResp( + oldValSet == 0 ? CmdStrings.RESP_RETURN_VAL_0 : CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); break; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - v = value.ToPointer() + functionsState.etagState.etagSkippedStart; - - if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, value.Length - functionsState.etagState.etagSkippedStart)) - return IPUResult.Failed; - - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); - value.ShrinkSerializedLength(value.Length + value.MetadataSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + if (!BitmapManager.IsLargeEnoughForType(bitFieldArgs, logRecord.ValueSpan.Length)) + { + var newLength = BitmapManager.LengthFromType(bitFieldArgs); + if (logRecord.Info.ValueIsInline) + { + // We are removing expiration and not changing ETag presence. Precheck adequate length before making any changes. + if (!logRecord.CanGrowPinnedValue(newLength, newETagLen: logRecord.ETagLen, newExpirationLen: 0, out var valueAddress, out var valueLength)) + return IPUResult.Failed; + // Remove Expiration first to free up the space for value growth. + _ = logRecord.RemoveExpiration(); + if (!logRecord.TrySetPinnedValueLength(newLength, valueAddress, ref valueLength, zeroInit: true)) + { + Debug.Fail("Should have succeeded in growing the value as we have ensured there was space there already"); + return IPUResult.Failed; + } + } + else + { + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetContentLengths(newLength, in sizeInfo2, zeroInit: true)) + return IPUResult.Failed; + _ = logRecord.RemoveExpiration(); + } + } - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, v, value.Length - functionsState.etagState.etagSkippedStart); + long bitfieldReturnValue; + bool overflow; + if (logRecord.IsPinnedValue) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, logRecord.PinnedValuePointer, logRecord.ValueSpan.Length); + else + fixed (byte* valuePtr = logRecord.ValueSpan) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, valuePtr, logRecord.ValueSpan.Length); if (overflow) { - CopyDefaultResp(functionsState.nilResp, ref output); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); + functionsState.CopyDefaultResp(functionsState.nilResp, ref output.SpanByteAndMemory); return IPUResult.Succeeded; } - CopyRespNumber(bitfieldReturnValue, ref output); + functionsState.CopyRespNumber(bitfieldReturnValue, ref output.SpanByteAndMemory); break; case RespCommand.PFADD: - v = value.ToPointer(); + bool result = false, parseOk = false; + var updated = false; + var valueLen = logRecord.ValueSpan.Length; + if (logRecord.IsPinnedValue) + { + parseOk = result = HyperLogLog.DefaultHLL.IsValidHYLL(logRecord.PinnedValuePointer, valueLen); + if (result) + { + _ = logRecord.RemoveExpiration(); + result = HyperLogLog.DefaultHLL.Update(ref input, logRecord.PinnedValuePointer, valueLen, ref updated); + } + } + else + { + fixed (byte* valuePtr = logRecord.ValueSpan) + { + parseOk = result = HyperLogLog.DefaultHLL.IsValidHYLL(valuePtr, valueLen); + if (result) + { + _ = logRecord.RemoveExpiration(); + result = HyperLogLog.DefaultHLL.Update(ref input, valuePtr, valueLen, ref updated); + } + } + } - if (!HyperLogLog.DefaultHLL.IsValidHYLL(v, value.Length)) + if (!parseOk) { - *output.SpanByte.ToPointer() = (byte)0xFF; - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); + *output.SpanByteAndMemory.SpanByte.ToPointer() = (byte)0xFF; // Flags invalid HLL return IPUResult.NotUpdated; } - var updated = false; - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(value.Length + value.MetadataSize); - var result = HyperLogLog.DefaultHLL.Update(ref input, v, value.Length, ref updated); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - if (result) - *output.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; - return result ? IPUResult.Succeeded : IPUResult.Failed; + *output.SpanByteAndMemory.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; + if (!result) + return IPUResult.Failed; + break; case RespCommand.PFMERGE: //srcHLL offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy +4 (skip len size) - var srcHLL = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); - var dstHLL = value.ToPointer(); + var srcHLL = input.parseState.GetArgSliceByRef(0).ToPointer(); + + result = parseOk = false; + valueLen = logRecord.ValueSpan.Length; + if (logRecord.IsPinnedValue) + { + var dstHLL = logRecord.PinnedValuePointer; + parseOk = result = HyperLogLog.DefaultHLL.IsValidHYLL(dstHLL, valueLen); + if (result) + { + _ = logRecord.RemoveExpiration(); + result = HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, valueLen); + } + } + else + { + fixed (byte* dstHLL = logRecord.ValueSpan) + { + parseOk = result = HyperLogLog.DefaultHLL.IsValidHYLL(dstHLL, valueLen); + if (result) + { + _ = logRecord.RemoveExpiration(); + result = HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, valueLen); + } + } + } - if (!HyperLogLog.DefaultHLL.IsValidHYLL(dstHLL, value.Length)) + if (!parseOk) { - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - //InvalidType - *(long*)output.SpanByte.ToPointer() = -1; + //InvalidType + *output.SpanByteAndMemory.SpanByte.ToPointer() = (byte)0xFF; // Flags invalid HLL return IPUResult.NotUpdated; } - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(value.Length + value.MetadataSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - return HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, value.Length) ? IPUResult.Succeeded : IPUResult.Failed; + if (!result) + return IPUResult.Failed; + break; case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - if (newValue.Length + offset > value.LengthWithoutMetadata - functionsState.etagState.etagSkippedStart) - return IPUResult.Failed; - - newValue.CopyTo(value.AsSpan(functionsState.etagState.etagSkippedStart).Slice(offset)); + var totalLength = newValue.Length + offset; + if (totalLength > logRecord.ValueSpan.Length) + { + // Try to grow in place. We are not changing the presence of ETag or Expiration + if (logRecord.Info.ValueIsInline) + { + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!logRecord.TrySetPinnedValueLength(totalLength, valueAddress, ref valueLength)) + return IPUResult.Failed; + } + else + { + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(in logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetContentLengths(totalLength, in sizeInfo2)) + return IPUResult.Failed; + } + } - CopyValueLengthToOutput(ref value, ref output, functionsState.etagState.etagSkippedStart); + newValue.CopyTo(logRecord.ValueSpan.Slice(offset)); + if (!TryCopyValueLengthToOutput(logRecord.ValueSpan, ref output)) + return IPUResult.Failed; break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(logRecord.ValueSpan, ref output); rmwInfo.Action = RMWAction.ExpireAndStop; return IPUResult.Failed; case RespCommand.GETEX: - CopyRespTo(ref value, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(logRecord.ValueSpan, ref output); + + var ipuResult = IPUResult.NotUpdated; + // If both EX and PERSIST were specified, EX wins if (input.arg1 > 0) { - byte* pbOutput = stackalloc byte[ObjectOutputHeader.Size]; - var _output = new SpanByteAndMemory(SpanByte.FromPinnedPointer(pbOutput, ObjectOutputHeader.Size)); + var _output = StringOutput.FromPinnedSpan(stackalloc byte[sizeof(int)]); var newExpiry = input.arg1; - return EvaluateExpireInPlace(ExpireOption.None, expiryExists: value.MetadataSize == 8, newExpiry, ref value, ref _output); + ipuResult = EvaluateExpireInPlace(ref logRecord, ExpireOption.None, newExpiry, ref _output); + if (ipuResult == IPUResult.Failed) + return IPUResult.Failed; } - - if (input.parseState.Count > 0) + else if (input.parseState.Count > 0) { - var persist = input.parseState.GetArgSliceByRef(0).ReadOnlySpan - .EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST); - - if (persist) // Persist the key + // PERSIST means to remove the Expiration; if there is no expiration, the following is a no-op. + if (input.parseState.GetArgSliceByRef(0).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST) && logRecord.Info.HasExpiration) { - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.AsSpan().CopyTo(value.AsSpanWithMetadata()); - value.ShrinkSerializedLength(value.Length - value.MetadataSize); - value.UnmarkExtraMetadata(); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.Succeeded; + _ = logRecord.RemoveExpiration(); + ipuResult = IPUResult.Succeeded; } } - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.NotUpdated; + // reset etag state that may have been initialized earlier, but don't update etag + return ipuResult; case RespCommand.APPEND: // If nothing to append, can avoid copy update. - var appendSize = input.parseState.GetArgSliceByRef(0).Length; - - if (appendSize == 0) + var appendValue = input.parseState.GetArgSliceByRef(0); + var appendLength = appendValue.Length; + if (appendLength > 0) { - CopyValueLengthToOutput(ref value, ref output, functionsState.etagState.etagSkippedStart); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.NotUpdated; + // Try to grow in place. + var originalLength = logRecord.ValueSpan.Length; + totalLength = originalLength + appendLength; + + // Try to grow in place. We are not changing the presence of ETag or Expiration + if (logRecord.Info.ValueIsInline) + { + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!logRecord.TrySetPinnedValueLength(totalLength, valueAddress, ref valueLength)) + return IPUResult.Failed; + } + else + { + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(in logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + if (!logRecord.TrySetContentLengths(totalLength, in sizeInfo2)) + return IPUResult.Failed; + } + + // Append the new value with the client input at the end of the old data + appendValue.ReadOnlySpan.CopyTo(logRecord.ValueSpan.Slice(originalLength)); + if (!TryCopyValueLengthToOutput(logRecord.ValueSpan, ref output)) + return IPUResult.Failed; + break; } - return IPUResult.Failed; - case RespCommand.DELIFEXPIM: - // this is the case where it isn't expired - shouldUpdateEtag = false; - break; + return TryCopyValueLengthToOutput(logRecord.ValueSpan, ref output) ? IPUResult.Succeeded : IPUResult.Failed; case RespCommand.VADD: // Adding to an existing VectorSet is modeled as a read operations // @@ -841,13 +821,13 @@ private IPUResult InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput inpu // Handle "make me delete-able" if (input.arg1 == VectorManager.DeleteAfterDropArg) { - value.AsSpan().Clear(); + logRecord.ValueSpan.Clear(); } else if (input.arg1 == VectorManager.RecreateIndexArg) { var newIndexPtr = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(11).Span); - functionsState.vectorManager.RecreateIndex(newIndexPtr, ref value); + functionsState.vectorManager.RecreateIndex(newIndexPtr, logRecord.ValueSpan); } // Ignore everything else @@ -863,54 +843,45 @@ private IPUResult InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput inpu default: if (cmd > RespCommandExtensions.LastValidCommand) { - if (shouldUpdateEtag) - { - CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return IPUResult.Succeeded; - } - var functions = functionsState.GetCustomCommandFunctions((ushort)cmd); var expirationInTicks = input.arg1; if (expirationInTicks == -1) { - // there is existing metadata, but we want to clear it. - // we remove metadata, shift the value, shrink length - if (value.ExtraMetadata > 0) - { - var oldValue = value.AsReadOnlySpan(); - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.UnmarkExtraMetadata(); - oldValue.CopyTo(value.AsSpan()); - value.ShrinkSerializedLength(oldValue.Length); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); - } + // There is existing expiration and we want to clear it. + _ = logRecord.RemoveExpiration(); } else if (expirationInTicks > 0) { - // there is no existing metadata, but we want to add it. we cannot do in place update. - if (value.ExtraMetadata == 0) return IPUResult.Failed; - // set expiration to the specific value - value.ExtraMetadata = expirationInTicks; + // There is no existing metadata, but we want to add it. Try to do in place update. + if (!logRecord.TrySetExpiration(expirationInTicks)) + return IPUResult.Failed; } + shouldCheckExpiration = false; - var valueLength = value.LengthWithoutMetadata; - - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var value = logRecord.ValueSpan; + var newLength = value.Length; + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { - var ret = functions.InPlaceUpdater(key.AsReadOnlySpan(), ref input, value.AsSpan(), ref valueLength, ref writer, ref rmwInfo); - Debug.Assert(valueLength <= value.LengthWithoutMetadata); + var ret = functions.InPlaceUpdater(logRecord.Key, ref input, value, ref newLength, ref writer, ref rmwInfo); - // Adjust value length if user shrinks it - if (valueLength < value.LengthWithoutMetadata) + // Adjust value length if user shrank it. Because we are shrinking this will always succeed. + // This assumes newLength has not been changed if !ret. + if (newLength < logRecord.ValueSpan.Length) { - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - value.ShrinkSerializedLength(valueLength + value.MetadataSize); - rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); + if (logRecord.Info.ValueIsInline) + { + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + _ = logRecord.TrySetPinnedValueLength(newLength, valueAddress, ref valueLength); + } + else + { + // Create local sizeInfo + sizeInfo2 = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(in logRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo2); + _ = logRecord.TrySetContentLengths(newLength, in sizeInfo2); + } } - return ret ? IPUResult.Succeeded : IPUResult.Failed; } finally @@ -919,99 +890,41 @@ private IPUResult InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput inpu } } throw new GarnetException("Unsupported operation on input"); + case RespCommand.RICREATE: + // Index already exists at this key — reject with error + return IPUResult.NotUpdated; + case RespCommand.RIPROMOTE: + // Record is in mutable region — no-op. Not logged to AOF (internal maintenance). + Debug.Assert(!RangeIndexManager.ReadIndex(logRecord.ValueSpan).IsFlushed, + "Mutable record should never have Flushed flag set"); + return IPUResult.NotUpdated; + case RespCommand.RIRESTORE: + // Set the TreeHandle from the restored BfTree pointer. Not logged to AOF (transient pointer). + RangeIndexManager.RecreateIndex((nint)input.arg1, logRecord.ValueSpan); + return IPUResult.NotUpdated; } - // increment the Etag transparently if in place update happened - if (shouldUpdateEtag) - { - value.SetEtagInPayload(this.functionsState.etagState.etag + 1); - } - - if (hadRecordPreMutation) - { - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - } - + sizeInfo2.AssertOptionalsIfSet(logRecord.Info, checkExpiration: shouldCheckExpiration); return IPUResult.Succeeded; } - // NOTE: In the below control flow if you decide to add a new command or modify a command such that it will now do an early return with FALSE, you must make sure you must reset etagState in FunctionState /// - public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public readonly bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { switch (input.header.cmd) { - case RespCommand.DELIFEXPIM: - if (oldValue.MetadataSize == 8 && input.header.CheckExpiry(oldValue.ExtraMetadata)) - { - rmwInfo.Action = RMWAction.ExpireAndStop; - } - - return false; case RespCommand.DELIFGREATER: - if (rmwInfo.RecordInfo.ETag) - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref oldValue); - - long etagFromClient = input.parseState.GetLong(0); - if (etagFromClient > functionsState.etagState.etag) - { - rmwInfo.Action = RMWAction.ExpireAndStop; - } - - EtagState.ResetState(ref functionsState.etagState); - // We always return false because we would rather not create a new record in hybrid log if we don't need to delete the object. - // Setting no Action and returning false for non-delete case will shortcircuit the InternalRMW code to not run CU, and return SUCCESS. - // If we want to delete the object setting the Action to ExpireAndStop will add the tombstone in hybrid log for us. - return false; - case RespCommand.SETIFGREATER: case RespCommand.SETIFMATCH: - if (rmwInfo.RecordInfo.ETag) - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref oldValue); - - long etagToCheckWith = input.parseState.GetLong(1); - - // in IFMATCH we check for equality, in IFGREATER we are checking for sent etag being strictly greater - int comparisonResult = etagToCheckWith.CompareTo(functionsState.etagState.etag); - int expectedResult = input.header.cmd is RespCommand.SETIFMATCH ? 0 : 1; - - if (comparisonResult == expectedResult) - { - return true; - } - - if (input.header.CheckSetGetFlag()) - { - // Copy value to output for the GET part of the command. - CopyRespWithEtagData(ref oldValue, ref output, hasEtagInVal: rmwInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); - } - else - { - // write back array of the format [etag, nil] - var nilResponse = functionsState.nilResp; - // *2\r\n: + + \r\n + - WriteValAndEtagToDst( - 4 + 1 + NumUtils.CountDigits(functionsState.etagState.etag) + 2 + nilResponse.Length, - ref nilResponse, - functionsState.etagState.etag, - ref output, - functionsState.memoryPool, - writeDirect: true - ); - } - - EtagState.ResetState(ref functionsState.etagState); - return false; + case RespCommand.SETWITHETAG: + return HandleEtagNeedCopyUpdate(input.header.cmd, in srcLogRecord, ref input, ref output, ref rmwInfo); case RespCommand.SETEXNX: // Expired data, return false immediately // ExpireAndResume ensures that we set as new value, since it does not exist - if (oldValue.MetadataSize == 8 && input.header.CheckExpiry(oldValue.ExtraMetadata)) + if (srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) { rmwInfo.Action = RMWAction.ExpireAndResume; - rmwInfo.RecordInfo.ClearHasETag(); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); return false; } @@ -1020,45 +933,36 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB if (input.header.CheckSetGetFlag()) { // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); - } - else if (input.header.CheckWithEtagFlag()) - { - // EXX when unsuccesful will write back NIL - CopyDefaultResp(functionsState.nilResp, ref output); + CopyRespTo(srcLogRecord.ValueSpan, ref output); } - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); return false; case RespCommand.SETEXXX: // Expired data, return false immediately so we do not set, since it does not exist // ExpireAndStop ensures that caller sees a NOTFOUND status - if (oldValue.MetadataSize == 8 && input.header.CheckExpiry(oldValue.ExtraMetadata)) + if (srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) { - rmwInfo.RecordInfo.ClearHasETag(); rmwInfo.Action = RMWAction.ExpireAndStop; - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); return false; } return true; + case RespCommand.RICREATE: + // Index already exists — never copy-update, reject in caller + return false; + case RespCommand.RIPROMOTE: + // Always copy to tail to promote from read-only region + return true; + case RespCommand.RIRESTORE: + // Copy to tail if needed, then IPU will set TreeHandle + return true; default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { - if (rmwInfo.RecordInfo.ETag) - { - CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return false; - } - - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { var ret = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd) - .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(functionsState.etagState.etagSkippedStart), ref writer); + .NeedCopyUpdate(srcLogRecord.Key, ref input, srcLogRecord.ValueSpan, ref writer); return ret; } finally @@ -1070,352 +974,370 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB } } - // NOTE: Before doing any return from this method, please make sure you are calling reset on etagState in functionsState. /// - public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public readonly bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { // Expired data - if (oldValue.MetadataSize == 8 && input.header.CheckExpiry(oldValue.ExtraMetadata)) + if (srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) { - recordInfo.ClearHasETag(); + _ = dstLogRecord.RemoveETag(); rmwInfo.Action = RMWAction.ExpireAndResume; - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); return false; } - rmwInfo.ClearExtraValueLength(ref recordInfo, ref newValue, newValue.TotalSize); + var oldValue = srcLogRecord.ValueSpan; // reduce redundant length calcs + // Do not pre-get newValue = dstLogRecord.ValueSpan here, because it may change, e.g. moving between inline and overflow RespCommand cmd = input.header.cmd; - bool recordHadEtagPreMutation = recordInfo.ETag; - bool shouldUpdateEtag = recordHadEtagPreMutation; - if (shouldUpdateEtag) - { - // during checkpointing we might skip the inplace calls and go directly to copy update so we need to initialize here if needed - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref oldValue); - } - switch (cmd) { - case RespCommand.SETIFGREATER: case RespCommand.SETIFMATCH: - // By now the comparison for etag against existing etag has already been done in NeedCopyUpdate - shouldUpdateEtag = true; - // Copy input to value - ReadOnlySpan src = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - - // retain metadata unless metadata sent - int metadataSize = input.arg1 != 0 ? sizeof(long) : oldValue.MetadataSize; - - Debug.Assert(src.Length + EtagConstants.EtagSize + metadataSize == newValue.Length); - - newValue.ExtraMetadata = oldValue.ExtraMetadata; - if (input.arg1 != 0) - { - newValue.ExtraMetadata = input.arg1; - } - - Span dest = newValue.AsSpan(EtagConstants.EtagSize); - src.CopyTo(dest); - - long etagFromClient = input.parseState.GetLong(1); - - functionsState.etagState.etag = etagFromClient; - - long etagForResponse = cmd == RespCommand.SETIFMATCH ? functionsState.etagState.etag + 1 : functionsState.etagState.etag; - - recordInfo.SetHasETag(); - - // Write Etag and Val back to Client - // write back array of the format [etag, nil] - var nilResp = functionsState.nilResp; - // *2\r\n: + + \r\n + - var numDigitsInEtag = NumUtils.CountDigits(etagForResponse); - WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, etagForResponse, ref output, functionsState.memoryPool, writeDirect: true); - break; + case RespCommand.SETIFGREATER: + case RespCommand.SETWITHETAG: + return HandleEtagCopyUpdateWorker(cmd, in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output); case RespCommand.SET: case RespCommand.SETEXXX: - var nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; - var nextUpdateEtagAccountedLength = functionsState.etagState.etagAccountedLength; - bool inputWithEtag = input.header.CheckWithEtagFlag(); - - // only when both are not false && false or true and true, do we need to readjust - if (inputWithEtag != shouldUpdateEtag) - { - // in the common path the above condition is skipped - if (inputWithEtag) - { - // nextUpdate will add etag but currently there is no etag - nextUpdateEtagOffset = EtagConstants.EtagSize; - shouldUpdateEtag = true; - recordInfo.SetHasETag(); - } - else - { - // nextUpdate will remove etag but currentyly there is an etag - nextUpdateEtagOffset = 0; - shouldUpdateEtag = false; - recordInfo.ClearHasETag(); - } - } - // Check if SetGet flag is set if (input.header.CheckSetGetFlag()) { - Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(srcLogRecord.ValueSpan, ref output); } - // Copy input to value var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - metadataSize = input.arg1 == 0 ? 0 : sizeof(long); - - // new value when allocated should have 8 bytes more if the previous record had etag and the cmd was not SETEXXX - Debug.Assert(newInputValue.Length + metadataSize + nextUpdateEtagOffset == newValue.Length); + Debug.Assert(newInputValue.Length == dstLogRecord.ValueSpan.Length); - newValue.ExtraMetadata = input.arg1; - newInputValue.CopyTo(newValue.AsSpan(nextUpdateEtagOffset)); + // Copy input to value, along with optionals from source record including Expiration. + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(newInputValue, in sizeInfo) || !dstLogRecord.TryCopyOptionals(in srcLogRecord, in sizeInfo)) + return false; - if (inputWithEtag) - { - CopyRespNumber(functionsState.etagState.etag + 1, ref output); - } + // Update expiration if it was supplied. + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; break; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: - nextUpdateEtagOffset = functionsState.etagState.etagSkippedStart; - nextUpdateEtagAccountedLength = functionsState.etagState.etagAccountedLength; - inputWithEtag = input.header.CheckWithEtagFlag(); - - // only when both are not false && false or true and true, do we need to readjust - if (inputWithEtag != shouldUpdateEtag) - { - // in the common path the above condition is skipped - if (inputWithEtag) - { - // nextUpdate will add etag but currently there is no etag - nextUpdateEtagOffset = EtagConstants.EtagSize; - shouldUpdateEtag = true; - recordInfo.SetHasETag(); - } - else - { - shouldUpdateEtag = false; - // nextUpdate will remove etag but currentyly there is an etag - nextUpdateEtagOffset = 0; - recordInfo.ClearHasETag(); - } - } - - var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - - Debug.Assert(oldValue.MetadataSize + setValue.Length + nextUpdateEtagOffset == newValue.Length); - - // Check if SetGet flag is set + // If the SetGet flag is set, copy the current value to output for the GET part of the command. if (input.header.CheckSetGetFlag()) { - Debug.Assert(!input.header.CheckWithEtagFlag(), "SET GET CANNNOT BE CALLED WITH WITHETAG"); // Copy value to output for the GET part of the command. - CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); - } - - // Copy input to value, retain metadata of oldValue - newValue.ExtraMetadata = oldValue.ExtraMetadata; - setValue.CopyTo(newValue.AsSpan(nextUpdateEtagOffset)); - - if (inputWithEtag) - { - CopyRespNumber(functionsState.etagState.etag + 1, ref output); + CopyRespTo(srcLogRecord.ValueSpan, ref output); } - break; - - case RespCommand.EXPIRE: - shouldUpdateEtag = false; - - var expiryExists = oldValue.MetadataSize == 8; - - var expirationWithOption = new ExpirationWithOption(input.arg1); - - EvaluateExpireCopyUpdate(expirationWithOption.ExpireOption, expiryExists, expirationWithOption.ExpirationTimeInTicks, ref oldValue, ref newValue, ref output); - break; + var inputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(inputValue, in sizeInfo)) + return false; - case RespCommand.PERSIST: - shouldUpdateEtag = false; - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - if (oldValue.MetadataSize == 8) - { - newValue.AsSpan().CopyTo(newValue.AsSpanWithMetadata()); - newValue.ShrinkSerializedLength(newValue.Length - newValue.MetadataSize); - newValue.UnmarkExtraMetadata(); - output.SpanByte.AsSpan()[0] = 1; - } break; case RespCommand.INCR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: 1, functionsState.etagState.etagSkippedStart); + if (!TryCopyUpdateNumber(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref output, input: 1)) + return false; break; case RespCommand.DECR: - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -1, functionsState.etagState.etagSkippedStart); + if (!TryCopyUpdateNumber(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref output, input: -1)) + return false; break; case RespCommand.INCRBY: var incrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrBy, functionsState.etagState.etagSkippedStart); + if (!TryCopyUpdateNumber(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref output, input: incrBy)) + return false; break; case RespCommand.DECRBY: var decrBy = input.arg1; - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: -decrBy, functionsState.etagState.etagSkippedStart); + if (!TryCopyUpdateNumber(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref output, input: -decrBy)) + return false; break; case RespCommand.INCRBYFLOAT: var incrByFloat = BitConverter.Int64BitsToDouble(input.arg1); - TryCopyUpdateNumber(ref oldValue, ref newValue, ref output, input: incrByFloat, functionsState.etagState.etagSkippedStart); + _ = TryCopyUpdateNumber(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref output, input: incrByFloat); break; case RespCommand.SETBIT: var bOffset = input.arg1; var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); - Buffer.MemoryCopy(oldValue.ToPointer(), newValue.ToPointer(), newValue.Length, oldValue.Length); - var oldValSet = BitmapManager.UpdateBitmap(newValue.ToPointer(), bOffset, bSetVal); - if (oldValSet == 0) - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output); + + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + + // Some duplicate code to avoid "fixed" when possible + var newValue = dstLogRecord.ValueSpan; + byte* oldValuePtr; + byte oldValSet; + if (srcLogRecord.IsPinnedValue) + { + oldValuePtr = srcLogRecord.PinnedValuePointer; + if (dstLogRecord.IsPinnedValue) + { + var newValuePtr = dstLogRecord.PinnedValuePointer; + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + oldValSet = BitmapManager.UpdateBitmap(newValuePtr, bOffset, bSetVal); + } + else + { + fixed (byte* newValuePtr = dstLogRecord.ValueSpan) + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + oldValSet = BitmapManager.UpdateBitmap(newValuePtr, bOffset, bSetVal); + } + } + } else - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output); + { + fixed (byte* oldPtr = srcLogRecord.ValueSpan) + { + oldValuePtr = oldPtr; + if (dstLogRecord.IsPinnedValue) + { + var newValuePtr = dstLogRecord.PinnedValuePointer; + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + oldValSet = BitmapManager.UpdateBitmap(newValuePtr, bOffset, bSetVal); + } + else + { + fixed (byte* newValuePtr = dstLogRecord.ValueSpan) + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + oldValSet = BitmapManager.UpdateBitmap(newValuePtr, bOffset, bSetVal); + } + } + } + } + + functionsState.CopyDefaultResp( + oldValSet == 0 ? CmdStrings.RESP_RETURN_VAL_0 : CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); break; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - var oldValuePtr = oldValue.ToPointer() + functionsState.etagState.etagSkippedStart; - var newValuePtr = newValue.ToPointer() + functionsState.etagState.etagSkippedStart; - var oldValueLength = oldValue.Length - functionsState.etagState.etagSkippedStart; - var newValueLength = newValue.Length - functionsState.etagState.etagSkippedStart; - Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValueLength, oldValueLength); - if (newValueLength > oldValueLength) + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + + newValue = dstLogRecord.ValueSpan; + oldValue = srcLogRecord.ValueSpan; + if (newValue.Length > oldValue.Length) { // Zero-init the rest of the new value before we do any bit operations (e.g. it may have been revivified, which for efficiency does not clear old data) - new Span(newValuePtr + oldValueLength, newValueLength - oldValueLength).Clear(); + newValue.Slice(oldValue.Length).Clear(); } - var (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValuePtr, newValueLength); + + long bitfieldReturnValue; + bool overflow; + if (dstLogRecord.IsPinnedValue) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, dstLogRecord.PinnedValuePointer, newValue.Length); + else + fixed (byte* newValuePtr = newValue) + (bitfieldReturnValue, overflow) = BitmapManager.BitFieldExecute(bitFieldArgs, newValuePtr, newValue.Length); if (!overflow) - CopyRespNumber(bitfieldReturnValue, ref output); + functionsState.CopyRespNumber(bitfieldReturnValue, ref output.SpanByteAndMemory); else - CopyDefaultResp(functionsState.nilResp, ref output); + functionsState.CopyDefaultResp(functionsState.nilResp, ref output.SpanByteAndMemory); break; case RespCommand.PFADD: var updated = false; - var newValPtr = newValue.ToPointer(); - var oldValPtr = oldValue.ToPointer(); + newValue = dstLogRecord.ValueSpan; + + if (!dstLogRecord.TryCopyOptionals(in srcLogRecord, in sizeInfo)) + return false; - if (newValue.Length != oldValue.Length) - updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValPtr, newValPtr, newValue.Length); + // Some duplicate code to avoid "fixed" when possible + newValue = dstLogRecord.ValueSpan; + if (srcLogRecord.IsPinnedValue) + { + oldValuePtr = srcLogRecord.PinnedValuePointer; + if (dstLogRecord.IsPinnedValue) + { + var newValuePtr = dstLogRecord.PinnedValuePointer; + if (newValue.Length != oldValue.Length) + updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValuePtr, newValuePtr, newValue.Length); + else + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + _ = HyperLogLog.DefaultHLL.Update(ref input, newValuePtr, newValue.Length, ref updated); + } + } + else + { + fixed (byte* newValuePtr = dstLogRecord.ValueSpan) + { + if (newValue.Length != oldValue.Length) + updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValuePtr, newValuePtr, newValue.Length); + else + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + _ = HyperLogLog.DefaultHLL.Update(ref input, newValuePtr, newValue.Length, ref updated); + } + } + } + } else { - Buffer.MemoryCopy(oldValPtr, newValPtr, newValue.Length, oldValue.Length); - HyperLogLog.DefaultHLL.Update(ref input, newValPtr, newValue.Length, ref updated); + fixed (byte* oldPtr = srcLogRecord.ValueSpan) + { + oldValuePtr = oldPtr; + if (dstLogRecord.IsPinnedValue) + { + var newValuePtr = dstLogRecord.PinnedValuePointer; + if (newValue.Length != oldValue.Length) + updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValuePtr, newValuePtr, newValue.Length); + else + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + _ = HyperLogLog.DefaultHLL.Update(ref input, newValuePtr, newValue.Length, ref updated); + } + } + else + { + fixed (byte* newValuePtr = dstLogRecord.ValueSpan) + { + if (newValue.Length != oldValue.Length) + updated = HyperLogLog.DefaultHLL.CopyUpdate(ref input, oldValuePtr, newValuePtr, newValue.Length); + else + { + Buffer.MemoryCopy(oldValuePtr, newValuePtr, newValue.Length, oldValue.Length); + _ = HyperLogLog.DefaultHLL.Update(ref input, newValuePtr, newValue.Length, ref updated); + } + } + } + } } - *output.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; + + *output.SpanByteAndMemory.SpanByte.ToPointer() = updated ? (byte)1 : (byte)0; break; case RespCommand.PFMERGE: + if (!dstLogRecord.TryCopyOptionals(in srcLogRecord, in sizeInfo)) + return false; + + // Explanation of variables: //srcA offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy +4 (skip len size) - var srcHLLPtr = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); // HLL merging from - var oldDstHLLPtr = oldValue.ToPointer(); // original HLL merging to (too small to hold its data plus srcA) - var newDstHLLPtr = newValue.ToPointer(); // new HLL merging to (large enough to hold srcA and srcB + var srcHLLPtr = input.parseState.GetArgSliceByRef(0).ToPointer(); // HLL merging from + // byte* oldDstHLLPtr = oldValue.ToPointer(); // original HLL merging to (too small to hold its data plus srcA) + // byte* newDstHLLPtr = newValue.ToPointer(); // new HLL merging to (large enough to hold srcA and srcB + + // Zeroinit any extra space in the new value (e.g. revivified record does not clear it out, for efficiency). + newValue = dstLogRecord.ValueSpan; + if (oldValue.Length < newValue.Length) + newValue.Slice(oldValue.Length).Clear(); + + // Some duplicate code to avoid "fixed" when possible + if (srcLogRecord.IsPinnedValue) + { + var oldDstHLLPtr = srcLogRecord.PinnedValuePointer; + if (dstLogRecord.IsPinnedValue) + { + var newDstHLLPtr = dstLogRecord.PinnedValuePointer; + HyperLogLog.DefaultHLL.CopyUpdateMerge(srcHLLPtr, oldDstHLLPtr, newDstHLLPtr, oldValue.Length, newValue.Length); + } + else + { + fixed (byte* newDstHLLPtr = dstLogRecord.ValueSpan) + HyperLogLog.DefaultHLL.CopyUpdateMerge(srcHLLPtr, oldDstHLLPtr, newDstHLLPtr, oldValue.Length, newValue.Length); + } + } + else + { + fixed (byte* oldDstHLLPtr = srcLogRecord.ValueSpan) + { + if (dstLogRecord.IsPinnedValue) + { + var newDstHLLPtr = dstLogRecord.PinnedValuePointer; + HyperLogLog.DefaultHLL.CopyUpdateMerge(srcHLLPtr, oldDstHLLPtr, newDstHLLPtr, oldValue.Length, newValue.Length); + } + else + { + fixed (byte* newDstHLLPtr = dstLogRecord.ValueSpan) + HyperLogLog.DefaultHLL.CopyUpdateMerge(srcHLLPtr, oldDstHLLPtr, newDstHLLPtr, oldValue.Length, newValue.Length); + } + } + } - HyperLogLog.DefaultHLL.CopyUpdateMerge(srcHLLPtr, oldDstHLLPtr, newDstHLLPtr, oldValue.Length, newValue.Length); break; case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); - oldValue.CopyTo(ref newValue); - newInputValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - var oldValueDataSpan = oldValue.AsSpan(functionsState.etagState.etagSkippedStart); - var newValueDataSpan = newValue.AsSpan(functionsState.etagState.etagSkippedStart); - if (oldValueDataSpan.Length < offset) + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + + newValue = dstLogRecord.ValueSpan; + if (oldValue.Length < offset) { - // If the offset is greater than the old value, we need to zero-fill the gap (e.g. new record might have been revivified). - var zeroFillLength = offset - oldValueDataSpan.Length; - newValueDataSpan.Slice(oldValueDataSpan.Length, zeroFillLength).Clear(); + // The offset is greater than the old value length so we need to zero-fill the gap (e.g. new record might have been revivified, which does not clear out all bytes). + newValue.Slice(oldValue.Length, offset - oldValue.Length).Clear(); } - newInputValue.CopyTo(newValueDataSpan.Slice(offset)); - CopyValueLengthToOutput(ref newValue, ref output, functionsState.etagState.etagSkippedStart); + input.parseState.GetArgSliceByRef(1).ReadOnlySpan.CopyTo(newValue.Slice(offset)); + + _ = TryCopyValueLengthToOutput(newValue, ref output); break; case RespCommand.GETDEL: // Copy value to output for the GET part of the command. // Then, set ExpireAndStop action to delete the record. - CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(oldValue, ref output); rmwInfo.Action = RMWAction.ExpireAndStop; - - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); return false; case RespCommand.GETEX: - shouldUpdateEtag = false; - CopyRespTo(ref oldValue, ref output, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); + CopyRespTo(oldValue, ref output); + + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + newValue = dstLogRecord.ValueSpan; + Debug.Assert(newValue.Length == oldValue.Length); if (input.arg1 > 0) { - Debug.Assert(newValue.Length == oldValue.Length + sizeof(long)); - byte* pbOutput = stackalloc byte[ObjectOutputHeader.Size]; - var _output = new SpanByteAndMemory(SpanByte.FromPinnedPointer(pbOutput, ObjectOutputHeader.Size)); + var _output = StringOutput.FromPinnedSpan(stackalloc byte[sizeof(int)]); var newExpiry = input.arg1; - EvaluateExpireCopyUpdate(ExpireOption.None, expiryExists: oldValue.MetadataSize == 8, newExpiry, ref oldValue, ref newValue, ref _output); + if (!EvaluateExpireCopyUpdate(ref dstLogRecord, in sizeInfo, ExpireOption.None, newExpiry, newValue, ref _output)) + return false; } - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - if (input.parseState.Count > 0) { - var persist = input.parseState.GetArgSliceByRef(0).ReadOnlySpan - .EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST); - + var persist = input.parseState.GetArgSliceByRef(0).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST); if (persist) // Persist the key - { - newValue.AsSpan().CopyTo(newValue.AsSpanWithMetadata()); - newValue.ShrinkSerializedLength(newValue.Length - newValue.MetadataSize); - newValue.UnmarkExtraMetadata(); - } + _ = dstLogRecord.RemoveExpiration(); } break; case RespCommand.APPEND: - // Copy any existing value with metadata to thew new value - oldValue.CopyTo(ref newValue); - var appendValue = input.parseState.GetArgSliceByRef(0); + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; // Append the new value with the client input at the end of the old data - appendValue.ReadOnlySpan.CopyTo(newValue.AsSpan().Slice(oldValue.LengthWithoutMetadata)); + newValue = dstLogRecord.ValueSpan; + appendValue.ReadOnlySpan.CopyTo(newValue.Slice(oldValue.Length)); - CopyValueLengthToOutput(ref newValue, ref output, functionsState.etagState.etagSkippedStart); + _ = TryCopyValueLengthToOutput(newValue, ref output); break; case RespCommand.VADD: // Handle "make me delete-able" if (input.arg1 == VectorManager.DeleteAfterDropArg) { - newValue.AsSpan().Clear(); + dstLogRecord.ValueSpan.Clear(); } else if (input.arg1 == VectorManager.RecreateIndexArg) { var newIndexPtr = MemoryMarshal.Read(input.parseState.GetArgSliceByRef(11).Span); - oldValue.CopyTo(ref newValue); + oldValue.CopyTo(dstLogRecord.ValueSpan); - functionsState.vectorManager.RecreateIndex(newIndexPtr, ref newValue); + functionsState.vectorManager.RecreateIndex(newIndexPtr, dstLogRecord.ValueSpan); } break; @@ -1427,33 +1349,19 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte default: if (input.header.cmd > RespCommandExtensions.LastValidCommand) { - if (recordInfo.ETag) - { - CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref output); - // reset etag state that may have been initialized earlier - EtagState.ResetState(ref functionsState.etagState); - return true; - } - var functions = functionsState.GetCustomCommandFunctions((ushort)input.header.cmd); var expirationInTicks = input.arg1; - if (expirationInTicks == 0) - { - // We want to retain the old metadata - newValue.ExtraMetadata = oldValue.ExtraMetadata; - } - else if (expirationInTicks > 0) + if (expirationInTicks > 0) { - // We want to add the given expiration - newValue.ExtraMetadata = expirationInTicks; + // We want to update to the given expiration + if (!dstLogRecord.TrySetExpiration(expirationInTicks)) + return false; } - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { - var ret = functions - .CopyUpdater(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(functionsState.etagState.etagSkippedStart), newValue.AsSpan(functionsState.etagState.etagSkippedStart), ref writer, ref rmwInfo); - return ret; + return functions.CopyUpdater(srcLogRecord.Key, ref input, oldValue, dstLogRecord.ValueSpan, ref writer, ref rmwInfo); } finally { @@ -1461,41 +1369,107 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte } } throw new GarnetException("Unsupported operation on input"); - } + case RespCommand.RICREATE: + // NeedCopyUpdate returns false for RICREATE, so CopyUpdater should never be reached. + throw new GarnetException("CopyUpdater should not be called for RICREATE"); + case RespCommand.RIPROMOTE: + { + // Copy stub bytes from source to destination, clearing the flushed flag. + var srcValue = srcLogRecord.ValueSpan; + if (!dstLogRecord.TrySetContentLengths(RangeIndexManager.IndexSizeBytes, in sizeInfo)) + return false; + srcValue.CopyTo(dstLogRecord.ValueSpan); + RangeIndexManager.ClearFlushedFlag(dstLogRecord.ValueSpan); - rmwInfo.SetUsedValueLength(ref recordInfo, ref newValue, newValue.TotalSize); + // Preserve the RecordType + var dataHeader = dstLogRecord.RecordDataHeader; + dataHeader.RecordType = RangeIndexManager.RangeIndexRecordType; - if (shouldUpdateEtag) - { - if (cmd is not RespCommand.SETIFGREATER) - functionsState.etagState.etag++; + // NOTE: Source TreeHandle is cleared in PostCopyUpdater (after CAS success) + // to avoid orphaning the tree if the CAS fails. + } + break; + case RespCommand.RIRESTORE: + { + // Copy stub bytes and set TreeHandle from restored BfTree. + var srcValue = srcLogRecord.ValueSpan; + if (!dstLogRecord.TrySetContentLengths(RangeIndexManager.IndexSizeBytes, in sizeInfo)) + return false; + srcValue.CopyTo(dstLogRecord.ValueSpan); + RangeIndexManager.RecreateIndex((nint)input.arg1, dstLogRecord.ValueSpan); - newValue.SetEtagInPayload(functionsState.etagState.etag); - EtagState.ResetState(ref functionsState.etagState); + var dataHeader = dstLogRecord.RecordDataHeader; + dataHeader.RecordType = RangeIndexManager.RangeIndexRecordType; + } + break; } - else if (recordHadEtagPreMutation) - EtagState.ResetState(ref functionsState.etagState); + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); return true; } /// - public bool PostCopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public readonly bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); - if (functionsState.appendOnlyFile != null) - rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + var cmd = input.header.cmd; + + // RIPROMOTE/RIRESTORE are internal store-maintenance ops — skip AOF. + if (cmd != RespCommand.RIPROMOTE && cmd != RespCommand.RIRESTORE) + { + if (functionsState.appendOnlyFile != null) + rmwInfo.UserData |= NeedAofLog; + } + + // RIPROMOTE: pass ownership of the BfTree from src to dst. + // • If src.TreeHandle != 0 (live transfer): dst inherited the handle via byte-copy in + // CopyUpdater; clear src.TreeHandle so a later eviction of src does not free the tree. + // • If src.TreeHandle == 0 (cold case — src was post-eviction or post-recovery): pre-stage + // data.bftree from .flush.bftree and register a pending entry so the next + // checkpoint captures dst's content. Cleanly handles steady-state cold-restore, recovery + // Scenario D (below-FUA-at-checkpoint stub recovered), and any other path that promotes + // a flushed stub with TreeHandle == 0. + // • In BOTH branches: set src.IsTransferred so a later OnEvict on the stale source does + // not remove the liveIndexes entry (live: owned by dst's tree; cold: owned by the new + // pending entry), and a later OnFlush on the stale source does not snapshot a stale view. + if (cmd == RespCommand.RIPROMOTE) + { + var srcSpan = srcLogRecord.ValueSpan; + var srcHandle = RangeIndexManager.ReadIndex(srcSpan).TreeHandle; + if (srcHandle != nint.Zero) + { + RangeIndexManager.ClearTreeHandle(srcSpan); + } + else + { + // rmwInfo.SourceAddress is the source logical address (preserved through + // CopyUpdater; rmwInfo.Address has been reassigned to the destination). + if (functionsState.storeWrapper?.rangeIndexManager is { } rim + && rmwInfo.SourceAddress != Tsavorite.core.LogAddress.kInvalidAddress) + { + rim.PreStageAndRegisterPending(dstLogRecord.Key, rmwInfo.SourceAddress); + } + } + + RangeIndexManager.SetTransferredFlag(srcSpan); + } + return true; } /// - public void PostRMWOperation(ref SpanByte key, ref RawStringInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostRMWOperation(TKey key, ref StringInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((rmwInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogRMW(ref key, ref input, rmwInfo.Version, rmwInfo.SessionID, epochAccessor); - } + WriteLogRMW(key.KeyBytes, ref input, rmwInfo.Version, rmwInfo.SessionID, epochAccessor); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/ReadMethods.Etags.cs b/libs/server/Storage/Functions/MainStore/ReadMethods.Etags.cs new file mode 100644 index 00000000000..49a3567fbdc --- /dev/null +++ b/libs/server/Storage/Functions/MainStore/ReadMethods.Etags.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// ETag-specific Read callback methods for main store, kept in a separate file + /// with NoInlining to minimize hot-path method footprint. + /// + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + { + /// + /// Handles GETWITHETAG and GETIFNOTMATCH commands. Called from Reader via early delegation. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private bool HandleEtagReader(in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, ref ReadInfo readInfo, + RespCommand cmd, ReadOnlySpan value) + where TSourceLogRecord : ISourceLogRecord + { + if (cmd == RespCommand.GETIFNOTMATCH) + { + // Check if the client's ETag matches; if so, return [etag, nil] without the value + long etagToMatchAgainst = input.parseState.GetLong(0); + long existingEtag = srcLogRecord.ETag; + + if (existingEtag == etagToMatchAgainst) + { + // Write back array of the format [etag, nil] + var nilResp = functionsState.nilResp; + var numDigitsInEtag = NumUtils.CountDigits(existingEtag); + WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, nilResp, existingEtag, ref output, functionsState.memoryPool, writeDirect: true); + return true; + } + } + + // For both GETWITHETAG and GETIFNOTMATCH (when etag didn't match), return [etag, value] + long etag = srcLogRecord.Info.HasETag ? srcLogRecord.ETag : LogRecord.NoETag; + CopyRespWithEtagData(value, ref output, srcLogRecord.Info.HasETag, etag, functionsState.memoryPool); + return true; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/ReadMethods.cs b/libs/server/Storage/Functions/MainStore/ReadMethods.cs index 53d6a72fe3f..dfcde19180d 100644 --- a/libs/server/Storage/Functions/MainStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/MainStore/ReadMethods.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Diagnostics; @@ -10,136 +10,51 @@ namespace Garnet.server /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { /// - public bool SingleReader( - ref SpanByte key, ref RawStringInput input, - ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + public bool Reader(in TSourceLogRecord srcLogRecord, ref StringInput input, ref StringOutput output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord { - if (value.MetadataSize == 8 && CheckExpiry(ref value)) - { - readInfo.RecordInfo.ClearHasETag(); - return false; - } - - var cmd = input.header.cmd; - - // Ignore special Vector Set logic if we're scanning, detected with cmd == NONE - if (cmd != RespCommand.NONE) - { - // Vector sets are reachable (key not mangled) and hidden. - // So we can use that to detect type mismatches. - if (readInfo.RecordInfo.VectorSet && !cmd.IsLegalOnVectorSet()) - { - // Attempted an illegal op on a VectorSet - readInfo.Action = ReadAction.CancelOperation; - return false; - } - else if (!readInfo.RecordInfo.VectorSet && cmd.IsLegalOnVectorSet()) - { - // Attempted a vector set op on a non-VectorSet - readInfo.Action = ReadAction.CancelOperation; - return false; - } - } - - // GET is used in a number of non-RESP contexts, which messes up existing logic - // - // Easiest to mark the actually-RESP commands with a < 0 arg1 and roll back to old logic - // after the Vector Set checks - // - // TODO: This is quite hacky, but requires a bunch of non-Vector Set changes - do those and remove - if (input.arg1 < 0 && cmd == RespCommand.GET) - { - cmd = RespCommand.NONE; - } + var info = srcLogRecord.Info; - if (cmd == RespCommand.GETIFNOTMATCH) - { - if (handleGetIfNotMatch(ref input, ref value, ref dst, ref readInfo)) - return true; - } - else if (cmd > RespCommandExtensions.LastValidCommand) + // Fast path for simple GET on a normal inline string key with no optional fields. + // HasOptionalOrObjectFields is false iff: KeyIsInline, ValueIsInline, !HasETag, !HasExpiration (implies !ValueIsObject). + // RecordType 0 means normal string (not VectorSet or RangeIndex). + // This avoids expiry checks (no expiration), type-safety checks, ETag handling, and custom command dispatch. + if (input.arg1 < 0 && !info.HasOptionalOrObjectFields && srcLogRecord.RecordType == 0) { - if (readInfo.RecordInfo.ETag) - { - CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref dst); - return true; - } - - var valueLength = value.LengthWithoutMetadata; - - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref dst); - try - { - var ret = functionsState.GetCustomCommandFunctions((ushort)cmd) - .Reader(key.AsReadOnlySpan(), ref input, value.AsReadOnlySpan(), ref writer, ref readInfo); - Debug.Assert(valueLength <= value.LengthWithoutMetadata); - return ret; - } - finally - { - writer.Dispose(); - } - } - - if (readInfo.RecordInfo.ETag) - { - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); - } - - // Unless the command explicitly asks for the ETag in response, we do not write back the ETag - if (cmd is (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH)) - { - CopyRespWithEtagData(ref value, ref dst, readInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); - EtagState.ResetState(ref functionsState.etagState); + CopyRespTo(srcLogRecord.ValueSpan, ref output); return true; } - if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); - else - { - CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); - } - - if (readInfo.RecordInfo.ETag) + if (info.ValueIsObject) { - EtagState.ResetState(ref functionsState.etagState); + readInfo.Action = ReadAction.WrongType; + return false; } - return true; - } - - /// - public bool ConcurrentReader( - ref SpanByte key, ref RawStringInput input, ref SpanByte value, - ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - if (value.MetadataSize == 8 && CheckExpiry(ref value)) - { - recordInfo.ClearHasETag(); + if (LogRecordUtils.CheckExpiry(in srcLogRecord)) return false; - } var cmd = input.header.cmd; - // Ignore special Vector Set logic if we're scanning, detected with cmd == NONE + // Type safety: prevent cross-type access (e.g., GET on RI key, RI.SET on string key). + // Hot path (normal GET/SET on normal string): RecordType == 0, cmd is not special → + // hits only the first condition (one byte compare), skips everything. if (cmd != RespCommand.NONE) { - // Vector sets are reachable (key not mangled) and hidden. - // So we can use that to detect type mismatches. - if (recordInfo.VectorSet && !cmd.IsLegalOnVectorSet()) + var recordType = srcLogRecord.RecordType; + if (recordType != 0) { - // Attempted an illegal op on a VectorSet - readInfo.Action = ReadAction.CancelOperation; - return false; + // Record has a special type (RI or Vector) — check if the command is allowed + if (CheckRecordTypeMismatch(recordType, cmd, ref readInfo)) + return false; } - else if (!recordInfo.VectorSet && cmd.IsLegalOnVectorSet()) + else if (cmd != RespCommand.GET && (cmd.IsRangeIndexCommand() || cmd.IsVectorSetCommand())) { - // Attempted a vector set op on a non-VectorSet - readInfo.Action = ReadAction.CancelOperation; + // RI/Vector command on a normal string key + readInfo.Action = ReadAction.WrongType; return false; } } @@ -155,27 +70,18 @@ public bool ConcurrentReader( cmd = RespCommand.NONE; } - if (cmd == RespCommand.GETIFNOTMATCH) - { - if (handleGetIfNotMatch(ref input, ref value, ref dst, ref readInfo)) - return true; - } - else if (cmd > RespCommandExtensions.LastValidCommand) - { - if (readInfo.RecordInfo.ETag) - { - CopyDefaultResp(CmdStrings.RESP_ERR_ETAG_ON_CUSTOM_PROC, ref dst); - return true; - } + var value = srcLogRecord.ValueSpan; // reduce redundant length calculations - var valueLength = value.LengthWithoutMetadata; + if (cmd > RespCommandExtensions.LastValidCommand) + { + var valueLength = value.Length; - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref dst); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { var ret = functionsState.GetCustomCommandFunctions((ushort)cmd) - .Reader(key.AsReadOnlySpan(), ref input, value.AsReadOnlySpan(), ref writer, ref readInfo); - Debug.Assert(valueLength <= value.LengthWithoutMetadata); + .Reader(srcLogRecord.Key, ref input, value, ref writer, ref readInfo); + Debug.Assert(valueLength <= value.Length); return ret; } finally @@ -184,48 +90,50 @@ public bool ConcurrentReader( } } - if (readInfo.RecordInfo.ETag) + switch (cmd) { - EtagState.SetValsForRecordWithEtag(ref functionsState.etagState, ref value); + case RespCommand.GETIFNOTMATCH: + case RespCommand.GETWITHETAG: + return HandleEtagReader(in srcLogRecord, ref input, ref output, ref readInfo, cmd, value); + case RespCommand.NONE: + CopyRespTo(value, ref output); + break; + default: + CopyRespToWithInput(in srcLogRecord, ref input, ref output, readInfo.IsFromPending); + break; } - // Unless the command explicitly asks for the ETag in response, we do not write back the ETag - if (cmd is (RespCommand.GETWITHETAG or RespCommand.GETIFNOTMATCH)) + return true; + } + + /// + /// Checks for type mismatches between the record's RecordType and the command. + /// Called only when RecordType != 0 (RI or Vector key). Separated from Reader + /// to keep the hot path compact. + /// + private static bool CheckRecordTypeMismatch(byte recordType, RespCommand cmd, ref ReadInfo readInfo) + { + // RangeIndex type safety + if (recordType == RangeIndexManager.RangeIndexRecordType && !cmd.IsLegalOnRangeIndex()) { - CopyRespWithEtagData(ref value, ref dst, readInfo.RecordInfo.ETag, functionsState.etagState.etagSkippedStart, functionsState.memoryPool); - EtagState.ResetState(ref functionsState.etagState); + readInfo.Action = ReadAction.WrongType; return true; } - - if (cmd == RespCommand.NONE) - CopyRespTo(ref value, ref dst, functionsState.etagState.etagSkippedStart, functionsState.etagState.etagAccountedLength); - else + if (recordType != RangeIndexManager.RangeIndexRecordType && cmd.IsRangeIndexCommand()) { - CopyRespToWithInput(ref input, ref value, ref dst, readInfo.IsFromPending); + readInfo.Action = ReadAction.WrongType; + return true; } - if (readInfo.RecordInfo.ETag) + // Vector set type safety + if (recordType == VectorManager.RecordType && !cmd.IsLegalOnVectorSet()) { - EtagState.ResetState(ref functionsState.etagState); + readInfo.Action = ReadAction.WrongType; + return true; } - - return true; - } - - private bool handleGetIfNotMatch(ref RawStringInput input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) - { - // Any value without an etag is treated the same as a value with an etag - long etagToMatchAgainst = input.parseState.GetLong(0); - - long existingEtag = readInfo.RecordInfo.ETag ? value.GetEtagInPayload() : EtagConstants.NoETag; - - if (existingEtag == etagToMatchAgainst) + if (recordType != VectorManager.RecordType && cmd.IsVectorSetCommand()) { - // write back array of the format [etag, nil] - var nilResp = functionsState.nilResp; - // *2\r\n: + + \r\n + - var numDigitsInEtag = NumUtils.CountDigits(existingEtag); - WriteValAndEtagToDst(4 + 1 + numDigitsInEtag + 2 + nilResp.Length, ref nilResp, existingEtag, ref dst, functionsState.memoryPool, writeDirect: true); + readInfo.Action = ReadAction.WrongType; return true; } diff --git a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs index 60c6ac225f5..b76a4d86f9a 100644 --- a/libs/server/Storage/Functions/MainStore/UpsertMethods.cs +++ b/libs/server/Storage/Functions/MainStore/UpsertMethods.cs @@ -1,58 +1,131 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Garnet.common; using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; namespace Garnet.server { /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { /// - public bool SingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ReadOnlySpan srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) { - // Since upsert may be on existing key we need to wipe out the record info property - recordInfo.ClearHasETag(); - return SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; } /// - public void PostSingleWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason) + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, IHeapObject srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) + => throw new GarnetException("String store should not be called with IHeapObject"); + + /// + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, in TSourceLogRecord inputLogRecord, ref StringOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (inputLogRecord.Info.ValueIsObject) + throw new GarnetException("String store should not be called with IHeapObject"); + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ReadOnlySpan srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) { functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); - if (reason == WriteReason.Upsert && functionsState.appendOnlyFile != null) + if (functionsState.appendOnlyFile != null) upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF } /// - public bool ConcurrentWriter(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, IHeapObject srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) + => throw new GarnetException("String store should not be called with IHeapObject"); + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, in TSourceLogRecord inputLogRecord, ref StringOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord { - // Since upsert may be on existing key we need to wipe out the record info property - recordInfo.ClearHasETag(); - if (ConcurrentWriterWorker(ref src, ref dst, ref input, ref upsertInfo, ref recordInfo)) + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) { - if (!upsertInfo.RecordInfo.Modified) - functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); - if (functionsState.appendOnlyFile != null) - upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF - return true; + Debug.Assert(!inputLogRecord.Info.ValueIsObject, "String store should not be called with IHeapObject"); + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + } + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref StringInput input, ReadOnlySpan srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) + { + // Prevent SET from overwriting VectorSet or RangeIndex stubs normal string records have RecordType 0; skip all checks in that common case. + var recordType = logRecord.RecordType; + if (recordType != 0 && (recordType == VectorManager.RecordType || recordType == RangeIndexManager.RangeIndexRecordType)) + { + upsertInfo.Action = UpsertAction.WrongType; + return false; } + + if (!InPlaceWriterForSpanValue(ref logRecord, ref input, srcValue, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + public bool InPlaceWriter(ref LogRecord logRecord, ref StringInput input, IHeapObject srcValue, ref StringOutput output, ref UpsertInfo upsertInfo) + { + GarnetException.Throw("String store should not be called with IHeapObject"); return false; } - static bool ConcurrentWriterWorker(ref SpanByte src, ref SpanByte dst, ref RawStringInput input, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - => SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, input.arg1); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref StringInput input, in TSourceLogRecord inputLogRecord, ref StringOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (inputLogRecord.Info.ValueIsObject) + GarnetException.Throw("String store should not be called with IHeapObject"); + if (!InPlaceWriterForLogRecordValue(ref logRecord, ref input, in inputLogRecord, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } /// - public void PostUpsertOperation(ref SpanByte key, ref RawStringInput input, ref SpanByte src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostUpsertOperation(TKey key, ref StringInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((upsertInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogUpsert(ref key, ref input, ref src, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); - } + WriteLogUpsert(key.KeyBytes, ref input, valueSpan, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostUpsertOperation(TKey key, ref StringInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + throw new GarnetException("String store should not be called with IHeapObject"); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs index 0130dcbe389..db28a191a53 100644 --- a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs @@ -10,133 +10,216 @@ namespace Garnet.server /// /// Callback functions for main store /// - public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions + public readonly unsafe partial struct MainSessionFunctions : ISessionFunctions { /// /// Parse ASCII byte array into long and validate that only contains ASCII decimal characters /// - /// Length of byte array - /// Pointer to byte array + /// Source string to evaluate /// Parsed long value /// True if input contained only ASCII decimal characters, otherwise false - static bool IsValidNumber(int length, byte* source, out long val) + static bool IsValidNumber(ReadOnlySpan source, out long val) { - val = 0; try { // Check for valid number - if (!NumUtils.TryReadInt64(length, source, out val)) - { - // Signal value is not a valid number - return false; - } + return NumUtils.TryReadInt64(source, out val); + } + catch + { + // Signal value is not a valid number + val = 0; + return false; + } + } + + /// + /// Parse ASCII byte array into long and validate that only contains ASCII decimal characters + /// + /// Source string to evaluate + /// Parsed long value + /// True if input contained only ASCII decimal characters, otherwise false + static bool IsValidNumber(byte* source, int sourceLen, out long val) + { + try + { + // Check for valid number + return NumUtils.TryReadInt64(sourceLen, source, out val); } catch { // Signal value is not a valid number + val = 0; return false; } - return true; } /// /// Parse ASCII byte array into double and validate that only contains ASCII decimal characters /// - /// Length of byte array - /// Pointer to byte array + /// Source string to evaluate /// Parsed long value /// True if input contained only ASCII decimal characters, otherwise false - static bool IsValidDouble(int length, byte* source, out double val) + static bool IsValidDouble(ReadOnlySpan source, out double val) { - val = 0; try { // Check for valid number - if (!NumUtils.TryReadDouble(length, source, out val) || !double.IsFinite(val)) - { - // Signal value is not a valid number - return false; - } + return NumUtils.TryReadDouble(source, out val) || !double.IsFinite(val); } catch { // Signal value is not a valid number + val = 0; + return false; + } + } + + /// + /// Parse ASCII byte array into double and validate that only contains ASCII decimal characters + /// + /// Source string to evaluate + /// Parsed long value + /// True if input contained only ASCII decimal characters, otherwise false + static bool IsValidDouble(byte* source, int sourceLen, out double val) + { + try + { + // Check for valid number + return NumUtils.TryReadDouble(sourceLen, source, out val) || !double.IsFinite(val); + } + catch + { + // Signal value is not a valid number + val = 0; return false; } - return true; } /// - public int GetRMWInitialValueLength(ref RawStringInput input) + public RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref StringInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { + // We know namespaces aren't present in string functions, so don't populate var cmd = input.header.cmd; + var fieldInfo = new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = 0, + HasETag = false + }; switch (cmd) { case RespCommand.SETBIT: var bOffset = input.arg1; - return sizeof(int) + BitmapManager.Length(bOffset); + fieldInfo.ValueSize = BitmapManager.Length(bOffset); + return fieldInfo; case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - return sizeof(int) + BitmapManager.LengthFromType(bitFieldArgs); + fieldInfo.ValueSize = BitmapManager.LengthFromType(bitFieldArgs); + return fieldInfo; case RespCommand.PFADD: - return sizeof(int) + HyperLogLog.DefaultHLL.SparseInitialLength(ref input); + fieldInfo.ValueSize = HyperLogLog.DefaultHLL.SparseInitialLength(ref input); + return fieldInfo; case RespCommand.PFMERGE: - var length = input.parseState.GetArgSliceByRef(0).SpanByte.Length; - return sizeof(int) + length; + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + return fieldInfo; + + case RespCommand.SETIFGREATER: + case RespCommand.SETIFMATCH: + case RespCommand.SETWITHETAG: + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasETag = true; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; + + case RespCommand.SET: + case RespCommand.SETEXNX: + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; + + case RespCommand.SETKEEPTTL: + // Copy input to value; do not change expiration + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + return fieldInfo; case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); - var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; - return sizeof(int) + newValue.Length + offset; + var newValue = input.parseState.GetArgSliceByRef(1); + fieldInfo.ValueSize = newValue.Length + offset; + return fieldInfo; case RespCommand.APPEND: var valueLength = input.parseState.GetArgSliceByRef(0).Length; - return sizeof(int) + valueLength; + fieldInfo.ValueSize = valueLength; + return fieldInfo; case RespCommand.INCR: - return sizeof(int) + 1; // # of digits in "1" + fieldInfo.ValueSize = 1; // # of digits in "1" + return fieldInfo; case RespCommand.DECR: - return sizeof(int) + 2; // # of digits in "-1" + fieldInfo.ValueSize = 2; // # of digits in "-1" + return fieldInfo; case RespCommand.INCRBY: var ndigits = NumUtils.CountDigits(input.arg1, out var isNegative); - return sizeof(int) + ndigits + (isNegative ? 1 : 0); + fieldInfo.ValueSize = ndigits + (isNegative ? 1 : 0); + return fieldInfo; case RespCommand.DECRBY: ndigits = NumUtils.CountDigits(-input.arg1, out isNegative); - return sizeof(int) + ndigits + (isNegative ? 1 : 0); + fieldInfo.ValueSize = ndigits + (isNegative ? 1 : 0); + return fieldInfo; + case RespCommand.INCRBYFLOAT: var incrByFloat = BitConverter.Int64BitsToDouble(input.arg1); - ndigits = NumUtils.CountCharsInDouble(incrByFloat, out var _, out var _, out var _); + fieldInfo.ValueSize = NumUtils.CountCharsInDouble(incrByFloat, out var _, out var _, out var _); + return fieldInfo; + + case RespCommand.RICREATE: + case RespCommand.RIPROMOTE: + case RespCommand.RIRESTORE: + fieldInfo.ValueSize = RangeIndexManager.IndexSizeBytes; + return fieldInfo; - return sizeof(int) + ndigits; case RespCommand.VADD: - return sizeof(int) + VectorManager.IndexSizeBytes; + fieldInfo.ValueSize = VectorManager.IndexSizeBytes; + fieldInfo.RecordType = VectorManager.RecordType; + return fieldInfo; default: if (cmd > RespCommandExtensions.LastValidCommand) { var functions = functionsState.GetCustomCommandFunctions((ushort)cmd); - // Compute metadata size for result - int metadataSize = input.arg1 switch - { - -1 => 0, - 0 => 0, - _ => 8, - }; - return sizeof(int) + metadataSize + functions.GetInitialLength(ref input); + fieldInfo.ValueSize = functions.GetInitialLength(ref input); } - - return sizeof(int) + input.parseState.GetArgSliceByRef(0).ReadOnlySpan.Length + (input.arg1 == 0 ? 0 : sizeof(long)) + this.functionsState.etagState.etagOffsetForVarlen; + else + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; } } /// - public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) + public RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref StringInput input) + where TSourceLogRecord : ISourceLogRecord { + var fieldInfo = new RecordFieldInfo() + { + KeySize = srcLogRecord.Key.Length, + ValueSize = 0, + HasETag = false, + HasExpiration = srcLogRecord.Info.HasExpiration, + RecordType = srcLogRecord.RecordType, + }; + if (input.header.cmd != RespCommand.NONE) { var cmd = input.header.cmd; @@ -147,132 +230,198 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) case RespCommand.INCRBY: var incrByValue = input.header.cmd == RespCommand.INCRBY ? input.arg1 : 1; - if (!NumUtils.TryReadInt64(t.AsSpan(functionsState.etagState.etagOffsetForVarlen), out var curr)) + var value = srcLogRecord.ValueSpan; + fieldInfo.ValueSize = 2; // # of digits in "-1", in case of invalid number (which may throw instead) + // TODO set error as in PrivateMethods.IsValidNumber and test in caller, to avoid the log record allocation. This would require 'output' + if (srcLogRecord.IsPinnedValue ? IsValidNumber(srcLogRecord.PinnedValuePointer, value.Length, out var curr) : IsValidNumber(value, out curr)) { - // Return enough space to copy over old value - return sizeof(int) + t.Length + functionsState.etagState.etagOffsetForVarlen; + // TODO Consider adding a way to cache curr/next for the IPU call + var next = curr + incrByValue; + fieldInfo.ValueSize = NumUtils.CountDigits(next, out var isNegative) + (isNegative ? 1 : 0); } - var next = curr + incrByValue; - - var ndigits = NumUtils.CountDigits(next, out var isNegative); - ndigits += isNegative ? 1 : 0; - - return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; + return fieldInfo; case RespCommand.DECR: case RespCommand.DECRBY: var decrByValue = input.header.cmd == RespCommand.DECRBY ? input.arg1 : 1; - curr = NumUtils.ReadInt64(t.AsSpan(functionsState.etagState.etagOffsetForVarlen)); - next = curr - decrByValue; - - ndigits = NumUtils.CountDigits(next, out isNegative); - ndigits += isNegative ? 1 : 0; - - return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; + value = srcLogRecord.ValueSpan; + fieldInfo.ValueSize = 2; // # of digits in "-1", in case of invalid number (which may throw instead). + if (srcLogRecord.IsPinnedValue ? IsValidNumber(srcLogRecord.PinnedValuePointer, value.Length, out curr) : IsValidNumber(value, out curr)) + { + // TODO Consider adding a way to cache curr/next for the IPU call + var next = curr - decrByValue; + fieldInfo.ValueSize = NumUtils.CountDigits(next, out var isNegative) + (isNegative ? 1 : 0); + } + return fieldInfo; case RespCommand.INCRBYFLOAT: var incrByFloat = BitConverter.Int64BitsToDouble(input.arg1); - NumUtils.TryReadDouble(t.AsSpan(functionsState.etagState.etagOffsetForVarlen), out var currVal); - var nextVal = currVal + incrByFloat; - - ndigits = NumUtils.CountCharsInDouble(nextVal, out _, out _, out _); + value = srcLogRecord.ValueSpan; + fieldInfo.ValueSize = 2; // # of digits in "-1", in case of invalid number (which may throw instead) + if (srcLogRecord.IsPinnedValue ? IsValidDouble(srcLogRecord.PinnedValuePointer, value.Length, out var currVal) : IsValidDouble(value, out currVal)) + { + // TODO Consider adding a way to cache currVal/nextVal for the IPU call + var nextVal = currVal + incrByFloat; + fieldInfo.ValueSize = NumUtils.CountCharsInDouble(nextVal, out _, out _, out _); + } + return fieldInfo; - return sizeof(int) + ndigits + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; case RespCommand.SETBIT: var bOffset = input.arg1; - return sizeof(int) + BitmapManager.NewBlockAllocLength(t.Length, bOffset); + fieldInfo.ValueSize = BitmapManager.NewBlockAllocLength(srcLogRecord.ValueSpan.Length, bOffset); + return fieldInfo; + case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); - return sizeof(int) + BitmapManager.NewBlockAllocLengthFromType(bitFieldArgs, t.Length); + fieldInfo.ValueSize = BitmapManager.NewBlockAllocLengthFromType(bitFieldArgs, srcLogRecord.ValueSpan.Length); + return fieldInfo; + case RespCommand.PFADD: - var length = sizeof(int); - var v = t.ToPointer(); - length += HyperLogLog.DefaultHLL.UpdateGrow(ref input, v); - return length + t.MetadataSize; + // TODO: call HyperLogLog.DefaultHLL.IsValidHYLL and check error return per RMWMethods. This would require 'output'. Also carry this result through to RMWMethods. + if (srcLogRecord.IsPinnedValue) + fieldInfo.ValueSize = HyperLogLog.DefaultHLL.UpdateGrow(ref input, srcLogRecord.PinnedValuePointer); + else + fixed (byte* valuePtr = srcLogRecord.ValueSpan) + fieldInfo.ValueSize = HyperLogLog.DefaultHLL.UpdateGrow(ref input, valuePtr); + return fieldInfo; case RespCommand.PFMERGE: - length = sizeof(int); - var srcHLL = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); - var dstHLL = t.ToPointer(); - length += HyperLogLog.DefaultHLL.MergeGrow(srcHLL, dstHLL); - return length + t.MetadataSize; + // TODO: call HyperLogLog.DefaultHLL.IsValidHYLL and check error return per RMWMethods. This would require 'output'. Also carry this result through to RMWMethods. + var srcHLL = input.parseState.GetArgSliceByRef(0).ToPointer(); + if (srcLogRecord.IsPinnedValue) + fieldInfo.ValueSize = HyperLogLog.DefaultHLL.MergeGrow(srcHLL, srcLogRecord.PinnedValuePointer); + else + fixed (byte* dstHLL = srcLogRecord.ValueSpan) + fieldInfo.ValueSize = HyperLogLog.DefaultHLL.MergeGrow(srcHLL, dstHLL); + return fieldInfo; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: - var setValue = input.parseState.GetArgSliceByRef(0); - return sizeof(int) + t.MetadataSize + setValue.Length + functionsState.etagState.etagOffsetForVarlen; + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + return fieldInfo; case RespCommand.SET: case RespCommand.SETEXXX: - return sizeof(int) + input.parseState.GetArgSliceByRef(0).Length + (input.arg1 == 0 ? 0 : sizeof(long)) + functionsState.etagState.etagOffsetForVarlen; - case RespCommand.PERSIST: - return sizeof(int) + t.LengthWithoutMetadata; + case RespCommand.SETEXNX: + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; + case RespCommand.SETIFGREATER: case RespCommand.SETIFMATCH: - var newValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; - int metadataSize = input.arg1 == 0 ? t.MetadataSize : sizeof(long); - return sizeof(int) + newValue.Length + EtagConstants.EtagSize + metadataSize; - case RespCommand.EXPIRE: - case RespCommand.PEXPIRE: - case RespCommand.EXPIREAT: - case RespCommand.PEXPIREAT: - return sizeof(int) + t.Length + sizeof(long); + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasETag = true; + fieldInfo.HasExpiration = input.arg1 != 0 || srcLogRecord.Info.HasExpiration; + return fieldInfo; + + case RespCommand.SETWITHETAG: + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasETag = true; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; case RespCommand.SETRANGE: var offset = input.parseState.GetInt(0); - newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; + var newValue = input.parseState.GetArgSliceByRef(1); - if (newValue.Length + offset > t.LengthWithoutMetadata - functionsState.etagState.etagOffsetForVarlen) - return sizeof(int) + newValue.Length + offset + t.MetadataSize + functionsState.etagState.etagOffsetForVarlen; - return sizeof(int) + t.Length; + fieldInfo.ValueSize = newValue.Length + offset; + if (fieldInfo.ValueSize < srcLogRecord.ValueSpan.Length) + fieldInfo.ValueSize = srcLogRecord.ValueSpan.Length; + return fieldInfo; case RespCommand.GETEX: - return sizeof(int) + t.LengthWithoutMetadata + (input.arg1 > 0 ? sizeof(long) : 0); + fieldInfo.ValueSize = srcLogRecord.ValueSpan.Length; + + // If both EX and PERSIST were specified, EX wins + if (input.arg1 > 0) + fieldInfo.HasExpiration = true; + else if (input.parseState.Count > 0) + { + if (input.parseState.GetArgSliceByRef(0).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST)) + fieldInfo.HasExpiration = false; + } + + return fieldInfo; case RespCommand.APPEND: - var valueLength = input.parseState.GetArgSliceByRef(0).Length; - return sizeof(int) + t.Length + valueLength; + fieldInfo.ValueSize = srcLogRecord.ValueSpan.Length + input.parseState.GetArgSliceByRef(0).Length; + return fieldInfo; case RespCommand.GETDEL: case RespCommand.DELIFGREATER: // Min allocation (only metadata) needed since this is going to be used for tombstoning anyway. - return sizeof(int); + return fieldInfo; + + case RespCommand.RICREATE: + case RespCommand.RIPROMOTE: + case RespCommand.RIRESTORE: + fieldInfo.ValueSize = RangeIndexManager.IndexSizeBytes; + return fieldInfo; case RespCommand.VADD: - return t.Length; + case RespCommand.VREM: + return fieldInfo; default: if (cmd > RespCommandExtensions.LastValidCommand) { var functions = functionsState.GetCustomCommandFunctions((ushort)cmd); - // compute metadata for result - metadataSize = input.arg1 switch - { - -1 => 0, - 0 => t.MetadataSize, - _ => 8, - }; - return sizeof(int) + metadataSize + functions.GetLength(t.AsReadOnlySpan(), ref input); + fieldInfo.ValueSize = functions.GetLength(srcLogRecord.ValueSpan, ref input); + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; } throw new GarnetException("Unsupported operation on input"); } } - return sizeof(int) + input.parseState.GetArgSliceByRef(0).Length + - (input.arg1 == 0 ? 0 : sizeof(long)); + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; } - public int GetUpsertValueLength(ref SpanByte t, ref RawStringInput input) + public RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref StringInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { + // We know namespaces aren't present in string functions, so don't populate + var fieldInfo = new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = value.Length, + HasETag = false + }; + switch (input.header.cmd) { case RespCommand.SET: case RespCommand.SETEX: - return input.arg1 == 0 ? t.TotalSize : sizeof(int) + t.LengthWithoutMetadata + sizeof(long); + case RespCommand.APPEND: + fieldInfo.HasExpiration = input.arg1 != 0; + break; } + return fieldInfo; + } - return t.TotalSize; + public RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref StringInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new GarnetException("String store should not be called with IHeapObject"); + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref StringInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + if (inputLogRecord.Info.ValueIsObject) + throw new GarnetException("String store should not be called with IHeapObject"); + return inputLogRecord.GetRecordFieldInfo(); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/VectorSessionFunctions.cs b/libs/server/Storage/Functions/MainStore/VectorSessionFunctions.cs deleted file mode 100644 index ca375af1717..00000000000 --- a/libs/server/Storage/Functions/MainStore/VectorSessionFunctions.cs +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Buffers.Binary; -using System.Diagnostics; -using System.Runtime.InteropServices; -using Tsavorite.core; - -namespace Garnet.server -{ - /// - /// Functions for operating against the Main Store, but for data stored as part of a Vector Set operation - not a RESP command. - /// - public readonly struct VectorSessionFunctions : ISessionFunctions - { - private readonly FunctionsState functionsState; - - /// - /// Constructor - /// - internal VectorSessionFunctions(FunctionsState functionsState) - { - this.functionsState = functionsState; - } - - #region Deletes - /// - public bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never delete a non-namespaced value with VectorSessionFunctions"); - - recordInfo.ClearHasETag(); - functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); - return true; - } - /// - public bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never delete a non-namespaced value with VectorSessionFunctions"); - - recordInfo.ClearHasETag(); - if (!deleteInfo.RecordInfo.Modified) - functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); - return true; - } - /// - public void PostSingleDeleter(ref SpanByte key, ref DeleteInfo deleteInfo) { } - - public void PostDeleteOperation(ref SpanByte key, ref DeleteInfo deleteInfo, TEpochAccessor epoch) where TEpochAccessor : IEpochAccessor { } - #endregion - - #region Reads - /// - public bool SingleReader(ref SpanByte key, ref VectorInput input, ref SpanByte value, ref SpanByte dst, ref ReadInfo readInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never read a non-namespaced value with VectorSessionFunctions"); - - unsafe - { - if (input.Callback != 0) - { - var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; - - callback(input.Index, input.CallbackContext, (nint)value.ToPointer(), (nuint)value.Length); - return true; - } - } - - if (input.ReadDesiredSize > 0) - { - Debug.Assert(dst.Length >= value.Length, "Should always have space for vector point reads"); - - dst.Length = value.Length; - value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.AsSpan()); - } - else - { - input.ReadDesiredSize = value.Length; - if (dst.Length >= value.Length) - { - value.AsReadOnlySpan(functionsState.etagState.etagSkippedStart).CopyTo(dst.AsSpan()); - dst.Length = value.Length; - } - } - - return true; - } - /// - public bool ConcurrentReader(ref SpanByte key, ref VectorInput input, ref SpanByte value, ref SpanByte dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => SingleReader(ref key, ref input, ref value, ref dst, ref readInfo); - - /// - public void ReadCompletionCallback(ref SpanByte key, ref VectorInput input, ref SpanByte output, long ctx, Status status, RecordMetadata recordMetadata) - { - } - #endregion - - #region Initial Values - /// - public bool NeedInitialUpdate(ref SpanByte key, ref VectorInput input, ref SpanByte output, ref RMWInfo rmwInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never write a non-namespaced value with VectorSessionFunctions"); - - // Only needed when updating ContextMetadata or InProgressDeletes via RMW or the DiskANN RMW callback, all of which set WriteDesiredSize - return input.WriteDesiredSize != 0; - } - /// - public bool InitialUpdater(ref SpanByte key, ref VectorInput input, ref SpanByte value, ref SpanByte output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never write a non-namespaced value with VectorSessionFunctions"); - - if (input.Callback == 0) - { - Debug.Assert(key.GetNamespaceInPayload() == 0, "Should be operating on special namespace"); - - if (key.LengthWithoutMetadata == 0) - { - // Operating on ContextMetadata - - SpanByte newMetadataValue; - unsafe - { - newMetadataValue = SpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); - } - - return SpanByteFunctions.DoSafeCopy(ref newMetadataValue, ref value, ref rmwInfo, ref recordInfo); - } - else - { - // Operating on InProgressDeletes - Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); - Debug.Assert(key.LengthWithoutMetadata == 1 && key.AsReadOnlySpan()[0] == 1, "Should be working on InProgressDeletes"); - - Span inProgressDeleteUpdateData; - bool adding; - - unsafe - { - var len = BinaryPrimitives.ReadInt32LittleEndian(new Span((byte*)input.CallbackContext + sizeof(long), sizeof(int))); - adding = len > 0; - if (!adding) - { - len = -len; - } - - inProgressDeleteUpdateData = new Span((byte*)input.CallbackContext, sizeof(ulong) + sizeof(int) + len); - } - - if (!adding) - { - // We may be recovering and doing some optimistic deletes, but since we're creating... just ignore the op, it does nothing - rmwInfo.Action = RMWAction.CancelOperation; - return false; - } - - var fits = VectorManager.TryUpdateInProgressDeletes(inProgressDeleteUpdateData, ref value, ref recordInfo, ref rmwInfo); - Debug.Assert(fits, "Initial size of record should have been correct for in progress deletes"); - - return true; - } - } - else - { - Debug.Assert(input.WriteDesiredSize <= value.LengthWithoutMetadata, "Insufficient space for initial update, this should never happen"); - - rmwInfo.ClearExtraValueLength(ref recordInfo, ref value, value.TotalSize); - - // Must explicitly 0 before passing if we're doing an initial update - value.AsSpan().Clear(); - - unsafe - { - // Callback takes: dataCallbackContext, dataPtr, dataLength - var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; - callback(input.CallbackContext, (nint)value.ToPointer(), (nuint)input.WriteDesiredSize); - - value.ShrinkSerializedLength(input.WriteDesiredSize); - value.Length = input.WriteDesiredSize; - } - - return true; - } - } - /// - public void PostInitialUpdater(ref SpanByte key, ref VectorInput input, ref SpanByte value, ref SpanByte output, ref RMWInfo rmwInfo) { } - #endregion - - #region Writes - /// - public bool SingleWriter(ref SpanByte key, ref VectorInput input, ref SpanByte src, ref SpanByte dst, ref SpanByte output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => ConcurrentWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, ref recordInfo); - - /// - public void PostSingleWriter(ref SpanByte key, ref VectorInput input, ref SpanByte src, ref SpanByte dst, ref SpanByte output, ref UpsertInfo upsertInfo, WriteReason reason) { } - /// - public bool ConcurrentWriter(ref SpanByte key, ref VectorInput input, ref SpanByte src, ref SpanByte dst, ref SpanByte output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never write a non-namespaced value with VectorSessionFunctions"); - - return SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo, 0); - } - - public void PostUpsertOperation(ref SpanByte key, ref VectorInput input, ref SpanByte src, ref UpsertInfo upsertInfo, TEpochAccessor epoch) where TEpochAccessor : IEpochAccessor { } - - #endregion - - #region RMW - /// - public int GetRMWInitialValueLength(ref VectorInput input) - { - var effectiveWriteDesiredSize = input.WriteDesiredSize; - - if (effectiveWriteDesiredSize < 0) - { - effectiveWriteDesiredSize = -effectiveWriteDesiredSize; - } - - return sizeof(int) + effectiveWriteDesiredSize; - } - /// - public int GetRMWModifiedValueLength(ref SpanByte value, ref VectorInput input) - { - if (input.WriteDesiredSize < 0) - { - // Add to value, this is a dynamically sized type - return value.Length + (-input.WriteDesiredSize); - } - - // Constant size indicated - return sizeof(int) + input.WriteDesiredSize; - } - - /// - public int GetUpsertValueLength(ref SpanByte value, ref VectorInput input) - => sizeof(int) + value.Length; - - /// - public bool InPlaceUpdater(ref SpanByte key, ref VectorInput input, ref SpanByte value, ref SpanByte output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never write a non-namespaced value with VectorSessionFunctions"); - - if (input.Callback == 0) - { - // We're doing a Metadata or InProgressDelete update - - Debug.Assert(key.GetNamespaceInPayload() == 0, "Should be operating on special namespace"); - - if (key.LengthWithoutMetadata == 0) - { - // Doing a Metadata update - Debug.Assert(value.LengthWithoutMetadata == VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); - Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); - - ref readonly var oldMetadata = ref MemoryMarshal.Cast(value.AsReadOnlySpan())[0]; - - SpanByte newMetadataValue; - unsafe - { - newMetadataValue = SpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); - } - - ref readonly var newMetadata = ref MemoryMarshal.Cast(newMetadataValue.AsReadOnlySpan())[0]; - - if (newMetadata.Version < oldMetadata.Version) - { - rmwInfo.Action = RMWAction.CancelOperation; - return false; - } - - return SpanByteFunctions.DoSafeCopy(ref newMetadataValue, ref value, ref rmwInfo, ref recordInfo); - } - else - { - // Doing an InProgressDelete update - Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); - Debug.Assert(key.LengthWithoutMetadata == 1 && key.AsReadOnlySpan()[0] == 1, "Should be working on InProgressDeletes"); - - Span inProgressDeleteUpdateData; - bool adding; - - unsafe - { - var len = BinaryPrimitives.ReadInt32LittleEndian(new Span(((byte*)input.CallbackContext + sizeof(long)), sizeof(int))); - adding = len > 0; - if (!adding) - { - len = -len; - } - - inProgressDeleteUpdateData = new Span((byte*)input.CallbackContext, sizeof(ulong) + sizeof(int) + len); - } - - return VectorManager.TryUpdateInProgressDeletes(inProgressDeleteUpdateData, ref value, ref recordInfo, ref rmwInfo); - } - } - else - { - Debug.Assert(input.WriteDesiredSize <= value.LengthWithoutMetadata, "Insufficient space for inplace update, this should never happen"); - - unsafe - { - // Callback takes: dataCallbackContext, dataPtr, dataLength - var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; - callback(input.CallbackContext, (nint)value.ToPointer(), (nuint)input.WriteDesiredSize); - } - - return true; - } - } - - /// - public bool NeedCopyUpdate(ref SpanByte key, ref VectorInput input, ref SpanByte oldValue, ref SpanByte output, ref RMWInfo rmwInfo) - => input.WriteDesiredSize != 0; - - /// - public bool CopyUpdater(ref SpanByte key, ref VectorInput input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByte output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - Debug.Assert(key.MetadataSize == 1, "Should never write a non-namespaced value with VectorSessionFunctions"); - - if (input.Callback == 0) - { - // We're doing a Metadata or InProgressDelete update - - Debug.Assert(key.GetNamespaceInPayload() == 0, "Should be operating on special namespace"); - - if (key.LengthWithoutMetadata == 0) - { - // Doing a Metadata update - Debug.Assert(oldValue.LengthWithoutMetadata == VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); - Debug.Assert(newValue.LengthWithoutMetadata == VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); - Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); - - ref readonly var oldMetadata = ref MemoryMarshal.Cast(oldValue.AsReadOnlySpan())[0]; - - SpanByte newMetadataValue; - unsafe - { - newMetadataValue = SpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); - } - - ref readonly var newMetadata = ref MemoryMarshal.Cast(newMetadataValue.AsReadOnlySpan())[0]; - - if (newMetadata.Version < oldMetadata.Version) - { - rmwInfo.Action = RMWAction.CancelOperation; - return false; - } - - return SpanByteFunctions.DoSafeCopy(ref newMetadataValue, ref newValue, ref rmwInfo, ref recordInfo); - } - else - { - // Doing an InProgressDelete update - Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); - Debug.Assert(key.LengthWithoutMetadata == 1 && key.AsReadOnlySpan()[0] == 1, "Should be working on InProgressDeletes"); - - Span inProgressDeleteUpdateData; - bool adding; - - oldValue.CopyTo(ref newValue); - - unsafe - { - var len = BinaryPrimitives.ReadInt32LittleEndian(new Span(((byte*)input.CallbackContext + sizeof(long)), sizeof(int))); - adding = len > 0; - if (!adding) - { - len = -len; - } - - inProgressDeleteUpdateData = new Span((byte*)input.CallbackContext, sizeof(ulong) + sizeof(int) + len); - } - - var fits = VectorManager.TryUpdateInProgressDeletes(inProgressDeleteUpdateData, ref newValue, ref recordInfo, ref rmwInfo); - Debug.Assert(fits, "Copy update should have allocated enough space for in progress deletes"); - - return true; - } - } - else - { - Debug.Assert(input.WriteDesiredSize <= newValue.LengthWithoutMetadata, "Insufficient space for copy update, this should never happen"); - Debug.Assert(input.WriteDesiredSize <= oldValue.LengthWithoutMetadata, "Insufficient space for copy update, this should never happen"); - - oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - - unsafe - { - // Callback takes: dataCallbackContext, dataPtr, dataLength - var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; - callback(input.CallbackContext, (nint)newValue.ToPointer(), (nuint)input.WriteDesiredSize); - } - - return true; - } - } - - /// - public bool PostCopyUpdater(ref SpanByte key, ref VectorInput input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByte output, ref RMWInfo rmwInfo) - => true; - /// - public void RMWCompletionCallback(ref SpanByte key, ref VectorInput input, ref SpanByte output, long ctx, Status status, RecordMetadata recordMetadata) { } - - public void PostRMWOperation(ref SpanByte key, ref VectorInput input, ref RMWInfo rmwInfo, TEpochAccessor epoch) where TEpochAccessor : IEpochAccessor { } - #endregion - - #region Utilities - /// - public void ConvertOutputToHeap(ref VectorInput input, ref SpanByte output) { } - #endregion - } -} \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/CallbackMethods.cs b/libs/server/Storage/Functions/ObjectStore/CallbackMethods.cs index 3765fb0bdb9..72a917fe11e 100644 --- a/libs/server/Storage/Functions/ObjectStore/CallbackMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/CallbackMethods.cs @@ -8,15 +8,15 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions { /// - public void ReadCompletionCallback(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, long ctx, Status status, RecordMetadata recordMetadata) + public void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref ObjectInput input, ref ObjectOutput output, long ctx, Status status, RecordMetadata recordMetadata) { } /// - public void RMWCompletionCallback(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, long ctx, Status status, RecordMetadata recordMetadata) + public void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref ObjectInput input, ref ObjectOutput output, long ctx, Status status, RecordMetadata recordMetadata) { } } diff --git a/libs/server/Storage/Functions/ObjectStore/DeleteMethods.cs b/libs/server/Storage/Functions/ObjectStore/DeleteMethods.cs index 73a7a5b6a0f..8c78d62c841 100644 --- a/libs/server/Storage/Functions/ObjectStore/DeleteMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/DeleteMethods.cs @@ -8,41 +8,45 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly partial struct ObjectSessionFunctions : ISessionFunctions { /// - public bool SingleDeleter(ref byte[] key, ref IGarnetObject value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) => true; /// - public void PostSingleDeleter(ref byte[] key, ref DeleteInfo deleteInfo) + public void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - if (!deleteInfo.RecordInfo.Modified) + if (!logRecord.Info.Modified) functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); if (functionsState.appendOnlyFile != null) deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF } /// - public bool ConcurrentDeleter(ref byte[] key, ref IGarnetObject value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - if (!deleteInfo.RecordInfo.Modified) + if (!logRecord.Info.Modified) functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); if (functionsState.appendOnlyFile != null) deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF - functionsState.objectStoreSizeTracker?.AddTrackedSize(-value.Size); - value = null; + + // Heap object cache-size tracking and disposal are handled by + // storeFunctions.OnDispose (GarnetRecordTriggers) which is called + // by Tsavorite after InPlaceDeleter returns. return true; } /// - public void PostDeleteOperation(ref byte[] key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + public void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((deleteInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogDelete(ref key, deleteInfo.Version, deleteInfo.SessionID, epochAccessor); - } + WriteLogDelete(key.KeyBytes, deleteInfo.Version, deleteInfo.SessionID, epochAccessor); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/ObjectSessionFunctions.cs b/libs/server/Storage/Functions/ObjectStore/ObjectSessionFunctions.cs index 34d78f6c199..89850eb372e 100644 --- a/libs/server/Storage/Functions/ObjectStore/ObjectSessionFunctions.cs +++ b/libs/server/Storage/Functions/ObjectStore/ObjectSessionFunctions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using Tsavorite.core; namespace Garnet.server @@ -8,24 +9,44 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly partial struct ObjectSessionFunctions : ISessionFunctions { const byte NeedAofLog = 0x1; readonly FunctionsState functionsState; + readonly ReadSessionState readSessionState; /// /// Constructor /// - internal ObjectSessionFunctions(FunctionsState functionsState) + /// + /// + internal ObjectSessionFunctions(FunctionsState functionsState, ReadSessionState readSessionState = null) { this.functionsState = functionsState; + this.readSessionState = readSessionState; } /// - public void ConvertOutputToHeap(ref ObjectInput input, ref GarnetObjectStoreOutput output) + public void ConvertOutputToHeap(ref ObjectInput input, ref ObjectOutput output) { // TODO: Inspect input to determine whether we're in a context requiring ConvertToHeap. //output.ConvertToHeap(); } + + /// + public void BeforeConsistentReadCallback(long hash) + => readSessionState?.BeforeConsistentReadKeyCallback(hash); + + /// + public void AfterConsistentReadKeyCallback() + => readSessionState?.AfterConsistentReadKeyCallback(); + + /// + public void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) + => readSessionState?.BeforeConsistentReadKeyBatch(parameters); + + /// + public bool AfterConsistentReadKeyBatchCallback(int keyCount) + => readSessionState != null && readSessionState.AfterConsistentReadKeyBatch(keyCount); } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs index 44b7e82d1b4..55fd018ad8a 100644 --- a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs @@ -2,10 +2,7 @@ // Licensed under the MIT license. using System; -using System.Buffers; -using System.Diagnostics; using System.Runtime.CompilerServices; -using Garnet.common; using Tsavorite.core; namespace Garnet.server @@ -13,31 +10,55 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions { /// /// Logging upsert from - /// a. ConcurrentWriter - /// b. PostSingleWriter + /// a. InPlaceWriter + /// b. PostInitialWriter /// - void WriteLogUpsert(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, long version, int sessionID, TEpochAccessor epochAccessor) + void WriteLogUpsert(ReadOnlySpan key, ref ObjectInput input, ReadOnlySpan value, long version, int sessionID, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { - if (functionsState.StoredProcMode) return; + if (functionsState.StoredProcMode) + return; input.header.flags |= RespInputFlags.Deterministic; - var valueBytes = GarnetObjectSerializer.Serialize(value); - fixed (byte* ptr = key) - { - fixed (byte* valPtr = valueBytes) - { - var keySB = SpanByte.FromPinnedPointer(ptr, key.Length); - var valSB = SpanByte.FromPinnedPointer(valPtr, valueBytes.Length); + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.ObjectStoreUpsert, + version, + sessionID, + key, + value, + ref input, + epochAccessor, + out _); + } - functionsState.appendOnlyFile.Enqueue( - new AofHeader { opType = AofEntryType.ObjectStoreUpsert, storeVersion = version, sessionID = sessionID }, - ref keySB, ref valSB, epochAccessor, out _); - } + /// + /// Logging upsert from + /// a. InPlaceWriter + /// b. PostInitialWriter + /// + void WriteLogUpsert(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, long version, int sessionID, TEpochAccessor epochAccessor) + where TEpochAccessor : IEpochAccessor + { + if (functionsState.StoredProcMode) + return; + input.header.flags |= RespInputFlags.Deterministic; + + GarnetObjectSerializer.Serialize(value, out var valueBytes); + fixed (byte* valPtr = valueBytes) + { + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.ObjectStoreUpsert, + version, + sessionID, + key, + new ReadOnlySpan(valPtr, valueBytes.Length), + ref input, + epochAccessor, + out _); } } @@ -47,143 +68,42 @@ void WriteLogUpsert(ref byte[] key, ref ObjectInput input, ref I /// b. InPlaceUpdater /// c. PostCopyUpdater /// - void WriteLogRMW(ref byte[] key, ref ObjectInput input, long version, int sessionID, TEpochAccessor epochAccessor) + void WriteLogRMW(ReadOnlySpan key, ref ObjectInput input, long version, int sessionID, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { if (functionsState.StoredProcMode) return; input.header.flags |= RespInputFlags.Deterministic; - // Serializing key & ObjectInput to RMW log - fixed (byte* keyPtr = key) - { - var sbKey = SpanByte.FromPinnedPointer(keyPtr, key.Length); - - functionsState.appendOnlyFile.Enqueue( - new AofHeader { opType = AofEntryType.ObjectStoreRMW, storeVersion = version, sessionID = sessionID }, - ref sbKey, ref input, epochAccessor, out _); - } + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.ObjectStoreRMW, + version, + sessionID, + key, + ref input, + epochAccessor, + out _); } /// /// Logging Delete from - /// a. ConcurrentDeleter - /// b. PostSingleDeleter + /// a. InPlaceDeleter + /// b. PostInitialDeleter /// - void WriteLogDelete(ref byte[] key, long version, int sessionID, TEpochAccessor epochAccessor) + void WriteLogDelete(ReadOnlySpan key, long version, int sessionID, TEpochAccessor epochAccessor) where TEpochAccessor : IEpochAccessor { - if (functionsState.StoredProcMode) return; - fixed (byte* ptr = key) - { - var keySB = SpanByte.FromPinnedPointer(ptr, key.Length); - SpanByte valSB = default; - - functionsState.appendOnlyFile.Enqueue( - new AofHeader { opType = AofEntryType.ObjectStoreDelete, storeVersion = version, sessionID = sessionID }, - ref keySB, ref valSB, epochAccessor, out _); - } - } - internal static bool CheckExpiry(IGarnetObject src) => src.Expiration < DateTimeOffset.UtcNow.Ticks; - - static void CopyRespNumber(long number, ref SpanByteAndMemory dst) - { - byte* curr = dst.SpanByte.ToPointer(); - byte* end = curr + dst.SpanByte.Length; - if (RespWriteUtils.TryWriteInt64(number, ref curr, end, out var integerLen, out int totalLen)) - { - dst.SpanByte.Length = (int)(curr - dst.SpanByte.ToPointer()); - return; - } - - //handle resp buffer overflow here - dst.ConvertToHeap(); - dst.Length = totalLen; - dst.Memory = MemoryPool.Shared.Rent(totalLen); - fixed (byte* ptr = dst.Memory.Memory.Span) - { - byte* cc = ptr; - *cc++ = (byte)':'; - NumUtils.WriteInt64(number, (int)integerLen, ref cc); - *cc++ = (byte)'\r'; - *cc++ = (byte)'\n'; - } - } - - static void CopyDefaultResp(ReadOnlySpan resp, ref SpanByteAndMemory dst) - { - if (resp.Length < dst.SpanByte.Length) - { - resp.CopyTo(dst.SpanByte.AsSpan()); - dst.SpanByte.Length = resp.Length; + if (functionsState.StoredProcMode) return; - } - - dst.ConvertToHeap(); - dst.Length = resp.Length; - dst.Memory = MemoryPool.Shared.Rent(resp.Length); - resp.CopyTo(dst.Memory.Memory.Span); - } - static bool EvaluateObjectExpireInPlace(ExpireOption optionType, bool expiryExists, long expiration, ref IGarnetObject value, ref GarnetObjectStoreOutput output) - { - Debug.Assert(output.SpanByteAndMemory.IsSpanByte, "This code assumes it is called in-place and did not go pending"); - var o = (ObjectOutputHeader*)output.SpanByteAndMemory.SpanByte.ToPointer(); - if (expiryExists) - { - switch (optionType) - { - case ExpireOption.NX: - o->result1 = 0; - break; - case ExpireOption.XX: - case ExpireOption.None: - value.Expiration = expiration; - o->result1 = 1; - break; - case ExpireOption.GT: - case ExpireOption.XXGT: - bool replace = expiration < value.Expiration; - value.Expiration = replace ? value.Expiration : expiration; - if (replace) - o->result1 = 0; - else - o->result1 = 1; - break; - case ExpireOption.LT: - case ExpireOption.XXLT: - replace = expiration > value.Expiration; - value.Expiration = replace ? value.Expiration : expiration; - if (replace) - o->result1 = 0; - else - o->result1 = 1; - break; - default: - throw new GarnetException($"EvaluateObjectExpireInPlace exception expiryExists:{expiryExists}, optionType{optionType}"); - } - } - else - { - switch (optionType) - { - case ExpireOption.NX: - case ExpireOption.None: - case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite - value.Expiration = expiration; - o->result1 = 1; - break; - case ExpireOption.XX: - case ExpireOption.GT: - case ExpireOption.XXGT: - case ExpireOption.XXLT: - o->result1 = 0; - break; - default: - throw new GarnetException($"EvaluateObjectExpireInPlace exception expiryExists:{expiryExists}, optionType{optionType}"); - } - } - return true; + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.ObjectStoreDelete, + version, + sessionID, + key, + value: default, + epochAccessor, + out _); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -195,7 +115,7 @@ private CustomObjectFunctions GetCustomObjectCommand(ref ObjectInput input, Garn } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private unsafe bool IncorrectObjectType(ref ObjectInput input, IGarnetObject value, ref SpanByteAndMemory output) + private bool IncorrectObjectType(ref ObjectInput input, IGarnetObject value, ref SpanByteAndMemory output) { var inputType = (byte)input.header.type; if (inputType != value.Type) // Indicates an incorrect type of key diff --git a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs index b25fe0cae6f..0fdd0fa18dc 100644 --- a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System.Diagnostics; +using System.Runtime.CompilerServices; using Garnet.common; using Tsavorite.core; @@ -10,72 +11,70 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly partial struct ObjectSessionFunctions : ISessionFunctions { /// - public bool NeedInitialUpdate(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo) + public bool NeedInitialUpdate(TKey key, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { var type = input.header.type; - switch (type) - { - case GarnetObjectType.Expire: - case GarnetObjectType.Persist: - case GarnetObjectType.DelIfExpIm: - return false; - default: - if ((byte)type < CustomCommandManager.CustomTypeIdStartOffset) - return GarnetObject.NeedToCreate(input.header); - else - { - var customObjectCommand = GetCustomObjectCommand(ref input, type); + if ((byte)type < CustomCommandManager.CustomTypeIdStartOffset) + return GarnetObject.NeedToCreate(input.header); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); - try - { - var ret = customObjectCommand.NeedInitialUpdate(key, ref input, ref writer); - return ret; - } - finally - { - writer.Dispose(); - } - } + var customObjectCommand = GetCustomObjectCommand(ref input, type); + + // TODO: Cannot use 'using' statement because writer is passed by ref. Change that to non-ref parameter and convert to 'using'. + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + try + { + // Deliberately hiding key type complexity from custom object commands + return customObjectCommand.NeedInitialUpdate(key.KeyBytes, ref input, ref writer); + } + finally + { + writer.Dispose(); } } /// - public bool InitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) { + Debug.Assert(!logRecord.Info.HasETag && !logRecord.Info.HasExpiration, "Should not have Expiration or ETag on InitialUpdater log records"); + var type = input.header.type; + IGarnetObject value; if ((byte)type < CustomCommandManager.CustomTypeIdStartOffset) { value = GarnetObject.Create(type); - value.Operate(ref input, ref output, functionsState.respProtocolVersion, out _); + _ = value.Operate(ref input, ref output, functionsState.respProtocolVersion); + _ = logRecord.TrySetValueObjectAndPrepareOptionals(value, in sizeInfo); return true; } - else - { - Debug.Assert(type != GarnetObjectType.Expire && type != GarnetObjectType.Persist, "Expire and Persist commands should have been handled already by NeedInitialUpdate."); - var customObjectCommand = GetCustomObjectCommand(ref input, type); - value = functionsState.GetCustomObjectFactory((byte)type).Create((byte)type); + var customObjectCommand = GetCustomObjectCommand(ref input, type); + value = functionsState.GetCustomObjectFactory((byte)type).Create((byte)type); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); - try - { - var result = customObjectCommand.InitialUpdater(key, ref input, value, ref writer, ref rmwInfo); - return result; - } - finally - { - writer.Dispose(); - } + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + try + { + var result = customObjectCommand.InitialUpdater(logRecord.Key, ref input, value, ref writer, ref rmwInfo); + _ = logRecord.TrySetValueObjectAndPrepareOptionals(value, in sizeInfo); + if (result) + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + return result; + } + finally + { + writer.Dispose(); } } /// - public void PostInitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo) + public void PostInitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) { functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); if (functionsState.appendOnlyFile != null) @@ -83,215 +82,172 @@ public void PostInitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarne input.header.SetExpiredFlag(); rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF } - - functionsState.objectStoreSizeTracker?.AddTrackedSize(MemoryUtils.CalculateKeyValueSize(key, value)); } /// - public bool InPlaceUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public bool InPlaceUpdater(ref LogRecord logRecord, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) { - if (InPlaceUpdaterWorker(ref key, ref input, ref value, ref output, ref rmwInfo, out var sizeChange)) + if (!logRecord.Info.ValueIsObject) { - if (!rmwInfo.RecordInfo.Modified) + rmwInfo.Action = RMWAction.WrongType; + output.OutputFlags |= ObjectOutputFlags.WrongType; + return true; + } + + if (InPlaceUpdaterWorker(ref logRecord, ref input, ref output, ref rmwInfo)) + { + if (!logRecord.Info.Modified) functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); if (functionsState.appendOnlyFile != null) rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF - functionsState.objectStoreSizeTracker?.AddTrackedSize(sizeChange); return true; } return false; } - bool InPlaceUpdaterWorker(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, out long sizeChange) + bool InPlaceUpdaterWorker(ref LogRecord logRecord, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) { - sizeChange = 0; - // Expired data - if (value.Expiration > 0 && input.header.CheckExpiry(value.Expiration)) + if (logRecord.Info.HasExpiration && input.header.CheckExpiry(logRecord.Expiration)) { - functionsState.objectStoreSizeTracker?.AddTrackedSize(-value.Size); - value = null; - rmwInfo.Action = input.header.type == GarnetObjectType.DelIfExpIm ? RMWAction.ExpireAndStop : RMWAction.ExpireAndResume; + rmwInfo.Action = RMWAction.ExpireAndResume; return false; } - switch (input.header.type) + if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) { - case GarnetObjectType.Expire: - var expiryExists = value.Expiration > 0; - - var expirationWithOption = new ExpirationWithOption(input.arg1, input.arg2); - - return EvaluateObjectExpireInPlace(expirationWithOption.ExpireOption, expiryExists, expirationWithOption.ExpirationTimeInTicks, ref value, ref output); - case GarnetObjectType.Persist: - if (value.Expiration > 0) - { - value.Expiration = 0; - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); - } - else - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); + var operateSuccessful = ((IGarnetObject)logRecord.ValueObject).Operate(ref input, ref output, functionsState.respProtocolVersion); + if (output.HasWrongType) return true; - case GarnetObjectType.DelIfExpIm: - return true; - default: - if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) - { - var operateSuccessful = value.Operate(ref input, ref output, functionsState.respProtocolVersion, out sizeChange); - if (output.HasWrongType) - return true; - - if (output.HasRemoveKey) - { - functionsState.objectStoreSizeTracker?.AddTrackedSize(-value.Size); - value = null; - if (!rmwInfo.RecordInfo.Modified) - functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); - if (functionsState.appendOnlyFile != null) - rmwInfo.UserData |= NeedAofLog; - rmwInfo.Action = RMWAction.ExpireAndStop; - return false; - } + if (output.HasRemoveKey) + { + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + rmwInfo.UserData |= NeedAofLog; + rmwInfo.Action = RMWAction.ExpireAndStop; + return false; + } - return operateSuccessful; - } - else - { - if (IncorrectObjectType(ref input, value, ref output.SpanByteAndMemory)) - { - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; - return true; - } + return operateSuccessful; + } - var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); - try - { - var result = customObjectCommand.Updater(key, ref input, value, ref writer, ref rmwInfo); - return result; - //return customObjectCommand.InPlaceUpdateWorker(key, ref input, value, ref output.spanByteAndMemory, ref rmwInfo); - } - finally - { - writer.Dispose(); - } - } + var garnetValueObject = Unsafe.As(logRecord.ValueObject); + if (IncorrectObjectType(ref input, garnetValueObject, ref output.SpanByteAndMemory)) + { + output.OutputFlags |= ObjectOutputFlags.WrongType; + return true; } - } - /// - public bool NeedCopyUpdate(ref byte[] key, ref ObjectInput input, ref IGarnetObject oldValue, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo) - { - if (input.header.type == GarnetObjectType.DelIfExpIm && oldValue.Expiration > 0 && input.header.CheckExpiry(oldValue.Expiration)) + var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + try { - rmwInfo.Action = RMWAction.ExpireAndStop; - return false; + var result = customObjectCommand.Updater(logRecord.Key, ref input, garnetValueObject, ref writer, ref rmwInfo); + if (!result) + return false; + } + finally + { + writer.Dispose(); } return true; } /// - public bool CopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject oldValue, ref IGarnetObject newValue, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref ObjectInput input, + ref ObjectOutput output, ref RMWInfo rmwInfo) where TSourceLogRecord : ISourceLogRecord + => true; + + /// + public bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { // Expired data - if (oldValue.Expiration > 0 && input.header.CheckExpiry(oldValue.Expiration)) + if (srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) { rmwInfo.Action = RMWAction.ExpireAndResume; return false; } + // Defer the actual copying of data to PostCopyUpdater, so we know the record has been successfully CASed into the hash chain before we potentially + // create large allocations (e.g. if srcLogRecord is from disk, we would have to allocate the overflow byte[]). Because we are doing an update we have + // and XLock, so nobody will see the unset data even after the CAS. Tsavorite will handle cloning the ValueObject and caching serialized data as needed, + // based on whether srcLogRecord is in-memory or a DiskLogRecord. return true; } /// - public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject oldValue, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo) + public bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ref ObjectOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { - // We're performing the object update here (and not in CopyUpdater) so that we are guaranteed that - // the record was CASed into the hash chain before it gets modified - var oldValueSize = oldValue.Size; - oldValue.CopyUpdate(ref oldValue, ref value, rmwInfo.RecordInfo.IsInNewVersion); + // We perform the object update here (and not in CopyUpdater) so that we are guaranteed that the record + // was CASed into the hash chain before it gets modified. Tsavorite's CacheSerializedObjectData (called + // by InternalRMW right before PCU, for both memory and disk sources) has already cloned src.ValueObject + // into dstLogRecord. Reuse it directly — do not clone again. + var value = Unsafe.As(dstLogRecord.ValueObject); + + // Do not set actually set dstLogRecord.Expiration until we know it is a command for which we allocated length in the LogRecord for it. + // TODO: Object store ETags functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); - switch (input.header.type) + if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) { - case GarnetObjectType.Expire: - var expiryExists = value.Expiration > 0; - - var expirationWithOption = new ExpirationWithOption(input.arg1, input.arg2); - - EvaluateObjectExpireInPlace(expirationWithOption.ExpireOption, expiryExists, expirationWithOption.ExpirationTimeInTicks, ref value, ref output); - break; - case GarnetObjectType.Persist: - if (value.Expiration > 0) - { - value.Expiration = 0; - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); - } - else - CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); - break; - case GarnetObjectType.DelIfExpIm: - break; - default: - if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) - { - value.Operate(ref input, ref output, functionsState.respProtocolVersion, out _); - if (output.HasWrongType) - return true; - - if (output.HasRemoveKey) - { - // Log to AOF before returning, so the mutation that emptied the collection - // is persisted and replayed correctly on recovery. - if (functionsState.appendOnlyFile != null) - rmwInfo.UserData |= NeedAofLog; - rmwInfo.Action = RMWAction.ExpireAndStop; - return false; - } - break; - } - else - { - // TODO: Update to invoke CopyUpdater of custom object command without creating a new object - // using Clone. Currently, expire and persist commands are performed on the new copy of the object. - if (IncorrectObjectType(ref input, value, ref output.SpanByteAndMemory)) - { - output.OutputFlags |= ObjectStoreOutputFlags.WrongType; - return true; - } + value.Operate(ref input, ref output, functionsState.respProtocolVersion); + if (output.HasWrongType) + return true; + if (output.HasRemoveKey) + { + // Log to AOF before returning, so the mutation that emptied the collection + // is persisted and replayed correctly on recovery. + if (functionsState.appendOnlyFile != null) + rmwInfo.UserData |= NeedAofLog; + rmwInfo.Action = RMWAction.ExpireAndStop; + return false; + } + } + else + { + // TODO: Update to invoke CopyUpdater of custom object command without creating a new object + // using Clone. Currently, expire and persist commands are performed on the new copy of the object. + if (IncorrectObjectType(ref input, value, ref output.SpanByteAndMemory)) + { + output.OutputFlags |= ObjectOutputFlags.WrongType; + return true; + } - var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); - try - { - var result = customObjectCommand.Updater(key, ref input, value, ref writer, ref rmwInfo); - return result; - } - finally - { - writer.Dispose(); - } - } + var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + try + { + var result = customObjectCommand.Updater(srcLogRecord.Key, ref input, value, ref writer, ref rmwInfo); + return result; + } + finally + { + writer.Dispose(); + } } - // If oldValue has been set to null, subtract it's size from the tracked heap size - var sizeAdjustment = oldValue == null ? value.Size - oldValueSize : value.Size; - functionsState.objectStoreSizeTracker?.AddTrackedSize(sizeAdjustment); + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); if (functionsState.appendOnlyFile != null) rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF return true; } + /// - public void PostRMWOperation(ref byte[] key, ref ObjectInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + public void PostRMWOperation(TKey key, ref ObjectInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((rmwInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogRMW(ref key, ref input, rmwInfo.Version, rmwInfo.SessionID, epochAccessor); - } + WriteLogRMW(key.KeyBytes, ref input, rmwInfo.Version, rmwInfo.SessionID, epochAccessor); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs index bdd6def893b..952df02484f 100644 --- a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs @@ -10,12 +10,19 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions { /// - public bool SingleReader(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput dst, ref ReadInfo readInfo) + public bool Reader(in TSourceLogRecord srcLogRecord, ref ObjectInput input, ref ObjectOutput output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord { - if (value.Expiration > 0 && value.Expiration < DateTimeOffset.Now.UtcTicks) + if (!srcLogRecord.Info.ValueIsObject) + { + readInfo.Action = ReadAction.WrongType; + return false; + } + + if (srcLogRecord.Info.HasExpiration && srcLogRecord.Expiration < DateTimeOffset.Now.UtcTicks) { // Do not set 'value = null' or otherwise mark this; Reads should not update the database. We rely on consistently checking for expiration everywhere. readInfo.Action = ReadAction.Expire; @@ -24,63 +31,37 @@ public bool SingleReader(ref byte[] key, ref ObjectInput input, ref IGarnetObjec if (input.header.type != 0) { - switch (input.header.type) + var garnetObject = (IGarnetObject)srcLogRecord.ValueObject; + if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) { - case GarnetObjectType.Ttl: - var ttlValue = ConvertUtils.SecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1); - CopyRespNumber(ttlValue, ref dst.SpanByteAndMemory); - return true; - case GarnetObjectType.PTtl: - ttlValue = ConvertUtils.MillisecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1); - CopyRespNumber(ttlValue, ref dst.SpanByteAndMemory); + var opResult = garnetObject.Operate(ref input, ref output, functionsState.respProtocolVersion); + if (output.HasWrongType) return true; - case GarnetObjectType.ExpireTime: - var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(value.Expiration > 0 ? value.Expiration : -1); - CopyRespNumber(expireTime, ref dst.SpanByteAndMemory); - return true; - case GarnetObjectType.PExpireTime: - expireTime = ConvertUtils.UnixTimeInMillisecondsFromTicks(value.Expiration > 0 ? value.Expiration : -1); - CopyRespNumber(expireTime, ref dst.SpanByteAndMemory); - return true; - - default: - if ((byte)input.header.type < CustomCommandManager.CustomTypeIdStartOffset) - { - var opResult = value.Operate(ref input, ref dst, functionsState.respProtocolVersion, out _); - if (dst.HasWrongType) - return true; - - return opResult; - } - - if (IncorrectObjectType(ref input, value, ref dst.SpanByteAndMemory)) - { - dst.OutputFlags |= ObjectStoreOutputFlags.WrongType; - return true; - } + return opResult; + } - var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref dst.SpanByteAndMemory); - try - { - var result = customObjectCommand.Reader(key, ref input, value, ref writer, ref readInfo); - return result; - } - finally - { - writer.Dispose(); - } + if (IncorrectObjectType(ref input, garnetObject, ref output.SpanByteAndMemory)) + { + output.OutputFlags |= ObjectOutputFlags.WrongType; + return true; + } + var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + try + { + var result = customObjectCommand.Reader(srcLogRecord.Key, ref input, garnetObject, ref writer, ref readInfo); + return result; + } + finally + { + writer.Dispose(); } } - dst.GarnetObject = value; + output.GarnetObject = (IGarnetObject)srcLogRecord.ValueObject; return true; } - - /// - public bool ConcurrentReader(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => SingleReader(ref key, ref input, ref value, ref dst, ref readInfo); } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/UpsertMethods.cs b/libs/server/Storage/Functions/ObjectStore/UpsertMethods.cs index f63ba333c8e..b72fb42ef56 100644 --- a/libs/server/Storage/Functions/ObjectStore/UpsertMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/UpsertMethods.cs @@ -1,56 +1,132 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Runtime.CompilerServices; using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; namespace Garnet.server { /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly partial struct ObjectSessionFunctions : ISessionFunctions { /// - public bool SingleWriter(ref byte[] key, ref ObjectInput input, ref IGarnetObject src, ref IGarnetObject dst, ref GarnetObjectStoreOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ReadOnlySpan srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) { - dst = src; + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + // TODO ETag + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); return true; } /// - public void PostSingleWriter(ref byte[] key, ref ObjectInput input, ref IGarnetObject src, ref IGarnetObject dst, ref GarnetObjectStoreOutput output, ref UpsertInfo upsertInfo, WriteReason reason) + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, IHeapObject srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) { - if (reason != WriteReason.CopyToTail) - functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); - if (reason == WriteReason.Upsert && functionsState.appendOnlyFile != null) + if (!dstLogRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + // TODO ETag + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + /// + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, in TSourceLogRecord inputLogRecord, ref ObjectOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, ReadOnlySpan srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) + { + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF - if (reason == WriteReason.CopyToReadCache) - functionsState.objectStoreSizeTracker?.AddReadCacheTrackedSize(MemoryUtils.CalculateKeyValueSize(key, src)); - else - functionsState.objectStoreSizeTracker?.AddTrackedSize(MemoryUtils.CalculateKeyValueSize(key, src)); } /// - public bool ConcurrentWriter(ref byte[] key, ref ObjectInput input, ref IGarnetObject src, ref IGarnetObject dst, ref GarnetObjectStoreOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, IHeapObject srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) + { + var garnetObject = (IGarnetObject)srcValue; + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + } + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ObjectInput input, in TSourceLogRecord inputLogRecord, ref ObjectOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + { + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + } + } + + /// + public bool InPlaceWriter(ref LogRecord logRecord, ref ObjectInput input, ReadOnlySpan srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) { - dst = src; - if (!upsertInfo.RecordInfo.Modified) - functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (!InPlaceWriterForSpanValue(ref logRecord, ref input, srcValue, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; if (functionsState.appendOnlyFile != null) upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF - functionsState.objectStoreSizeTracker?.AddTrackedSize(dst.Size - src.Size); return true; } /// - public void PostUpsertOperation(ref byte[] key, ref ObjectInput input, ref IGarnetObject src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref ObjectInput input, IHeapObject srcValue, ref ObjectOutput output, ref UpsertInfo upsertInfo) + { + if (!InPlaceWriterForHeapObjectValue(ref logRecord, ref input, srcValue, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref ObjectInput input, in TSourceLogRecord inputLogRecord, ref ObjectOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (!InPlaceWriterForLogRecordValue(ref logRecord, ref input, in inputLogRecord, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + public void PostUpsertOperation(TKey key, ref ObjectInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { if ((upsertInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF - { - WriteLogUpsert(ref key, ref input, ref src, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); - } + WriteLogUpsert(key.KeyBytes, ref input, valueSpan, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); + } + + /// + public void PostUpsertOperation(TKey key, ref ObjectInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + if ((upsertInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF + WriteLogUpsert(key.KeyBytes, ref input, (IGarnetObject)valueObject, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs index 63af04149be..4067054eff8 100644 --- a/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/VarLenInputMethods.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Garnet.common; +using System; using Tsavorite.core; namespace Garnet.server @@ -9,23 +9,91 @@ namespace Garnet.server /// /// Object store functions /// - public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions + public readonly unsafe partial struct ObjectSessionFunctions : ISessionFunctions { /// - public int GetRMWModifiedValueLength(ref IGarnetObject value, ref ObjectInput input) + public RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref ObjectInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - throw new GarnetException("GetRMWModifiedValueLength is not available on the object store"); + // We know namespaces aren't present in object functions, so don't populate + return new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = ObjectIdMap.ObjectIdSize, + ValueIsObject = true, + HasETag = false + // No object commands take an Expiration for InitialUpdater. + }; } /// - public int GetRMWInitialValueLength(ref ObjectInput input) + public RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref ObjectInput input) + where TSourceLogRecord : ISourceLogRecord { - throw new GarnetException("GetRMWInitialValueLength is not available on the object store"); + return new RecordFieldInfo() + { + KeySize = srcLogRecord.Key.Length, + ValueSize = ObjectIdMap.ObjectIdSize, + ValueIsObject = true, + HasETag = false, + HasExpiration = srcLogRecord.Info.HasExpiration, + RecordType = srcLogRecord.RecordType, + }; } - public int GetUpsertValueLength(ref IGarnetObject value, ref ObjectInput input) + public RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref ObjectInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - throw new GarnetException("GetUpsertInitialValueLength is not available on the object store"); + // We know namespaces aren't present in object functions, so don't populate + return new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = value.Length, + ValueIsObject = false, + HasETag = false + // No object commands take an Expiration for Upsert. + }; + } + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref ObjectInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // We know namespaces aren't present in object functions, so don't populate + return new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = ObjectIdMap.ObjectIdSize, + ValueIsObject = true, + HasETag = false + // No object commands take an Expiration for Upsert. + }; + } + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref ObjectInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + return new RecordFieldInfo() + { + KeySize = key.KeyBytes.Length, + ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, + ValueIsObject = true, + HasETag = false, + RecordType = inputLogRecord.RecordType, + // No object commands take an Expiration for Upsert. + }; } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/SessionFunctionsUtils.cs b/libs/server/Storage/Functions/SessionFunctionsUtils.cs new file mode 100644 index 00000000000..128fc3eba39 --- /dev/null +++ b/libs/server/Storage/Functions/SessionFunctionsUtils.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Garnet.common; +using Microsoft.Extensions.Logging; +using Tsavorite.core; + +namespace Garnet.server +{ + internal static class SessionFunctionsUtils + { + internal enum IPUResult : byte + { + Failed = 0, + Succeeded, + NotUpdated, + } + + /// + /// Attempts to set the expiration time on a log record based on the specified . + /// + /// The log record to update. + /// The expiration option that determines how the expiration should be set. + /// The new expiration value to set. + /// True if method should log an error when failed to set expiration. + /// The logger for error reporting. + /// Set to true if the expiration was changed; otherwise, false. + /// True if the expiration was set or the operation was valid for the given option; otherwise, false. + internal static bool EvaluateExpire(ref LogRecord logRecord, ExpireOption optionType, long newExpiry, bool hasExpiration, bool logErrorOnFail, ILogger logger, out bool expirationChanged) + { + expirationChanged = false; + + if (hasExpiration) + { + // Expiration already exists so there is no need to check for space (i.e. failure of TrySetExpiration) + switch (optionType) + { + case ExpireOption.NX: + return true; + case ExpireOption.XX: + case ExpireOption.None: + _ = logRecord.TrySetExpiration(newExpiry); + expirationChanged = true; + return true; + case ExpireOption.GT: + case ExpireOption.XXGT: + if (newExpiry > logRecord.Expiration) + { + _ = logRecord.TrySetExpiration(newExpiry); + expirationChanged = true; + } + return true; + case ExpireOption.LT: + case ExpireOption.XXLT: + if (newExpiry < logRecord.Expiration) + { + _ = logRecord.TrySetExpiration(newExpiry); + expirationChanged = true; + } + return true; + default: + throw new GarnetException($"{nameof(EvaluateExpire)} exception when HasExpiration is true. optionType: {optionType}"); + } + } + + // No expiration yet. + switch (optionType) + { + case ExpireOption.NX: + case ExpireOption.None: + case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite, so the new value must be less + var isSuccessful = logRecord.TrySetExpiration(newExpiry); + if (!isSuccessful && logErrorOnFail) + { + logger?.LogError("Failed to add expiration in {methodName}.{caseName}", nameof(EvaluateExpire), optionType); + return false; + } + expirationChanged = isSuccessful; + return isSuccessful; + case ExpireOption.XX: + case ExpireOption.GT: + case ExpireOption.XXGT: + case ExpireOption.XXLT: + return true; + default: + throw new GarnetException($"{nameof(EvaluateExpire)} exception when HasExpiration is false. optionType: {optionType}"); + } + } + + internal static bool InPlaceWriterForSpanValue(ref LogRecord logRecord, ref TInput input, ReadOnlySpan newValue, + ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, TVariableLengthInput varlenInput, FunctionsState functionsState, long expiration) + where TVariableLengthInput : IVariableLengthInput + { + RecordSizeInfo sizeInfo; + + if (logRecord.Info.ValueIsInline && (expiration == 0 || logRecord.Info.HasExpiration)) + { + var (valueAddress, valueLength) = logRecord.PinnedValueAddressAndLength; + if (!logRecord.TrySetPinnedValueSpan(newValue, valueAddress, ref valueLength)) + return false; + sizeInfo = new(); + } + else + { + // Create local sizeInfo + sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(logRecord, newValue, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo); + + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newValue, in sizeInfo)) + return false; + } + + UpdateExpiration(ref logRecord, expiration); + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + return true; + } + + internal static bool InPlaceWriterForHeapObjectValue(ref LogRecord logRecord, ref TInput input, IHeapObject newValue, + ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, TVariableLengthInput varlenInput, FunctionsState functionsState, long expiration) + where TVariableLengthInput : IVariableLengthInput + { + + // Create local sizeInfo + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(logRecord, newValue, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo); + + if (!logRecord.TrySetValueObjectAndPrepareOptionals(newValue, in sizeInfo)) + return false; + + UpdateExpiration(ref logRecord, expiration); + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + return true; + } + + /// + internal static bool InPlaceWriterForLogRecordValue(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, + ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, TVariableLengthInput varlenInput, FunctionsState functionsState, long expiration) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + { + + // Create local sizeInfo + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(logRecord, inputLogRecord, ref input) }; + functionsState.storeWrapper.store.Log.PopulateRecordSizeInfo(ref sizeInfo); + _ = logRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + + UpdateExpiration(ref logRecord, expiration); + sizeInfo.AssertOptionalsIfSet(logRecord.Info); + + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void UpdateExpiration(ref LogRecord logRecord, long expiration) + { + if (expiration != 0) + { + if (!logRecord.TrySetExpiration(expiration)) + Debug.Fail("Should have succeeded in setting Expiration as we should have ensured there was space there already"); + } + else if (logRecord.Info.HasExpiration) + _ = logRecord.RemoveExpiration(); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/SimpleGarnetObjectSessionFunctions.cs b/libs/server/Storage/Functions/SimpleGarnetObjectSessionFunctions.cs new file mode 100644 index 00000000000..98ead1a6d1e --- /dev/null +++ b/libs/server/Storage/Functions/SimpleGarnetObjectSessionFunctions.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Default simple functions base class with TInput and TOutput both being IGarnetObject; it assumes the Value is always an Object, never a Span. + /// + public class SimpleGarnetObjectSessionFunctions : SessionFunctionsBase + { + /// + public override bool Reader(in TSourceLogRecord srcLogRecord, ref IGarnetObject input, ref IGarnetObject output, ref ReadInfo readInfo) + { + if (srcLogRecord.Info.ValueIsObject) + { + output = (IGarnetObject)srcLogRecord.ValueObject; + return true; + } + return false; // TODO: possibly create an IGarnetObject from the serialized bytes + } + + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref IGarnetObject input, IHeapObject srcValue, ref IGarnetObject output, ref UpsertInfo upsertInfo) + { + var result = base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + if (result) + output = (IGarnetObject)srcValue; + return result; + } + + public override bool InPlaceWriter(ref LogRecord logRecord, ref IGarnetObject input, IHeapObject srcValue, ref IGarnetObject output, ref UpsertInfo upsertInfo) + { + var result = base.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo); + if (result) + output = (IGarnetObject)srcValue; + return result; + } + + /// + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref IGarnetObject input, ref IGarnetObject output, ref RMWInfo rmwInfo) + { + var result = dstLogRecord.TrySetValueObjectAndPrepareOptionals(input, in sizeInfo); + if (result) + output = input; + return result; + } + /// + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref IGarnetObject input, ref IGarnetObject output, ref RMWInfo rmwInfo) + { + // Simple base implementation does not use upsertInfo + var upsertInfo = new UpsertInfo(); + return base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, in srcLogRecord, ref output, ref upsertInfo); + } + /// + public override bool InPlaceUpdater(ref LogRecord logRecord, ref IGarnetObject input, ref IGarnetObject output, ref RMWInfo rmwInfo) + { + // Simple base implementation does not use upsertInfo + var upsertInfo = new UpsertInfo(); + return InPlaceWriter(ref logRecord, ref input, input, ref output, ref upsertInfo); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/CallbackMethods.cs b/libs/server/Storage/Functions/UnifiedStore/CallbackMethods.cs new file mode 100644 index 00000000000..57987b0b8d1 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/CallbackMethods.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly unsafe partial struct UnifiedSessionFunctions : ISessionFunctions + { + public void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref UnifiedInput input, + ref UnifiedOutput output, long ctx, Status status, RecordMetadata recordMetadata) + { + } + + public void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref UnifiedInput input, + ref UnifiedOutput output, long ctx, Status status, RecordMetadata recordMetadata) + { + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/DeleteMethods.cs b/libs/server/Storage/Functions/UnifiedStore/DeleteMethods.cs new file mode 100644 index 00000000000..caf28327abf --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/DeleteMethods.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly partial struct UnifiedSessionFunctions : ISessionFunctions + { + public bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + { + if (!logRecord.Info.ValueIsObject) + { + logRecord.InfoRef.ClearHasETag(); + functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); + } + + return true; + } + + public void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + { + if (logRecord.Info.ValueIsObject && !logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); + + if (functionsState.appendOnlyFile != null) + deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + } + + public bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + { + if (!logRecord.Info.ValueIsObject) + logRecord.ClearOptionals(); + + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); + + if (functionsState.appendOnlyFile != null) + deleteInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + // Heap object cache-size tracking and disposal are handled by + // storeFunctions.OnDispose (GarnetRecordTriggers) which is called + // by Tsavorite after InPlaceDeleter returns. + return true; + } + + /// + public void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + + if ((deleteInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF + WriteLogDelete(key.KeyBytes, deleteInfo.Version, deleteInfo.SessionID, epochAccessor); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/PrivateMethods.cs b/libs/server/Storage/Functions/UnifiedStore/PrivateMethods.cs new file mode 100644 index 00000000000..8d7eec836f5 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/PrivateMethods.cs @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly unsafe partial struct UnifiedSessionFunctions : ISessionFunctions + { + /// + /// Logging upsert from + /// a. InPlaceWriter + /// b. PostInitialWriter + /// + void WriteLogUpsert(ReadOnlySpan key, ref UnifiedInput input, ReadOnlySpan value, long version, int sessionID, TEpochAccessor epochAccessor) + where TEpochAccessor : IEpochAccessor + { + if (functionsState.StoredProcMode) + return; + + // We need this check because when we ingest records from the primary + // if the input is zero then input overlaps with value so any update to RespInputHeader->flags + // will incorrectly modify the total length of value. + if (input.SerializedLength > 0) + input.header.flags |= RespInputFlags.Deterministic; + + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.UnifiedStoreStringUpsert, + version, + sessionID, + key, + value, + ref input, + epochAccessor, + out _); + } + + /// + /// Logging upsert from + /// a. InPlaceWriter + /// b. PostInitialWriter + /// + void WriteLogUpsert(ReadOnlySpan key, ref UnifiedInput input, IGarnetObject value, long version, int sessionID, TEpochAccessor epochAccessor) + where TEpochAccessor : IEpochAccessor + { + if (functionsState.StoredProcMode) + return; + + input.header.flags |= RespInputFlags.Deterministic; + + GarnetObjectSerializer.Serialize(value, out var valueBytes); + fixed (byte* valPtr = valueBytes) + { + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.UnifiedStoreObjectUpsert, + version, + sessionID, + key, + new ReadOnlySpan(valPtr, valueBytes.Length), + ref input, + epochAccessor, + out _); + } + } + + /// + /// Logging Delete from + /// a. InPlaceDeleter + /// b. PostInitialDeleter + /// + void WriteLogDelete(ReadOnlySpan key, long version, int sessionID, TEpochAccessor epochAccessor) + where TEpochAccessor : IEpochAccessor + { + if (functionsState.StoredProcMode) + return; + + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.UnifiedStoreDelete, + version, + sessionID, + key, + value: default, + epochAccessor, + out _); + } + + /// + /// Logging RMW from + /// a. PostInitialUpdater + /// b. InPlaceUpdater + /// c. PostCopyUpdater + /// + void WriteLogRMW(ReadOnlySpan key, ref UnifiedInput input, long version, int sessionId, TEpochAccessor epochAccessor) + where TEpochAccessor : IEpochAccessor + { + if (functionsState.StoredProcMode) return; + input.header.flags |= RespInputFlags.Deterministic; + + functionsState.appendOnlyFile.Log.Enqueue( + AofEntryType.UnifiedStoreRMW, + version, + sessionId, + key, + ref input, + epochAccessor, + out _); + } + + bool EvaluateExpireCopyUpdate(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ExpireOption optionType, long newExpiry, ReadOnlySpan newValue, ref UnifiedOutput output) + { + var hasExpiration = logRecord.Info.HasExpiration; + + // TODO ETag? + if (!logRecord.TrySetValueSpanAndPrepareOptionals(newValue, in sizeInfo)) + { + functionsState.logger?.LogError("Failed to set value in {methodName}", nameof(EvaluateExpireCopyUpdate)); + return false; + } + + var isSuccessful = EvaluateExpire(ref logRecord, optionType, newExpiry, hasExpiration, + logErrorOnFail: true, functionsState.logger, out var expirationChanged); + + functionsState.CopyDefaultResp( + isSuccessful && expirationChanged ? CmdStrings.RESP_RETURN_VAL_1 : CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); + + return isSuccessful; + } + + IPUResult EvaluateExpireInPlace(ref LogRecord logRecord, ExpireOption optionType, long newExpiry, bool hasExpiration, ref UnifiedOutput output) + { + Debug.Assert(output.SpanByteAndMemory.IsSpanByte, "This code assumes it is called in-place and did not go pending"); + + if (!EvaluateExpire(ref logRecord, optionType, newExpiry, hasExpiration, logErrorOnFail: false, functionsState.logger, out var expirationChanged)) + return IPUResult.Failed; + + if (expirationChanged) + { + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); + return IPUResult.Succeeded; + } + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); + return IPUResult.NotUpdated; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/RMWMethods.cs b/libs/server/Storage/Functions/UnifiedStore/RMWMethods.cs new file mode 100644 index 00000000000..deab048cd64 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/RMWMethods.cs @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly partial struct UnifiedSessionFunctions : ISessionFunctions + { + /// + public bool NeedInitialUpdate(TKey key, ref UnifiedInput input, ref UnifiedOutput output, + ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + return input.header.cmd switch + { + RespCommand.DELIFEXPIM or + RespCommand.PERSIST or + RespCommand.EXPIRE => false, + _ => true + }; + } + + /// + public bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + ref UnifiedOutput output, ref RMWInfo rmwInfo) + { + Debug.Assert(logRecord.Info.ValueIsObject || (!logRecord.Info.HasETag && !logRecord.Info.HasExpiration), + "Should not have Expiration or ETag on InitialUpdater log records"); + + return input.header.cmd switch + { + RespCommand.DELIFEXPIM or + RespCommand.PERSIST or + RespCommand.EXPIRE => throw new Exception(), + _ => true + }; + } + + /// + public void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + ref UnifiedOutput output, ref RMWInfo rmwInfo) + { + functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + { + input.header.SetExpiredFlag(); + rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + } + } + + /// + public bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref UnifiedInput input, + ref UnifiedOutput output, ref RMWInfo rmwInfo) where TSourceLogRecord : ISourceLogRecord + { + var cmd = input.header.cmd; + if (cmd == RespCommand.DELIFEXPIM && srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) + { + rmwInfo.Action = RMWAction.ExpireAndStop; + return false; + } + + return true; + } + + /// + public bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, + in RecordSizeInfo sizeInfo, ref UnifiedInput input, ref UnifiedOutput output, + ref RMWInfo rmwInfo) where TSourceLogRecord : ISourceLogRecord + { + + if (srcLogRecord.Info.HasExpiration && input.header.CheckExpiry(srcLogRecord.Expiration)) + { + rmwInfo.Action = RMWAction.ExpireAndResume; + return false; + } + + if (srcLogRecord.Info.ValueIsObject) + { + // Defer the actual copying of data to PostCopyUpdater, so we know the record has been successfully CASed into the hash chain before we potentially + // create large allocations (e.g. if srcLogRecord is from disk, we would have to allocate the overflow byte[]). Because we are doing an update we have + // and XLock, so nobody will see the unset data even after the CAS. Tsavorite will handle cloning the ValueObject and caching serialized data as needed, + // based on whether srcLogRecord is in-memory or a DiskLogRecord. + return true; + } + + var cmd = input.header.cmd; + bool shouldUpdateEtag = false; + + var result = cmd switch + { + RespCommand.EXPIRE => HandleExpireCopyUpdate(srcLogRecord, ref dstLogRecord, in sizeInfo, ref shouldUpdateEtag, ref input, ref output), + RespCommand.PERSIST => HandlePersistCopyUpdate(srcLogRecord, ref dstLogRecord, in sizeInfo, ref shouldUpdateEtag, ref output), + _ => throw new NotImplementedException() + }; + + if (!result) + return false; + + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + /// + public bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, + in RecordSizeInfo sizeInfo, ref UnifiedInput input, ref UnifiedOutput output, + ref RMWInfo rmwInfo) where TSourceLogRecord : ISourceLogRecord + { + functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); + + if (srcLogRecord.Info.ValueIsObject) + { + // We're performing the object update here (and not in CopyUpdater) so that we are guaranteed that + // the record was CASed into the hash chain before it gets modified + var value = Unsafe.As(srcLogRecord.ValueObject.Clone()); + + // First copy the new Value and optionals to the new record. This will also ensure space and set the flag for expiration if it's present. + // Do not set actually set dstLogRecord.Expiration until we know it is a command for which we allocated length in the LogRecord for it. + var hasExpiration = dstLogRecord.Info.HasExpiration; + if (!dstLogRecord.TrySetValueObjectAndPrepareOptionals(value, in sizeInfo)) + return false; + + var cmd = input.header.cmd; + switch (cmd) + { + case RespCommand.EXPIRE: + if (HandleExpireInPlaceUpdate(ref dstLogRecord, hasExpiration, ref input, ref output) == IPUResult.Failed) + return false; + break; + + case RespCommand.PERSIST: + HandlePersistInPlaceUpdate(ref dstLogRecord, hasExpiration, ref output); + break; + } + + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + + } + + if (functionsState.appendOnlyFile != null) + rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + return true; + } + + /// + public bool InPlaceUpdater(ref LogRecord logRecord, ref UnifiedInput input, ref UnifiedOutput output, ref RMWInfo rmwInfo) + { + var ipuResult = InPlaceUpdaterWorker(ref logRecord, ref input, ref output, ref rmwInfo); + switch (ipuResult) + { + case IPUResult.Failed: + return false; + case IPUResult.Succeeded: + if (!logRecord.Info.Modified) + functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + rmwInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + case IPUResult.NotUpdated: + default: + return true; + } + } + + IPUResult InPlaceUpdaterWorker(ref LogRecord logRecord, ref UnifiedInput input, ref UnifiedOutput output, ref RMWInfo rmwInfo) + { + var cmd = input.header.cmd; + + // Expired data + if (logRecord.Info.HasExpiration && input.header.CheckExpiry(logRecord.Expiration)) + { + // Heap disposal and cache size tracking are handled by + // OnDispose(Deleted) in InternalRMW for both ExpireAndStop and ExpireAndResume. + rmwInfo.Action = cmd == RespCommand.DELIFEXPIM ? RMWAction.ExpireAndStop : RMWAction.ExpireAndResume; + return IPUResult.Failed; + } + + var hasExpiration = logRecord.Info.HasExpiration; + + var ipuResult = IPUResult.Succeeded; + switch (cmd) + { + case RespCommand.EXPIRE: + ipuResult = HandleExpireInPlaceUpdate(ref logRecord, hasExpiration, ref input, ref output); + if (ipuResult == IPUResult.Failed) + return IPUResult.Failed; + break; + case RespCommand.PERSIST: + HandlePersistInPlaceUpdate(ref logRecord, hasExpiration, ref output); + break; + case RespCommand.DELIFEXPIM: + // Not expired — no-op + break; + default: + throw new NotImplementedException(); + } + + return ipuResult; + } + + private bool HandleExpireCopyUpdate(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, + in RecordSizeInfo sizeInfo, ref bool shouldUpdateEtag, ref UnifiedInput input, ref UnifiedOutput output) where TSourceLogRecord : ISourceLogRecord + { + shouldUpdateEtag = false; + var expirationWithOption = new ExpirationWithOption(input.arg1); + + // First copy the old Value and non-Expiration optionals to the new record. This will also ensure space for expiration. + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + + return EvaluateExpireCopyUpdate(ref dstLogRecord, in sizeInfo, expirationWithOption.ExpireOption, + expirationWithOption.ExpirationTimeInTicks, dstLogRecord.ValueSpan, ref output); + } + + private IPUResult HandleExpireInPlaceUpdate(ref LogRecord logRecord, bool hasExpiration, ref UnifiedInput input, ref UnifiedOutput output) + { + var expirationWithOption = new ExpirationWithOption(input.arg1); + return EvaluateExpireInPlace(ref logRecord, expirationWithOption.ExpireOption, expirationWithOption.ExpirationTimeInTicks, hasExpiration, ref output); + } + + private bool HandlePersistCopyUpdate(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, + in RecordSizeInfo sizeInfo, ref bool shouldUpdateEtag, ref UnifiedOutput output) where TSourceLogRecord : ISourceLogRecord + { + shouldUpdateEtag = false; + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; + + if (srcLogRecord.Info.HasExpiration) + { + dstLogRecord.RemoveExpiration(); + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); + } + else + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); + + return true; + } + + private void HandlePersistInPlaceUpdate(ref LogRecord logRecord, bool hasExpiration, ref UnifiedOutput output) + { + if (hasExpiration) + { + logRecord.RemoveExpiration(); + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_1, ref output.SpanByteAndMemory); + } + else + functionsState.CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.SpanByteAndMemory); + } + + + /// + public void PostRMWOperation(TKey key, ref UnifiedInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + if ((rmwInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF + WriteLogRMW(key.KeyBytes, ref input, rmwInfo.Version, rmwInfo.SessionID, epochAccessor); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/ReadMethods.cs b/libs/server/Storage/Functions/UnifiedStore/ReadMethods.cs new file mode 100644 index 00000000000..fd9c50634b4 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/ReadMethods.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static LogRecordUtils; + using static Utility; + + /// + /// Unified store functions + /// + public readonly unsafe partial struct UnifiedSessionFunctions : ISessionFunctions + { + public bool Reader(in TSourceLogRecord srcLogRecord, ref UnifiedInput input, + ref UnifiedOutput output, ref ReadInfo readInfo) where TSourceLogRecord : ISourceLogRecord + { + if (CheckExpiry(in srcLogRecord)) + { + readInfo.Action = ReadAction.Expire; + return false; + } + + var cmd = input.header.cmd; + return cmd switch + { + RespCommand.EXISTS => true, + RespCommand.MIGRATE => HandleMigrate(in srcLogRecord, (int)input.arg1, ref output), + RespCommand.MEMORY_USAGE => HandleMemoryUsage(in srcLogRecord, ref output), + RespCommand.TYPE => HandleType(in srcLogRecord, ref output), + RespCommand.TTL or + RespCommand.PTTL => HandleTtl(in srcLogRecord, ref output, cmd == RespCommand.PTTL), + RespCommand.EXPIRETIME or + RespCommand.PEXPIRETIME => HandleExpireTime(in srcLogRecord, ref output, cmd == RespCommand.PEXPIRETIME), + RespCommand.RENAME => HandleRename(in srcLogRecord, ref output), + _ => throw new NotImplementedException(), + }; + } + + private bool HandleMemoryUsage(in TSourceLogRecord srcLogRecord, + ref UnifiedOutput output) where TSourceLogRecord : ISourceLogRecord + { + var inlineRecordSize = srcLogRecord.AllocatedSize; + long heapMemoryUsage = 0; + if (srcLogRecord.Info.KeyIsOverflow) + heapMemoryUsage += srcLogRecord.Key.Length + MemoryUtils.ByteArrayOverhead; + + if (srcLogRecord.Info.ValueIsOverflow) + heapMemoryUsage += srcLogRecord.ValueSpan.Length + MemoryUtils.ByteArrayOverhead; + else if (srcLogRecord.Info.ValueIsObject) + { + heapMemoryUsage = RecordInfo.Size + (2 * IntPtr.Size) + // Log record length + Utility.RoundUp(srcLogRecord.Key.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + // Key allocation in heap with overhead + srcLogRecord.ValueObject.HeapMemorySize; // Value allocation in heap + } + + using var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + writer.WriteInt64(heapMemoryUsage + inlineRecordSize); + + return true; + } + + private bool HandleType(in TSourceLogRecord srcLogRecord, + ref UnifiedOutput output) where TSourceLogRecord : ISourceLogRecord + { + using var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + + if (srcLogRecord.Info.ValueIsObject) + { + switch (srcLogRecord.ValueObject) + { + case SortedSetObject: + writer.WriteSimpleString(CmdStrings.zset); + break; + case ListObject: + writer.WriteSimpleString(CmdStrings.list); + break; + case SetObject: + writer.WriteSimpleString(CmdStrings.set); + break; + case HashObject: + writer.WriteSimpleString(CmdStrings.hash); + break; + } + } + else + { + if (srcLogRecord.RecordType == RangeIndexManager.RangeIndexRecordType) + writer.WriteSimpleString(CmdStrings.rangeindext); + else + writer.WriteSimpleString(CmdStrings.stringt); + } + + return true; + } + + private bool HandleTtl(in TSourceLogRecord srcLogRecord, + ref UnifiedOutput output, bool milliseconds) where TSourceLogRecord : ISourceLogRecord + { + using var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + + var expiration = srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1; + var ttlValue = milliseconds + ? ConvertUtils.MillisecondsFromDiffUtcNowTicks(expiration) + : ConvertUtils.SecondsFromDiffUtcNowTicks(expiration); + + writer.WriteInt64(ttlValue); + return true; + } + + private bool HandleExpireTime(in TSourceLogRecord srcLogRecord, + ref UnifiedOutput output, bool milliseconds) where TSourceLogRecord : ISourceLogRecord + { + using var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); + + var expiration = srcLogRecord.Info.HasExpiration ? srcLogRecord.Expiration : -1; + var expireTime = milliseconds + ? ConvertUtils.UnixTimeInMillisecondsFromTicks(expiration) + : ConvertUtils.UnixTimeInSecondsFromTicks(expiration); + + writer.WriteInt64(expireTime); + return true; + } + + private bool HandleMigrate(in TSourceLogRecord srcLogRecord, int maxHeapAllocationSize, ref UnifiedOutput output) + where TSourceLogRecord : ISourceLogRecord + { + DiskLogRecord.Serialize(in srcLogRecord, maxHeapAllocationSize, + valueObjectSerializer: srcLogRecord.Info.ValueIsObject ? functionsState.garnetObjectSerializer : null, + memoryPool: functionsState.memoryPool, output: ref output.SpanByteAndMemory); + return true; + } + + private bool HandleRename(in TSourceLogRecord srcLogRecord, ref UnifiedOutput output) + where TSourceLogRecord : ISourceLogRecord + { + // First, copy the inline portion of the record to the output. Any object references are retained in this step; we do *not* serialize, + // but rather hand off the object references (remapped to the transient allocator if needed), because RENAME is an in-memory operation. + + // network In case of significant shrinkage, calculate this AllocatedSize separately rather than logRecord.GetInlineRecordSizes().allocatedSize. + var inlineRecordSize = RoundUp(srcLogRecord.ActualSize, 8); // TODO: Constants.kRecordAlignment + DiskLogRecord.DirectCopyInlinePortionOfRecord(in srcLogRecord, inlineRecordSize, estimatedTotalSize: inlineRecordSize, maxHeapAllocationSize: inlineRecordSize, + functionsState.memoryPool, ref output.SpanByteAndMemory); + if (srcLogRecord.Info.RecordHasObjects) + { + fixed (byte* recordPtr = output.SpanByteAndMemory.Span) + { + var logRecord = new LogRecord(recordPtr, srcLogRecord.ObjectIdMap); + logRecord.RemapOverPinnedTransientMemory(srcLogRecord.ObjectIdMap, functionsState.transientObjectIdMap); + } + } + return true; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/UnifiedSessionFunctions.cs b/libs/server/Storage/Functions/UnifiedStore/UnifiedSessionFunctions.cs new file mode 100644 index 00000000000..7f33c36a8d3 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/UnifiedSessionFunctions.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Garnet.server +{ + public readonly partial struct UnifiedSessionFunctions : ISessionFunctions + { + const byte NeedAofLog = 0x1; + readonly FunctionsState functionsState; + readonly ReadSessionState readSessionState; + + /// + /// Constructor + /// + /// + /// + internal UnifiedSessionFunctions(FunctionsState functionsState, ReadSessionState readSessionState = null) + { + this.functionsState = functionsState; + this.readSessionState = readSessionState; + } + + public void ConvertOutputToHeap(ref UnifiedInput input, ref UnifiedOutput output) + { + // TODO: Inspect input to determine whether we're in a context requiring ConvertToHeap. + //output.ConvertToHeap(); + } + + /// + public void BeforeConsistentReadCallback(long hash) + => readSessionState?.BeforeConsistentReadKeyCallback(hash); + + /// + public void AfterConsistentReadKeyCallback() + => readSessionState?.AfterConsistentReadKeyCallback(); + + /// + public void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) + => readSessionState?.BeforeConsistentReadKeyBatch(parameters); + + /// + public bool AfterConsistentReadKeyBatchCallback(int keyCount) + => readSessionState != null && readSessionState.AfterConsistentReadKeyBatch(keyCount); + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/UpsertMethods.cs b/libs/server/Storage/Functions/UnifiedStore/UpsertMethods.cs new file mode 100644 index 00000000000..0dd9b3b0876 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/UpsertMethods.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Tsavorite.core; +using static Garnet.server.SessionFunctionsUtils; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly partial struct UnifiedSessionFunctions : ISessionFunctions + { + /// + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + ReadOnlySpan srcValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + /// + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + IHeapObject srcValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + if (!dstLogRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + // TODO ETag + if (input.arg1 != 0 && !dstLogRecord.TrySetExpiration(input.arg1)) + return false; + sizeInfo.AssertOptionalsIfSet(dstLogRecord.Info); + return true; + } + + /// + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, + ref UnifiedInput input, in TSourceLogRecord inputLogRecord, ref UnifiedOutput output, + ref UpsertInfo upsertInfo) where TSourceLogRecord : ISourceLogRecord + { + if (!dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo)) + return false; + return true; + } + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + ReadOnlySpan srcValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + } + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref UnifiedInput input, + IHeapObject srcValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + var garnetObject = (IGarnetObject)srcValue; + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + } + + /// + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, + ref UnifiedInput input, in TSourceLogRecord inputLogRecord, ref UnifiedOutput output, + ref UpsertInfo upsertInfo) where TSourceLogRecord : ISourceLogRecord + { + functionsState.watchVersionMap.IncrementVersion(upsertInfo.KeyHash); + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref UnifiedInput input, ReadOnlySpan newValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + if (!InPlaceWriterForSpanValue(ref logRecord, ref input, newValue, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref UnifiedInput input, IHeapObject newValue, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + { + if (!InPlaceWriterForHeapObjectValue(ref logRecord, ref input, newValue, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + public bool InPlaceWriter(ref LogRecord logRecord, ref UnifiedInput input, + in TSourceLogRecord inputLogRecord, ref UnifiedOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (!InPlaceWriterForLogRecordValue(ref logRecord, ref input, in inputLogRecord, ref output.SpanByteAndMemory, ref upsertInfo, this, functionsState, input.arg1)) + return false; + if (functionsState.appendOnlyFile != null) + upsertInfo.UserData |= NeedAofLog; // Mark that we need to write to AOF + return true; + } + + /// + public void PostUpsertOperation(TKey key, ref UnifiedInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + if ((upsertInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF + WriteLogUpsert(key.KeyBytes, ref input, valueSpan, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); + } + + /// + public void PostUpsertOperation(TKey key, ref UnifiedInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + if ((upsertInfo.UserData & NeedAofLog) == NeedAofLog) // Check if we need to write to AOF + WriteLogUpsert(key.KeyBytes, ref input, (IGarnetObject)valueObject, upsertInfo.Version, upsertInfo.SessionID, epochAccessor); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/UnifiedStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/UnifiedStore/VarLenInputMethods.cs new file mode 100644 index 00000000000..697448b88b5 --- /dev/null +++ b/libs/server/Storage/Functions/UnifiedStore/VarLenInputMethods.cs @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Unified store functions + /// + public readonly unsafe partial struct UnifiedSessionFunctions : ISessionFunctions + { + public RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, + ref UnifiedInput input) where TSourceLogRecord : ISourceLogRecord + { + var fieldInfo = new RecordFieldInfo + { + KeySize = srcLogRecord.Key.Length, + ValueSize = srcLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : 0, + ValueIsObject = srcLogRecord.Info.ValueIsObject, + HasETag = !srcLogRecord.Info.ValueIsObject && srcLogRecord.Info.HasETag, + HasExpiration = srcLogRecord.Info.HasExpiration, + RecordType = srcLogRecord.RecordType, + }; + + if (input.header.cmd != RespCommand.NONE) + { + var cmd = input.header.cmd; + + switch (cmd) + { + case RespCommand.EXPIRE: + { + // Set HasExpiration to match with EvaluateExpireInPlace. + if (srcLogRecord.Info.HasExpiration) + { + // case ExpireOption.NX: // HasExpiration is true so we will retain it + // case ExpireOption.XX: + // case ExpireOption.None: + // case ExpireOption.GT: + // case ExpireOption.XXGT: + // case ExpireOption.LT: + // case ExpireOption.XXLT: + fieldInfo.HasExpiration = true; // Will update or retain + } + else + { + var expirationWithOption = new ExpirationWithOption(input.arg1); + switch (expirationWithOption.ExpireOption) + { + case ExpireOption.NX: + case ExpireOption.None: + case ExpireOption.LT: + // If expiry doesn't exist, LT should treat the current expiration as infinite, so the new value must be less + fieldInfo.HasExpiration = true; // Will update or retain + break; + default: + // case ExpireOption.XX: + // case ExpireOption.GT: // If expiry doesn't exist, GT should treat the current expiration as infinite, so the new value cannot be greater + // case ExpireOption.XXGT: + // case ExpireOption.XXLT: + fieldInfo.HasExpiration = false; // Will not add one and there is not one there now + break; + } + } + } + + if (!srcLogRecord.Info.ValueIsObject) + fieldInfo.ValueSize = srcLogRecord.ValueSpan.Length; + return fieldInfo; + case RespCommand.PERSIST: + fieldInfo.HasExpiration = false; + if (!srcLogRecord.Info.ValueIsObject) + fieldInfo.ValueSize = srcLogRecord.ValueSpan.Length; + return fieldInfo; + default: + return fieldInfo; + } + } + + fieldInfo.ValueSize = input.parseState.GetArgSliceByRef(0).Length; + fieldInfo.HasExpiration = input.arg1 != 0; + return fieldInfo; + } + + public RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref UnifiedInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // We know namespaces aren't present in string/object functions, so don't populate + return new RecordFieldInfo + { + KeySize = key.KeyBytes.Length, + ValueSize = 0, + HasETag = false + }; + } + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, + ref UnifiedInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // We know namespaces aren't present in string/object functions, so don't populate + return new RecordFieldInfo + { + KeySize = key.KeyBytes.Length, + ValueSize = value.Length, + ValueIsObject = false, + HasETag = false + }; + } + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref UnifiedInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + return new RecordFieldInfo + { + KeySize = key.KeyBytes.Length, + ValueSize = ObjectIdMap.ObjectIdSize, + ValueIsObject = true, + HasETag = false + }; + } + + public RecordFieldInfo GetUpsertFieldInfo(TKey key, + in TSourceLogRecord inputLogRecord, + ref UnifiedInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + return new RecordFieldInfo + { + KeySize = key.KeyBytes.Length, + ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, + ValueIsObject = inputLogRecord.Info.ValueIsObject, + HasETag = false, + HasExpiration = inputLogRecord.Info.HasExpiration + }; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Functions/VectorStore/VectorSessionFunctions.cs b/libs/server/Storage/Functions/VectorStore/VectorSessionFunctions.cs new file mode 100644 index 00000000000..c67423ce2aa --- /dev/null +++ b/libs/server/Storage/Functions/VectorStore/VectorSessionFunctions.cs @@ -0,0 +1,722 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers.Binary; +using System.Collections.Frozen; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Functions for operating against the Main Store, but for data stored as part of a Vector Set operation - not a RESP command. + /// + public readonly struct VectorSessionFunctions : ISessionFunctions + { + private const int ValueAlignmentBytes = 4; + + private readonly FunctionsState functionsState; + private readonly ReadSessionState readSessionState; + + /// + /// Constructor + /// + /// + /// + internal VectorSessionFunctions(FunctionsState functionsState, ReadSessionState readSessionState = null) + { + this.functionsState = functionsState; + this.readSessionState = readSessionState; + } + + #region Reads + /// + public readonly bool Reader(in TSourceLogRecord srcLogRecord, ref VectorInput input, ref VectorOutput output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord + { + Debug.Assert(srcLogRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(srcLogRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var value = AlignOrPin(in srcLogRecord, ref input, out var pin); + try + { + if (input.IsMigrationRead) + { + Debug.Assert(input.Callback == 0, "No callback expected"); + + // We can't ship the log record over because of alignment shenanigans + // TODO: When alignment is handled at the Tsavorite level, we CAN start shipping the log over like everything else + + var neededSpace = + sizeof(int) + srcLogRecord.NamespaceBytes.Length + + sizeof(int) + srcLogRecord.KeyBytes.Length + + sizeof(int) + value.Length; + + output.SpanByteAndMemory.EnsureHeapMemorySize(neededSpace); + + var writeTo = output.SpanByteAndMemory.Span; + + BinaryPrimitives.WriteInt32LittleEndian(writeTo, srcLogRecord.NamespaceBytes.Length); + writeTo = writeTo[sizeof(int)..]; + srcLogRecord.NamespaceBytes.CopyTo(writeTo); + writeTo = writeTo[srcLogRecord.NamespaceBytes.Length..]; + + BinaryPrimitives.WriteInt32LittleEndian(writeTo, srcLogRecord.KeyBytes.Length); + writeTo = writeTo[sizeof(int)..]; + srcLogRecord.KeyBytes.CopyTo(writeTo); + writeTo = writeTo[srcLogRecord.KeyBytes.Length..]; + + // Move value over _without_ any padding for alignment + BinaryPrimitives.WriteInt32LittleEndian(writeTo, value.Length); + writeTo = writeTo[sizeof(int)..]; + value.CopyTo(writeTo); + + return true; + } + + unsafe + { + if (input.Callback != 0) + { + var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; + + var dataPtr = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(value)); + var dataLen = (nuint)value.Length; + + callback(input.Index, input.CallbackContext, dataPtr, dataLen); + return true; + } + } + + if (input.ReadDesiredSize > 0) + { + Debug.Assert(output.SpanByteAndMemory.Length >= value.Length, "Should always have space for vector point reads"); + + output.SpanByteAndMemory.Length = value.Length; + value.CopyTo(output.SpanByteAndMemory.Span); + } + else + { + input.ReadDesiredSize = value.Length; + if (output.SpanByteAndMemory.Length >= value.Length) + { + value.CopyTo(output.SpanByteAndMemory.Span); + output.SpanByteAndMemory.Length = value.Length; + } + } + + return true; + } + finally + { + pin?.Free(); + } + } + + /// + public readonly void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref VectorInput input, ref VectorOutput output, long ctx, Status status, RecordMetadata recordMetadata) + { + } + #endregion Reads + + #region Upserts + /// + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ReadOnlySpan srcValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + { + Debug.Assert(logRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(logRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var value = AlignOrPin(in logRecord, ref input, out var pin); + try + { + srcValue.CopyTo(value); + + return logRecord.TrySetContentLengths(logRecord.ValueSpan.Length, in sizeInfo); + } + finally + { + pin?.Free(); + } + } + + /// + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, IHeapObject srcValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + => ObjectOperationsNotExpected(); + + /// + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, in TSourceLogRecord inputLogRecord, ref VectorOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => LogRecordOperationsNotExpected(); + + /// + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref VectorInput input, ReadOnlySpan newValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + { + Debug.Assert(logRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(logRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var value = AlignOrPin(in logRecord, ref input, out var pin); + try + { + newValue.CopyTo(value); + + return true; + } + finally + { + pin?.Free(); + } + } + + /// + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref VectorInput input, IHeapObject newValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + => ObjectOperationsNotExpected(); + + /// + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref VectorInput input, in TSourceLogRecord inputLogRecord, ref VectorOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => LogRecordOperationsNotExpected(); + #endregion Upserts + + #region RMWs + #region Variable Length + /// Length of resulting value object when performing RMW modification of value using given input + public readonly RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref VectorInput input) + where TSourceLogRecord : ISourceLogRecord + { + var value = srcLogRecord.ValueSpan; + + if (input.WriteDesiredSize < 0) + { + // Add to value, this is a dynamically sized type - which are only used from Garnet, not DiskANN + return new RecordFieldInfo() { KeySize = srcLogRecord.Key.Length, ValueSize = value.Length + (-input.WriteDesiredSize) }; + } + + var needsAlignmentPadding = input.AlignmentExpected || input.Callback != 0; + + // Constant size indicated + if (needsAlignmentPadding) + { + return new RecordFieldInfo() { KeySize = srcLogRecord.Key.Length, ValueSize = input.WriteDesiredSize + ValueAlignmentBytes }; + } + else + { + return new RecordFieldInfo() { KeySize = srcLogRecord.Key.Length, ValueSize = input.WriteDesiredSize }; + } + } + + /// Initial expected length of value object when populated by RMW using given input + public readonly RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref VectorInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + var effectiveWriteDesiredSize = input.WriteDesiredSize; + + var needsAlignmentPadding = input.AlignmentExpected || input.Callback != 0; + + if (effectiveWriteDesiredSize < 0) + { + effectiveWriteDesiredSize = -effectiveWriteDesiredSize; + } + + if (!needsAlignmentPadding) + { + return new() { KeySize = key.KeyBytes.Length, ValueSize = effectiveWriteDesiredSize }; + } + else + { + return new() { KeySize = key.KeyBytes.Length, ValueSize = effectiveWriteDesiredSize + ValueAlignmentBytes }; + } + } + + /// Length of value object, when populated by Upsert using given value and input + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref VectorInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length + ValueAlignmentBytes }; + + /// Length of value object, when populated by Upsert using given value and input + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref VectorInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => ObjectOperationsNotExpected(); + + /// Length of value object, when populated by Upsert using given log record + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref VectorInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + => new() { KeySize = key.KeyBytes.Length, ValueSize = inputLogRecord.ValueSpan.Length }; + #endregion Variable Length + + #region InitialUpdater + /// + public readonly bool NeedInitialUpdate(TKey key, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + Debug.Assert(key.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(key.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + // Only needed when updating ContextMetadata or InProgressDeletes via RMW or the DiskANN RMW callback, all of which set WriteDesiredSize + return input.WriteDesiredSize != 0; + } + + /// + public readonly bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + { + Debug.Assert(logRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(logRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var key = logRecord.Key; + var alignedValue = AlignOrPin(in logRecord, ref input, out var pin); + + try + { + + if (input.Callback == 0) + { + Debug.Assert(logRecord.NamespaceBytes.Length == 1 && logRecord.NamespaceBytes[0] == VectorManager.MetadataNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(key.Length == 0, "Shouldn't have a non-zero key, expected to working on ContextMetadata"); + + // Operating on ContextMetadata + + PinnedSpanByte newMetadataValue; + unsafe + { + newMetadataValue = PinnedSpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); + } + + newMetadataValue.CopyTo(alignedValue); + + return logRecord.TrySetContentLengths(logRecord.ValueSpan.Length, in sizeInfo); + } + else + { + Debug.Assert(input.WriteDesiredSize <= alignedValue.Length, "Insufficient space for initial update, this should never happen"); + + // Must explicitly 0 before passing if we're doing an initial update + alignedValue.Clear(); + + unsafe + { + // Callback takes: dataCallbackContext, dataPtr, dataLength + var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; + + var dataPtr = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(alignedValue)); + var dataLen = (nuint)input.WriteDesiredSize; + callback(input.CallbackContext, dataPtr, dataLen); + + return logRecord.TrySetContentLengths(logRecord.ValueSpan.Length, in sizeInfo); + } + } + } + finally + { + pin?.Free(); + } + } + #endregion InitialUpdater + + #region CopyUpdater + /// + public readonly bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => input.WriteDesiredSize != 0; + + /// + public readonly bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + { + Debug.Assert(srcLogRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(srcLogRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var key = srcLogRecord.Key; + + var oldValueAligned = AlignOrPin(in srcLogRecord, ref input, out var srcPin); + var newValueAligned = AlignOrPin(in dstLogRecord, ref input, out var dstPin); + + try + { + if (input.Callback == 0) + { + // We're doing a Metadata or InProgressDelete update + + Debug.Assert(srcLogRecord.NamespaceBytes[0] == VectorManager.MetadataNamespace, "Should be operating on special namespace"); + Debug.Assert(key.Length == 0, "Shouldn't have a non-zero key, expected to working on ContextMetadata"); + + // Doing a Metadata update + Debug.Assert(srcLogRecord.ValueSpan.Length == VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); + Debug.Assert(dstLogRecord.ValueSpan.Length == VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); + Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); + + ref readonly var oldMetadata = ref MemoryMarshal.Cast(oldValueAligned)[0]; + + PinnedSpanByte newMetadataValue; + unsafe + { + newMetadataValue = PinnedSpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); + } + + ref readonly var newMetadata = ref MemoryMarshal.Cast(newMetadataValue.ReadOnlySpan)[0]; + + if (newMetadata.Version < oldMetadata.Version) + { + rmwInfo.Action = RMWAction.CancelOperation; + return false; + } + + newMetadataValue.CopyTo(newValueAligned); + return dstLogRecord.TrySetContentLengths(srcLogRecord.ValueSpan.Length, in sizeInfo); + } + else + { + Debug.Assert(input.WriteDesiredSize <= newValueAligned.Length, "Insufficient space for copy update, this should never happen"); + Debug.Assert(input.WriteDesiredSize <= oldValueAligned.Length, "Insufficient space for copy update, this should never happen"); + + oldValueAligned.CopyTo(newValueAligned); + + unsafe + { + // Callback takes: dataCallbackContext, dataPtr, dataLength + var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; + + var dataPtr = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(newValueAligned)); + var dataLen = (nuint)input.WriteDesiredSize; + + callback(input.CallbackContext, dataPtr, dataLen); + } + + return true; + + } + } + finally + { + srcPin?.Free(); + dstPin?.Free(); + } + } + #endregion CopyUpdater + + #region InPlaceUpdater + /// + public readonly bool InPlaceUpdater(ref LogRecord logRecord, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + { + Debug.Assert(logRecord.HasNamespace, "Should never write a non-namespaced value with VectorSessionFunctions"); + Debug.Assert(logRecord.NamespaceBytes.Length == 1, "Variable length namespaces not supported"); + + var key = logRecord.Key; + + var alignedValue = AlignOrPin(in logRecord, ref input, out var pin); + try + { + if (input.Callback == 0) + { + // We're doing a Metadata or InProgressDelete update + + Debug.Assert(logRecord.NamespaceBytes.Length == 1 && logRecord.NamespaceBytes[0] == VectorManager.MetadataNamespace, "Should be operating on special namespace"); + + if (key.Length == 0) + { + // Doing a Metadata update + Debug.Assert(alignedValue.Length >= VectorManager.ContextMetadata.Size, "Should be ContextMetadata"); + Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); + + ref readonly var oldMetadata = ref MemoryMarshal.Cast(alignedValue)[0]; + + PinnedSpanByte newMetadataValue; + unsafe + { + newMetadataValue = PinnedSpanByte.FromPinnedPointer((byte*)input.CallbackContext, VectorManager.ContextMetadata.Size); + } + + ref readonly var newMetadata = ref MemoryMarshal.Cast(newMetadataValue.ReadOnlySpan)[0]; + + if (newMetadata.Version < oldMetadata.Version) + { + rmwInfo.Action = RMWAction.CancelOperation; + return false; + } + + newMetadataValue.CopyTo(alignedValue); + return true; + } + else + { + // Doing an InProgressDelete update + Debug.Assert(input.CallbackContext != 0, "Should have data on VectorInput"); + Debug.Assert(key.Length == 1 && key[0] == 1, "Should be working on InProgressDeletes"); + + Span inProgressDeleteUpdateData; + bool adding; + + unsafe + { + var len = BinaryPrimitives.ReadInt32LittleEndian(new Span((byte*)input.CallbackContext + sizeof(long), sizeof(int))); + adding = len > 0; + if (!adding) + { + len = -len; + } + + inProgressDeleteUpdateData = new Span((byte*)input.CallbackContext, sizeof(ulong) + sizeof(int) + len); + } + + return true; + } + } + else + { + Debug.Assert(input.WriteDesiredSize <= alignedValue.Length, "Insufficient space for inplace update, this should never happen"); + + unsafe + { + // Callback takes: dataCallbackContext, dataPtr, dataLength + var callback = (delegate* unmanaged[Cdecl, SuppressGCTransition])input.Callback; + + var dataPtr = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(alignedValue)); + var dataLen = (nuint)input.WriteDesiredSize; + + callback(input.CallbackContext, dataPtr, dataLen); + } + + return true; + } + } + finally + { + pin?.Free(); + } + } + #endregion InPlaceUpdater + + /// + public readonly void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref VectorInput input, ref VectorOutput output, long ctx, Status status, RecordMetadata recordMetadata) + { + } + #endregion RMWs + + #region Deletes + /// + public readonly bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + { + //Debug.Assert(key.MetadataSize == 1, "Should never delete a non-namespaced value with VectorSessionFunctions"); + + functionsState.watchVersionMap.IncrementVersion(deleteInfo.KeyHash); + return true; + } + + /// + public readonly bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + => InitialDeleter(ref logRecord, ref deleteInfo); + #endregion Deletes + + #region Utilities + /// + public readonly void ConvertOutputToHeap(ref VectorInput input, ref VectorOutput output) + { + } + #endregion Utilities + + [DoesNotReturn] + private static TReturn ObjectOperationsNotExpected([CallerMemberName] string callerName = null, [CallerLineNumber] int lineNum = -1) + => throw new InvalidOperationException($"Object related operations are not expected, was: {callerName} on {lineNum}"); + + [DoesNotReturn] + private static TReturn LogRecordOperationsNotExpected([CallerMemberName] string callerName = null, [CallerLineNumber] int lineNum = -1) + => throw new InvalidOperationException($"LogRecord related operations are not expected, was: {callerName} on {lineNum}"); + + // TODO: Remove all this alignment hackery when Tsavorite can enforce it + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static unsafe Span AlignOrPin(in TSourceLogRecord logRecord, ref VectorInput input, out GCHandle? pin) + where TSourceLogRecord : ISourceLogRecord + { + var maybeUnaligned = logRecord.ValueSpan; + + // Alignment is expected if we're passing to DiskANN or Garnet code explicitly requested it + var inputRequiresAligment = input.AlignmentExpected || input.Callback != 0; + + if (inputRequiresAligment) + { + if (logRecord.IsPinnedValue) + { + // LogRecord itself is in POH, but value might not be aligned so we need to do some checking + + Span ret; + + var leading = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(maybeUnaligned)) % 4; + if (leading == 0) + { + ret = maybeUnaligned[..^ValueAlignmentBytes]; + } + else + { + var skip = (int)(ValueAlignmentBytes - leading); + var tail = ValueAlignmentBytes - skip; + ret = maybeUnaligned[skip..^tail]; + } + + AssertAlignment(ret); + + pin = null; + return ret; + } + else + { + // Value isn't in log record, it's on the (presumably unpinned) heap as a byte[] + // + // This guarantees it's aligned, but it might move during any callback so pin + + pin = logRecord.ValueOverflow.Pin(); + + // We over allocated (we don't know how Tsavorite is going to place the value in advance) so trim the extra allocation off the end. + var ret = maybeUnaligned[..^ValueAlignmentBytes]; + + AssertAlignment(ret); + + return ret; + } + } + else + { + pin = null; + return maybeUnaligned; + } + } + + [Conditional("DEBUG")] + private static unsafe void AssertAlignment(ReadOnlySpan aligned) + { + var ptr = (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(aligned)); + Debug.Assert((ptr % ValueAlignmentBytes) == 0, "Must guarantee 4-byte alignment before invoking callback"); + } + + #region Post operation callbacks + /// + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ReadOnlySpan srcValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + { + } + + /// + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, IHeapObject srcValue, ref VectorOutput output, ref UpsertInfo upsertInfo) + => ObjectOperationsNotExpected(); + + /// + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, in TSourceLogRecord inputLogRecord, ref VectorOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => LogRecordOperationsNotExpected(); + + /// + public readonly void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + { + } + + /// + public readonly bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref VectorInput input, ref VectorOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + /// + public void PostUpsertOperation(TKey key, ref VectorInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + } + + /// + public void PostUpsertOperation(TKey key, ref VectorInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + => ObjectOperationsNotExpected(); + + /// + public void PostRMWOperation(TKey key, ref VectorInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + } + + /// + public void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { + } + + /// + public readonly void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + { + } + #endregion + + /// + /// Update the namespaces stored in according to . + /// + /// should have been used to populate with a Tsavorite Read prior to this call. + /// + public static void UpdateMigratedElementNamespaces(FrozenDictionary oldToNewNamespaces, ref VectorInput readInput, ref VectorOutput readOutput) + { + Debug.Assert(readInput.IsMigrationRead, "Unexpected input"); + + // This should contain the results from the IsMigrationRead block in Reader + var span = readOutput.SpanByteAndMemory.Span; + + var nsLen = BinaryPrimitives.ReadInt32LittleEndian(span); + Debug.Assert(nsLen == 1, "Longer namespaces not supported"); + + var oldNs = (ulong)span[sizeof(int)]; + + if (!oldToNewNamespaces.TryGetValue(oldNs, out var newNs)) + { + return; + } + + Debug.Assert(newNs <= byte.MaxValue, "Namespace too large"); + + span[sizeof(int)] = (byte)newNs; + } + + /// + public void BeforeConsistentReadCallback(long hash) + => readSessionState?.BeforeConsistentReadKeyCallback(hash); + + /// + public void AfterConsistentReadKeyCallback() + => readSessionState?.AfterConsistentReadKeyCallback(); + + /// + public void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) + => readSessionState?.BeforeConsistentReadKeyBatch(parameters); + + /// + public bool AfterConsistentReadKeyBatchCallback(int keyCount) + => readSessionState != null && readSessionState.AfterConsistentReadKeyBatch(keyCount); + } +} \ No newline at end of file diff --git a/libs/server/Storage/Session/Common/ArrayKeyIterationFunctions.cs b/libs/server/Storage/Session/Common/ArrayKeyIterationFunctions.cs index 319f440ff9a..29ff815a1d9 100644 --- a/libs/server/Storage/Session/Common/ArrayKeyIterationFunctions.cs +++ b/libs/server/Storage/Session/Common/ArrayKeyIterationFunctions.cs @@ -3,30 +3,33 @@ using System; using System.Collections.Generic; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { + using static Garnet.server.StorageSession.ArrayKeyIterationFunctions; +#pragma warning disable IDE0005 // Using directive is unnecessary. + using static LogRecordUtils; + sealed partial class StorageSession : IDisposable { // These contain classes so instantiate once and re-initialize - private ArrayKeyIterationFunctions.MainStoreGetDBSize mainStoreDbSizeFuncs; - private ArrayKeyIterationFunctions.ObjectStoreGetDBSize objectStoreDbSizeFuncs; + private ArrayKeyIterationFunctions.UnifiedStoreGetDBSize unifiedStoreDbSizeFuncs; + + // Iterator for SCAN command + private ArrayKeyIterationFunctions.UnifiedStoreGetDBKeys unifiedStoreDbScanFuncs; - // Iterators for SCAN command - private ArrayKeyIterationFunctions.MainStoreGetDBKeys mainStoreDbScanFuncs; - private ArrayKeyIterationFunctions.ObjectStoreGetDBKeys objStoreDbScanFuncs; + // Iterator for expired key deletion + private ArrayKeyIterationFunctions.ExpiredKeyDeletionScan expiredKeyDeletionScanFuncs; - // Iterators for expired key deletion - private ArrayKeyIterationFunctions.MainStoreExpiredKeyDeletionScan mainStoreExpiredKeyDeletionScanFuncs; - private ArrayKeyIterationFunctions.ObjectStoreExpiredKeyDeletionScan objectStoreExpiredKeyDeletionScanFuncs; + // Iterator for KEYS command + private ArrayKeyIterationFunctions.UnifiedStoreGetDBKeys unifiedStoreDbKeysFuncs; - // Iterators for KEYS command - private ArrayKeyIterationFunctions.MainStoreGetDBKeys mainStoreDbKeysFuncs; - private ArrayKeyIterationFunctions.ObjectStoreGetDBKeys objStoreDbKeysFuncs; + // Iterator for cluster slot deletion (DeleteSlotKeys) + private ArrayKeyIterationFunctions.DeleteSlotKeysScan deleteSlotKeysFuncs; long lastScanCursor; - List objStoreKeys; List Keys; /// @@ -41,15 +44,11 @@ sealed partial class StorageSession : IDisposable /// size of every block or keys to return /// The type object to filter out /// - internal unsafe bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out long storeCursor, out List keys, long count = 10, ReadOnlySpan typeObject = default) + internal unsafe bool DbScan(PinnedSpanByte patternB, bool allKeys, long cursor, out long storeCursor, out List keys, long count = 10, ReadOnlySpan typeObject = default) { - const long IsObjectStoreCursor = 1L << 49; Keys ??= new(); Keys.Clear(); - objStoreKeys ??= new(); - objStoreKeys.Clear(); - keys = Keys; Type matchType = null; @@ -71,6 +70,10 @@ internal unsafe bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out lo { matchType = typeof(HashObject); } + else if (typeObject.SequenceEqual(CmdStrings.STRING) || typeObject.SequenceEqual(CmdStrings.stringt)) + { + matchType = typeof(string); + } else if (!typeObject.SequenceEqual(CmdStrings.STRING) && !typeObject.SequenceEqual(CmdStrings.stringt)) { // Unexpected typeObject type @@ -79,64 +82,33 @@ internal unsafe bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out lo } } - byte* patternPtr = patternB.ptr; + var patternPtr = patternB.ToPointer(); - mainStoreDbScanFuncs ??= new(); - mainStoreDbScanFuncs.Initialize(Keys, allKeys ? null : patternPtr, patternB.Length); - objStoreDbScanFuncs ??= new(); - objStoreDbScanFuncs.Initialize(objStoreKeys, allKeys ? null : patternPtr, patternB.Length, matchType); + unifiedStoreDbScanFuncs ??= IsConsistentReadSession ? new ConsistentUnifiedStoreGetDBKeys(readSessionState) : new UnifiedStoreGetDBKeys(); + unifiedStoreDbScanFuncs.Initialize(Keys, allKeys ? null : patternPtr, patternB.Length, matchType); storeCursor = cursor; long remainingCount = count; - // Cursor is zero or not an object store address - // Scan main store only for string or default key type - if ((cursor & IsObjectStoreCursor) == 0 && (typeObject.IsEmpty || typeObject.SequenceEqual(CmdStrings.STRING) || typeObject.SequenceEqual(CmdStrings.stringt))) - { - basicContext.Session.ScanCursor(ref storeCursor, count, mainStoreDbScanFuncs, validateCursor: cursor != 0 && cursor != lastScanCursor); - remainingCount -= Keys.Count; - } - - // Scan object store with the type parameter - // Check the cursor value corresponds to the object store - if (!objectStoreBasicContext.IsNull && remainingCount > 0 && (typeObject.IsEmpty || (!typeObject.SequenceEqual(CmdStrings.STRING) && !typeObject.SequenceEqual(CmdStrings.stringt)))) - { - var validateCursor = storeCursor != 0 && storeCursor != lastScanCursor; - storeCursor &= ~IsObjectStoreCursor; - objectStoreBasicContext.Session.ScanCursor(ref storeCursor, remainingCount, objStoreDbScanFuncs, validateCursor: validateCursor); - if (storeCursor != 0) - storeCursor |= IsObjectStoreCursor; - Keys.AddRange(objStoreKeys); - } + unifiedBasicContext.Session.ScanCursor(ref storeCursor, count, unifiedStoreDbScanFuncs, validateCursor: cursor != 0 && cursor != lastScanCursor); lastScanCursor = storeCursor; return true; } /// - /// Iterates over main store memory collecting expired records. - /// - internal (long, long) MainStoreExpiredKeyDeletionScan(long fromAddress, long untilAddress) - { - mainStoreExpiredKeyDeletionScanFuncs ??= new(); - mainStoreExpiredKeyDeletionScanFuncs.Initialize(this); - _ = basicContext.Session.ScanCursor(ref fromAddress, untilAddress, mainStoreExpiredKeyDeletionScanFuncs); - return (mainStoreExpiredKeyDeletionScanFuncs.deletedCount, mainStoreExpiredKeyDeletionScanFuncs.totalCount); - } - - /// - /// Iterates over object store memory collecting expired records. + /// Iterates over store memory collecting expired records. /// - internal (long, long) ObjectStoreExpiredKeyDeletionScan(long fromAddress, long untilAddress) + internal (long, long) ExpiredKeyDeletionScan(long fromAddress, long untilAddress) { - objectStoreExpiredKeyDeletionScanFuncs ??= new(); - objectStoreExpiredKeyDeletionScanFuncs.Initialize(this); - _ = objectStoreBasicContext.Session.ScanCursor(ref fromAddress, untilAddress, objectStoreExpiredKeyDeletionScanFuncs); - return (objectStoreExpiredKeyDeletionScanFuncs.deletedCount, objectStoreExpiredKeyDeletionScanFuncs.totalCount); + expiredKeyDeletionScanFuncs ??= new(); + expiredKeyDeletionScanFuncs.Initialize(this); + _ = unifiedBasicContext.Session.ScanCursor(ref fromAddress, untilAddress, expiredKeyDeletionScanFuncs); + return (expiredKeyDeletionScanFuncs.deletedCount, expiredKeyDeletionScanFuncs.totalCount); } /// - /// Iterate the contents of the main store (push-based) + /// Iterate the contents of the store (push-based) /// /// /// @@ -146,59 +118,46 @@ internal unsafe bool DbScan(ArgSlice patternB, bool allKeys, long cursor, out lo /// /// /// - internal bool IterateMainStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool validateCursor = false, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => basicContext.Session.IterateLookup(ref scanFunctions, ref cursor, untilAddress, validateCursor: validateCursor, maxAddress: maxAddress, resetCursor: false, includeTombstones: includeTombstones); - - /// - /// Iterate the contents of the main store (pull based) - /// - internal ITsavoriteScanIterator IterateMainStore() - => basicContext.Session.Iterate(); + internal bool IterateStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool validateCursor = false, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + => stringBasicContext.Session.IterateLookup(ref scanFunctions, ref cursor, untilAddress, validateCursor: validateCursor, maxAddress: maxAddress, resetCursor: false, includeTombstones: includeTombstones); /// - /// Iterate the contents of the object store (push-based) + /// Delete every live key whose hash slot is in . + /// Uses lookup-based push iteration over the unified context (no tempKv) with + /// snapshot semantics: every key live at scan-start whose slot matches is deleted. + /// Preserves the previous pull-iterator semantics — every matched live key is deleted, + /// including expired-but-not-yet-tombstoned records (no expiry filter). /// - /// - /// - /// - /// - /// - /// - /// - /// - internal bool IterateObjectStore(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, long maxAddress = long.MaxValue, bool validateCursor = false, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => objectStoreBasicContext.Session.IterateLookup(ref scanFunctions, ref cursor, untilAddress, validateCursor: validateCursor, maxAddress: maxAddress, resetCursor: false, includeTombstones: includeTombstones); + /// Hash slot set to delete. + internal void DeleteSlotKeys(HashSet slots) + { + deleteSlotKeysFuncs ??= new(); + deleteSlotKeysFuncs.Initialize(this, slots); - /// - /// Iterate the contents of the main store (pull based) - /// - internal ITsavoriteScanIterator IterateObjectStore() - => objectStoreBasicContext.Session.Iterate(); + // Snapshot semantics: ensures records RCU'd above TailAddress during the scan are + // not silently suppressed (which would leave keys behind). + _ = unifiedBasicContext.Session.IterateLookupSnapshot(ref deleteSlotKeysFuncs); + } /// - /// Get a list of the keys in the store and object store - /// when using pattern + /// Get a list of the keys in the store and object store when using pattern /// /// - internal unsafe List DBKeys(ArgSlice pattern) + internal unsafe List DBKeys(PinnedSpanByte pattern) { Keys ??= new(); Keys.Clear(); - var allKeys = *pattern.ptr == '*' && pattern.Length == 1; + var allKeys = *pattern.ToPointer() == '*' && pattern.Length == 1; - mainStoreDbKeysFuncs ??= new(); - mainStoreDbKeysFuncs.Initialize(Keys, allKeys ? null : pattern.ptr, pattern.Length); - basicContext.Session.Iterate(ref mainStoreDbKeysFuncs); + unifiedStoreDbKeysFuncs ??= IsConsistentReadSession ? new ConsistentUnifiedStoreGetDBKeys(readSessionState) : new UnifiedStoreGetDBKeys(); + unifiedStoreDbKeysFuncs.Initialize(Keys, allKeys ? null : pattern.ToPointer(), pattern.Length); - if (!objectStoreBasicContext.IsNull) - { - objStoreDbKeysFuncs ??= new(); - objStoreDbKeysFuncs.Initialize(Keys, allKeys ? null : pattern.ptr, pattern.Length, matchType: null); - objectStoreBasicContext.Session.Iterate(ref objStoreDbKeysFuncs); - } + // Snapshot semantics: emit each unique live key exactly once based on its latest in-range + // version at scan-start, even if a concurrent RCU moves the key's tail above the captured + // TailAddress during the scan. Equivalent to the legacy tempKv-backed Iterate(...). + _ = unifiedBasicContext.Session.IterateLookupSnapshot(ref unifiedStoreDbKeysFuncs); return Keys; } @@ -209,21 +168,12 @@ internal unsafe List DBKeys(ArgSlice pattern) /// internal int DbSize() { - mainStoreDbSizeFuncs ??= new(); - mainStoreDbSizeFuncs.Initialize(); + unifiedStoreDbSizeFuncs ??= new(); + unifiedStoreDbSizeFuncs.Initialize(); long cursor = 0; - basicContext.Session.ScanCursor(ref cursor, long.MaxValue, mainStoreDbSizeFuncs); - int count = mainStoreDbSizeFuncs.Count; - if (objectStoreBasicContext.Session != null) - { - objectStoreDbSizeFuncs ??= new(); - objectStoreDbSizeFuncs.Initialize(); - cursor = 0; - _ = objectStoreBasicContext.Session.ScanCursor(ref cursor, long.MaxValue, objectStoreDbSizeFuncs); - count += objectStoreDbSizeFuncs.Count; - } + unifiedBasicContext.Session.ScanCursor(ref cursor, long.MaxValue, unifiedStoreDbSizeFuncs); - return count; + return unifiedStoreDbSizeFuncs.Count; } internal static unsafe class ArrayKeyIterationFunctions @@ -245,28 +195,15 @@ internal void Initialize(List keys, byte* patternB, int length, Type mat } } - internal sealed class ObjectStoreExpiredKeyDeletionScan : ExpiredKeysBase - { - protected override bool IsExpired(ref IGarnetObject value) => value.Expiration > 0 && ObjectSessionFunctions.CheckExpiry(value); - protected override bool DeleteIfExpiredInMemory(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata) - { - var input = new ObjectInput(new RespInputHeader(GarnetObjectType.DelIfExpIm)); - var output = new GarnetObjectStoreOutput(); - return GarnetStatus.OK == storageSession.RMW_ObjectStore(ref key, ref input, ref output, ref storageSession.objectStoreBasicContext); - } - } - - internal sealed class MainStoreExpiredKeyDeletionScan : ExpiredKeysBase + internal sealed class ExpiredKeyDeletionScan : ExpiredKeysBase { - protected override bool IsExpired(ref SpanByte value) => value.MetadataSize == 8 && MainSessionFunctions.CheckExpiry(ref value); - protected override bool DeleteIfExpiredInMemory(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata) - { - var input = new RawStringInput(RespCommand.DELIFEXPIM); - return GarnetStatus.OK == storageSession.DEL_Conditional(ref key, ref input, ref storageSession.basicContext); - } + protected override bool DeleteIfExpiredInMemory(in TSourceLogRecord logRecord, + RecordMetadata recordMetadata) + => GarnetStatus.OK == storageSession.DELIFEXPIM(PinnedSpanByte.FromPinnedSpan(logRecord.Key), + ref storageSession.unifiedBasicContext); } - internal abstract class ExpiredKeysBase : IScanIteratorFunctions + internal abstract class ExpiredKeysBase : IScanIteratorFunctions { public long totalCount; public long deletedCount; @@ -275,20 +212,17 @@ internal abstract class ExpiredKeysBase : IScanIteratorFunctions this.storageSession = storageSession; - protected abstract bool IsExpired(ref TValue value); + protected abstract bool DeleteIfExpiredInMemory(in TSourceLogRecord logRecord, RecordMetadata recordMetadata) + where TSourceLogRecord : ISourceLogRecord; - protected abstract bool DeleteIfExpiredInMemory(ref TKey key, ref TValue value, RecordMetadata recordMetadata); - - public bool SingleReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public bool ConcurrentReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { totalCount++; - if (IsExpired(ref value)) + if (CheckExpiry(in logRecord)) { cursorRecordResult = CursorRecordResult.Accept; - if (DeleteIfExpiredInMemory(ref key, ref value, recordMetadata)) + if (DeleteIfExpiredInMemory(in logRecord, recordMetadata)) deletedCount++; } else @@ -309,84 +243,64 @@ public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } } - internal sealed class MainStoreGetDBKeys : IScanIteratorFunctions + internal sealed class ConsistentUnifiedStoreGetDBKeys : UnifiedStoreGetDBKeys { - private readonly GetDBKeysInfo info; - - internal MainStoreGetDBKeys() => info = new(); - - internal void Initialize(List keys, byte* patternB, int length) - => info.Initialize(keys, patternB, length); - - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); + readonly ReadSessionState readSessionState; + internal ConsistentUnifiedStoreGetDBKeys(ReadSessionState readSessionState) : base() + => this.readSessionState = readSessionState; - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public override bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) { - // TODO: A better check for "is probably a vector key" - if (key.MetadataSize == 1) - { - cursorRecordResult = CursorRecordResult.Skip; - return true; - } - - if ((info.patternB != null && !GlobUtils.Match(info.patternB, info.patternLength, key.ToPointer(), key.Length, true)) - || (value.MetadataSize == 8 && MainSessionFunctions.CheckExpiry(ref value))) - { - cursorRecordResult = CursorRecordResult.Skip; - } - else - { - cursorRecordResult = CursorRecordResult.Accept; - info.keys.Add(key.ToByteArray()); - } - return true; + readSessionState.BeforeConsistentReadKeyCallback(GarnetLog.HASH(logRecord.Key)); + var status = base.Reader(in logRecord, recordMetadata, numberOfRecords, out cursorRecordResult); + readSessionState.AfterConsistentReadKeyCallback(); + return status; } - - public bool OnStart(long beginAddress, long endAddress) => true; - public void OnStop(bool completed, long numberOfRecords) { } - public void OnException(Exception exception, long numberOfRecords) { } } - internal sealed class ObjectStoreGetDBKeys : IScanIteratorFunctions + internal class UnifiedStoreGetDBKeys : IScanIteratorFunctions { private readonly GetDBKeysInfo info; - internal ObjectStoreGetDBKeys() => info = new(); + internal UnifiedStoreGetDBKeys() => info = new(); internal void Initialize(List keys, byte* patternB, int length, Type matchType = null) => info.Initialize(keys, patternB, length, matchType); - public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public virtual bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - if (value.Expiration > 0 && ObjectSessionFunctions.CheckExpiry(value)) + if (CheckExpiry(in logRecord)) { cursorRecordResult = CursorRecordResult.Skip; return true; } + var key = logRecord.Key; if (info.patternB != null) { - fixed (byte* keyPtr = key) + bool ok; + if (logRecord.IsPinnedKey) + ok = GlobUtils.Match(info.patternB, info.patternLength, logRecord.PinnedKeyPointer, key.Length, true); + else + fixed (byte* keyPtr = key) + ok = GlobUtils.Match(info.patternB, info.patternLength, keyPtr, key.Length, true); + if (!ok) { - if (!GlobUtils.Match(info.patternB, info.patternLength, keyPtr, key.Length, true)) - { - cursorRecordResult = CursorRecordResult.Skip; - return true; - } + cursorRecordResult = CursorRecordResult.Skip; + return true; } } - if (info.matchType != null && value.GetType() != info.matchType) + if (info.matchType != null && + ((logRecord.Info.ValueIsObject && (info.matchType == typeof(string) || info.matchType != logRecord.ValueObject.GetType())) || + (!logRecord.Info.ValueIsObject && info.matchType != typeof(string)))) { cursorRecordResult = CursorRecordResult.Skip; return true; } - info.keys.Add(key); + info.keys.Add(key.ToArray()); cursorRecordResult = CursorRecordResult.Accept; return true; } @@ -404,60 +318,94 @@ internal class GetDBSizeInfo internal void Initialize() => count = 0; } - internal sealed class MainStoreGetDBSize : IScanIteratorFunctions + internal sealed class UnifiedStoreGetDBSize : IScanIteratorFunctions { private readonly GetDBSizeInfo info; internal int Count => info.count; - internal MainStoreGetDBSize() => info = new(); + internal UnifiedStoreGetDBSize() => info = new(); internal void Initialize() => info.Initialize(); - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Skip; + if (!CheckExpiry(in logRecord)) + ++info.count; + return true; + } - // TODO: Better way to ignore internal vector set elements - if (key.MetadataSize == 1) - { - return true; - } + public bool OnStart(long beginAddress, long endAddress) => true; + public void OnStop(bool completed, long numberOfRecords) { } + public void OnException(Exception exception, long numberOfRecords) { } + } + + /// + /// Lookup-based push iterator callback that deletes every live key whose hash slot is + /// in the supplied set. Cached on via the + /// deleteSlotKeysFuncs field; re-initialised per call. + /// IMPORTANT: matches the previous pull-iterator semantics — every matched live key is + /// deleted, including expired-but-not-yet-tombstoned records (no expiry filter). + /// + internal sealed class DeleteSlotKeysScan : IScanIteratorFunctions + { + private StorageSession storageSession; + private HashSet slots; - if (value.MetadataSize != 8 || !MainSessionFunctions.CheckExpiry(ref value)) + internal void Initialize(StorageSession storageSession, HashSet slots) + { + this.storageSession = storageSession; + this.slots = slots; + } + + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord + { + cursorRecordResult = CursorRecordResult.Skip; + if (slots.Contains(HashSlotUtils.HashSlot(logRecord.Key))) { - ++info.count; + _ = storageSession.DELETE(PinnedSpanByte.FromPinnedSpan(logRecord.Key), ref storageSession.unifiedBasicContext); + cursorRecordResult = CursorRecordResult.Accept; } return true; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); + public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } } - internal sealed class ObjectStoreGetDBSize : IScanIteratorFunctions + /// + /// Lookup-based push iterator callback that returns true from if + /// any live record's key hashes to a slot in the supplied set. Stops scanning on the + /// first match by returning false from . Cached on + /// via the hasKeysInSlotsFuncs field; re-initialised per call. + /// + internal sealed class HasKeysInSlotsScan : IScanIteratorFunctions { - private readonly GetDBSizeInfo info; - - internal int Count => info.count; - - internal ObjectStoreGetDBSize() => info = new(); + private List slots; + internal bool Found; - internal void Initialize() => info.Initialize(); + internal void Initialize(List slots) + { + this.slots = slots; + Found = false; + } - public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Skip; - if (value.Expiration == 0 || !ObjectSessionFunctions.CheckExpiry(value)) + if (slots.Contains(HashSlotUtils.HashSlot(logRecord.Key))) { - ++info.count; + Found = true; + return false; // early exit } return true; } - public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); + public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } public void OnException(Exception exception, long numberOfRecords) { } diff --git a/libs/server/Storage/Session/MainStore/AdvancedOps.cs b/libs/server/Storage/Session/MainStore/AdvancedOps.cs index 22a45016c16..adf340d2ef3 100644 --- a/libs/server/Storage/Session/MainStore/AdvancedOps.cs +++ b/libs/server/Storage/Session/MainStore/AdvancedOps.cs @@ -8,15 +8,12 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - sealed partial class StorageSession : IDisposable { - public GarnetStatus GET_WithPending(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, long ctx, out bool pending, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus GET_WithPending(ReadOnlySpan key, ref StringInput input, ref StringOutput output, long ctx, out bool pending, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.Read(ref key, ref input, ref output, ctx); + var status = context.Read((FixedSpanByteKey)key, ref input, ref output, ctx); if (status.IsPending) { @@ -38,8 +35,8 @@ public GarnetStatus GET_WithPending(ref SpanByte key, ref RawStringInp } } - public bool GET_CompletePending((GarnetStatus, SpanByteAndMemory)[] outputArr, bool wait, ref TContext context) - where TContext : ITsavoriteContext + public bool GET_CompletePending((GarnetStatus, StringOutput)[] outputArr, bool wait, ref TStringContext context) + where TStringContext : ITsavoriteContext { Debug.Assert(outputArr != null); @@ -62,8 +59,8 @@ public bool GET_CompletePending((GarnetStatus, SpanByteAndMemory)[] ou return ret; } - public bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait, ref TContext context) - where TContext : ITsavoriteContext + public bool GET_CompletePending(out CompletedOutputIterator completedOutputs, bool wait, ref TStringContext context) + where TStringContext : ITsavoriteContext { latencyMetrics?.Start(LatencyMetricsType.PENDING_LAT); var ret = context.CompletePendingWithOutputs(out completedOutputs, wait); @@ -71,10 +68,10 @@ public bool GET_CompletePending(out CompletedOutputIterator(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus RMW_MainStore(ReadOnlySpan key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.RMW(ref key, ref input, ref output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) CompletePendingForSession(ref status, ref output, ref context); @@ -85,36 +82,61 @@ public GarnetStatus RMW_MainStore(ref SpanByte key, ref RawStringInput return GarnetStatus.NOTFOUND; } - public GarnetStatus Read_MainStore(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus Read_MainStore(ReadOnlySpan key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.Read(ref key, ref input, ref output); + var status = context.Read((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) CompletePendingForSession(ref status, ref output, ref context); if (status.Found) - { return GarnetStatus.OK; - } - else if (status.IsCanceled) - { - // Vector Sets signal WRONGTYPE via cancellation - everything else will fall into NOTFOUND + else if (status.IsWrongType) return GarnetStatus.WRONGTYPE; - } else - { return GarnetStatus.NOTFOUND; - } } + /// + /// Specialized Read for RangeIndex stubs. Suppresses Tsavorite's automatic + /// CopyReadsToTail / CopyReadsToReadCache for this single Read by passing + /// , then calls into the standard Read pipeline. + /// + /// Why a separate API: RangeIndex performs its own controlled promotion via + /// RIPROMOTE RMW (which propagates RecordType, manages TreeHandle ownership in + /// PostCopyUpdater, and pre-stages data.bftree with proper locking). + /// Allowing Tsavorite's CTT to race with that path would (a) leave the destination + /// record without RecordType=RangeIndexRecordType (CTT does not propagate + /// RecordType), and (b) trigger PostCopyToTail-cold which takes the per-key + /// X-lock, self-deadlocking against the reader's S-lock when CopyReadsToTail is + /// enabled at the session/KV level. Keeping this on a dedicated API ensures every + /// RangeIndex stub Read goes through the suppression and other Read callers (Bitmap, + /// HLL, etc.) incur zero overhead. + /// + public GarnetStatus Read_RangeIndex(ReadOnlySpan key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + { + var readOptions = new ReadOptions { CopyOptions = ReadCopyOptions.None }; + var status = context.Read((FixedSpanByteKey)key, ref input, ref output, ref readOptions); + + if (status.IsPending) + CompletePendingForSession(ref status, ref output, ref context); + + if (status.Found) + return GarnetStatus.OK; + else if (status.IsWrongType) + return GarnetStatus.WRONGTYPE; + else + return GarnetStatus.NOTFOUND; + } public void ReadWithPrefetch(ref TBatch batch, ref TContext context, long userContext = default) - where TBatch : IReadArgBatch + where TBatch : IReadArgBatch #if NET9_0_OR_GREATER , allows ref struct #endif - where TContext : ITsavoriteContext + where TContext : ITsavoriteContext => context.ReadWithPrefetch(ref batch, userContext); } } \ No newline at end of file diff --git a/libs/server/Storage/Session/MainStore/BitmapOps.cs b/libs/server/Storage/Session/MainStore/BitmapOps.cs index 3b13ea4e551..a4f0c98b755 100644 --- a/libs/server/Storage/Session/MainStore/BitmapOps.cs +++ b/libs/server/Storage/Session/MainStore/BitmapOps.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System; +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Text; @@ -10,13 +11,10 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - sealed partial class StorageSession : IDisposable { - public unsafe GarnetStatus StringSetBit(ArgSlice key, ArgSlice offset, bool bit, out bool previous, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringSetBit(PinnedSpanByte key, PinnedSpanByte offset, bool bit, out bool previous, ref TStringContext context) + where TStringContext : ITsavoriteContext { previous = false; @@ -25,22 +23,20 @@ public unsafe GarnetStatus StringSetBit(ArgSlice key, ArgSlice offset, var setValBytes = stackalloc byte[1]; setValBytes[0] = (byte)(bit ? '1' : '0'); - var setValSlice = new ArgSlice(setValBytes, 1); + var setValSlice = PinnedSpanByte.FromPinnedPointer(setValBytes, 1); parseState.InitializeWithArguments(offset, setValSlice); - var input = new RawStringInput(RespCommand.SETBIT, ref parseState, - arg1: ParseUtils.ReadLong(ref offset)); + var input = new StringInput(RespCommand.SETBIT, ref parseState, arg1: ParseUtils.ReadLong(offset)); - SpanByteAndMemory output = new(null); - var keySp = key.SpanByte; - RMW_MainStore(ref keySp, ref input, ref output, ref context); + StringOutput output = new(); + RMW_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); return GarnetStatus.OK; } - public unsafe GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, out bool bValue, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringGetBit(PinnedSpanByte key, PinnedSpanByte offset, out bool bValue, ref TStringContext context) + where TStringContext : ITsavoriteContext { bValue = false; @@ -49,16 +45,14 @@ public unsafe GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, parseState.InitializeWithArgument(offset); - var input = new RawStringInput(RespCommand.GETBIT, ref parseState, - arg1: ParseUtils.ReadLong(ref offset)); + var input = new StringInput(RespCommand.GETBIT, ref parseState, arg1: ParseUtils.ReadLong(offset)); - SpanByteAndMemory output = new(null); - var keySp = key.SpanByte; - var status = Read_MainStore(ref keySp, ref input, ref output, ref context); + StringOutput output = new(); + var status = Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - if (status == GarnetStatus.OK && !output.IsSpanByte) + if (status == GarnetStatus.OK && !output.SpanByteAndMemory.IsSpanByte) { - fixed (byte* outputPtr = output.Memory.Memory.Span) + fixed (byte* outputPtr = output.SpanByteAndMemory.MemorySpan) { var refPtr = outputPtr; if (*refPtr == ':') @@ -67,13 +61,13 @@ public unsafe GarnetStatus StringGetBit(ArgSlice key, ArgSlice offset, bValue = *refPtr == '1'; } } - output.Memory.Dispose(); + output.SpanByteAndMemory.Memory.Dispose(); } return status; } - public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOperation bitOp, out long result) + public unsafe GarnetStatus StringBitOperation(ref StringInput input, BitmapOperation bitOp, out long result) { var maxBitmapLen = int.MinValue; var minBitmapLen = int.MaxValue; @@ -81,30 +75,41 @@ public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOp var keys = input.parseState.Parameters; var keyCount = keys.Length; - // 8 byte start pointer - // 4 byte int length - Span output = stackalloc byte[12]; var srcBitmapPtrs = stackalloc byte*[keyCount - 1]; var srcBitmapEndPtrs = stackalloc byte*[keyCount - 1]; + // Tracks heap-allocated source value buffers (used when the value is stored as overflow byte[] + // outside the log) and the pin handle held over them for the duration of BITOP execution. + // Allocated lazily and disposed in the finally block. + IMemoryOwner[] overflowOwners = null; + MemoryHandle[] overflowHandles = null; + var overflowCount = 0; + var createTransaction = false; if (txnManager.state != TxnState.Running) { createTransaction = true; Debug.Assert(txnManager.state == TxnState.None); - txnManager.SaveKeyEntryToLock(keys[0], false, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main); + txnManager.SaveKeyEntryToLock(keys[0], LockType.Exclusive); for (var i = 1; i < keys.Length; i++) - txnManager.SaveKeyEntryToLock(keys[i], false, LockType.Shared); - txnManager.Run(true); + txnManager.SaveKeyEntryToLock(keys[i], LockType.Shared); + _ = txnManager.Run(true); } // Perform under unsafe epoch control for pointer safety. - var uc = txnManager.LockableUnsafeContext; + var uc = txnManager.TransactionalUnsafeContext; try { uc.BeginUnsafe(); readFromScratch: + // Reset all per-attempt state and release any pin handles / heap buffers accumulated by the + // previous attempt before re-reading sources from scratch. + ReleaseOverflowBuffers(overflowOwners, overflowHandles, ref overflowCount); + maxBitmapLen = int.MinValue; + minBitmapLen = int.MaxValue; + var localHeadAddress = uc.Session.HeadAddress; var localReadCacheHeadAddress = uc.Session.ReadCacheHeadAddress; var keysFound = 0; @@ -112,8 +117,10 @@ public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOp for (var i = 1; i < keys.Length; i++) { var srcKey = keys[i]; - //Read srcKey - var outputBitmap = SpanByteAndMemory.FromPinnedSpan(output); + // Read srcKey. The backend populates either SpanByteAndMemory.SpanByte (inline values + // already pinned in log memory) or SpanByteAndMemory.Memory (overflow values copied to + // a heap-rented buffer that we must pin here for BITOP execution). + var outputBitmap = new StringOutput(); status = ReadWithUnsafeContext(srcKey, ref input, ref outputBitmap, localHeadAddress, localReadCacheHeadAddress, out bool epochChanged, ref uc); if (epochChanged) { @@ -124,9 +131,31 @@ public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOp if (status == GarnetStatus.NOTFOUND) continue; - var outputBitmapPtr = outputBitmap.SpanByte.ToPointer(); - var localBitmapPtr = (byte*)(nuint)(*(ulong*)outputBitmapPtr); - var localBitmapLength = *(int*)(outputBitmapPtr + 8); + byte* localBitmapPtr; + int localBitmapLength; + + if (outputBitmap.SpanByteAndMemory.IsSpanByte) + { + // Inline value: SpanByte points directly into the log buffer. + localBitmapPtr = outputBitmap.SpanByteAndMemory.SpanByte.ToPointer(); + localBitmapLength = outputBitmap.SpanByteAndMemory.SpanByte.Length; + } + else + { + // Overflow value: data was copied into a heap buffer. Pin it so that GC compaction + // cannot relocate the underlying array between now and the BITOP execution below. + var owner = outputBitmap.SpanByteAndMemory.Memory; + localBitmapLength = outputBitmap.SpanByteAndMemory.Length; + var memHandle = owner.Memory.Pin(); + + overflowOwners ??= new IMemoryOwner[keys.Length - 1]; + overflowHandles ??= new MemoryHandle[keys.Length - 1]; + overflowOwners[overflowCount] = owner; + overflowHandles[overflowCount] = memHandle; + overflowCount++; + + localBitmapPtr = (byte*)memHandle.Pointer; + } // Keep track of pointers returned from ISessionFunctions srcBitmapPtrs[keysFound] = localBitmapPtr; @@ -154,9 +183,9 @@ public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOp if (maxBitmapLen > 0) { - var dstKey = keys[0].SpanByte; - var dstBitmapSpanByte = SpanByte.FromPinnedPointer(dstBitmapPtr, maxBitmapLen); - status = SET(ref dstKey, ref dstBitmapSpanByte, ref uc); + var dstKey = keys[0]; + var dstBitmapSpanByte = PinnedSpanByte.FromPinnedPointer(dstBitmapPtr, maxBitmapLen); + status = SET(dstKey, dstBitmapSpanByte, ref uc); } } else @@ -170,32 +199,44 @@ public unsafe GarnetStatus StringBitOperation(ref RawStringInput input, BitmapOp { // Suspend Thread uc.EndUnsafe(); + // Release any overflow pin handles and heap buffers regardless of whether BITOP succeeded. + ReleaseOverflowBuffers(overflowOwners, overflowHandles, ref overflowCount); if (createTransaction) txnManager.Commit(true); } result = maxBitmapLen; return status; + + static void ReleaseOverflowBuffers(IMemoryOwner[] owners, MemoryHandle[] handles, ref int count) + { + for (var h = 0; h < count; h++) + { + handles[h].Dispose(); + owners[h].Dispose(); + } + count = 0; + } } - public GarnetStatus StringBitOperation(BitmapOperation bitOp, ArgSlice destinationKey, ArgSlice[] keys, out long result) + public GarnetStatus StringBitOperation(BitmapOperation bitOp, PinnedSpanByte destinationKey, PinnedSpanByte[] keys, out long result) { result = 0; if (destinationKey.Length == 0) return GarnetStatus.OK; - var args = new ArgSlice[keys.Length + 1]; + var args = new PinnedSpanByte[keys.Length + 1]; args[0] = destinationKey; keys.CopyTo(args, 1); parseState.InitializeWithArguments(args); - var input = new RawStringInput(RespCommand.BITOP, ref parseState); + var input = new StringInput(RespCommand.BITOP, ref parseState); return StringBitOperation(ref input, bitOp, out result); } - public unsafe GarnetStatus StringBitCount(ArgSlice key, long start, long end, bool useBitInterval, out long result, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringBitCount(PinnedSpanByte key, long start, long end, bool useBitInterval, out long result, ref TStringContext context) + where TStringContext : ITsavoriteContext { result = 0; @@ -219,55 +260,52 @@ public unsafe GarnetStatus StringBitCount(ArgSlice key, long start, lo // Use bit interval var useBitIntervalSpan = paramsSpan.Slice(paramsSpanOffset, 1); (useBitInterval ? "1"u8 : "0"u8).CopyTo(useBitIntervalSpan); - var useBitIntervalSlice = ArgSlice.FromPinnedSpan(useBitIntervalSpan); + var useBitIntervalSlice = PinnedSpanByte.FromPinnedSpan(useBitIntervalSpan); paramsSpanOffset += 1; // Start var startSpan = paramsSpan.Slice(paramsSpanOffset, startLength); - NumUtils.WriteInt64(start, startSpan); - var startSlice = ArgSlice.FromPinnedSpan(startSpan); + _ = NumUtils.WriteInt64(start, startSpan); + var startSlice = PinnedSpanByte.FromPinnedSpan(startSpan); paramsSpanOffset += startLength; // End var endSpan = paramsSpan.Slice(paramsSpanOffset, endLength); - NumUtils.WriteInt64(end, endSpan); - var endSlice = ArgSlice.FromPinnedSpan(endSpan); + _ = NumUtils.WriteInt64(end, endSpan); + var endSlice = PinnedSpanByte.FromPinnedSpan(endSpan); - SpanByteAndMemory output = new(null); + StringOutput output = new(); parseState.InitializeWithArguments(startSlice, endSlice, useBitIntervalSlice); - var input = new RawStringInput(RespCommand.BITCOUNT, ref parseState); - - scratchBufferBuilder.RewindScratchBuffer(ref paramsSlice); + var input = new StringInput(RespCommand.BITCOUNT, ref parseState); - var keySp = key.SpanByte; + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); - var status = Read_MainStore(ref keySp, ref input, ref output, ref context); + var status = Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); if (status == GarnetStatus.OK) { - if (!output.IsSpanByte) + if (!output.SpanByteAndMemory.IsSpanByte) { - fixed (byte* outputPtr = output.Memory.Memory.Span) + fixed (byte* outputPtr = output.SpanByteAndMemory.MemorySpan) { var refPtr = outputPtr; - RespReadUtils.TryReadInt64(out result, ref refPtr, refPtr + sizeof(long)); + _ = RespReadUtils.TryReadInt64(out result, ref refPtr, refPtr + sizeof(long)); } - output.Memory.Dispose(); + output.SpanByteAndMemory.Memory.Dispose(); } } return status; } - public unsafe GarnetStatus StringBitField(ArgSlice key, List commandArguments, out List result, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringBitField(PinnedSpanByte key, List commandArguments, out List result, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var input = new RawStringInput(RespCommand.BITFIELD); + var input = new StringInput(RespCommand.BITFIELD); - result = new(); - var keySp = key.SpanByte; + result = []; for (var i = 0; i < commandArguments.Count; i++) { @@ -298,40 +336,40 @@ public unsafe GarnetStatus StringBitField(ArgSlice key, List(ArgSlice key, List(ArgSlice key, List(ArgSlice key, List(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - => RMW_MainStore(ref key, ref input, ref output, ref context); + public GarnetStatus StringSetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => RMW_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - public GarnetStatus StringGetBit(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - => Read_MainStore(ref key, ref input, ref output, ref context); + public GarnetStatus StringGetBit(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - public unsafe GarnetStatus StringBitCount(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - => Read_MainStore(ref key, ref input, ref output, ref context); + public unsafe GarnetStatus StringBitCount(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - public unsafe GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - => Read_MainStore(ref key, ref input, ref output, ref context); + public unsafe GarnetStatus StringBitPosition(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - public unsafe GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringBitField(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { GarnetStatus status; if (secondaryCommand == RespCommand.GET) - status = Read_MainStore(ref key, ref input, ref output, ref context); + status = Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); else { Debug.Assert(input.header.cmd != RespCommand.BITFIELD_RO); - status = RMW_MainStore(ref key, ref input, ref output, ref context); + status = RMW_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); } return status; } - public unsafe GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus StringBitFieldReadOnly(PinnedSpanByte key, ref StringInput input, RespCommand secondaryCommand, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - GarnetStatus status = GarnetStatus.NOTFOUND; + var status = GarnetStatus.NOTFOUND; if (secondaryCommand == RespCommand.GET) - status = Read_MainStore(ref key, ref input, ref output, ref context); + status = Read_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); return status; } diff --git a/libs/server/Storage/Session/MainStore/CompletePending.cs b/libs/server/Storage/Session/MainStore/CompletePending.cs index 04f68a094ed..d6879c5a1c8 100644 --- a/libs/server/Storage/Session/MainStore/CompletePending.cs +++ b/libs/server/Storage/Session/MainStore/CompletePending.cs @@ -2,29 +2,32 @@ // Licensed under the MIT license. using System.Diagnostics; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - sealed partial class StorageSession { /// /// Handles the complete pending status for Session Store /// - /// - /// - /// - static void CompletePendingForSession(ref Status status, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + internal static void CompletePendingForSession(ref Status status, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => CompletePendingForSession(ref status, ref output, ref context, out _); + + /// + /// Handles the complete pending status for Session Store + /// + static void CompletePendingForSession(ref Status status, ref StringOutput output, ref TStringContext stringContext, out RecordMetadata recordMetadata) + where TStringContext : ITsavoriteContext { - context.CompletePendingWithOutputs(out var completedOutputs, wait: true); + stringContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); Debug.Assert(more); status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; + recordMetadata = completedOutputs.Current.RecordMetadata; more = completedOutputs.Next(); Debug.Assert(!more); completedOutputs.Dispose(); @@ -34,7 +37,7 @@ static void CompletePendingForSession(ref Status status, ref SpanByteA /// Handles the complete pending status for Session Store, without outputs. /// static void CompletePendingForSession(ref TContext context) - where TContext : ITsavoriteContext + where TContext : ITsavoriteContext => context.CompletePending(wait: true); } } \ No newline at end of file diff --git a/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs b/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs index 6ab0c12b4c0..ad27afe7b0e 100644 --- a/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs +++ b/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs @@ -3,28 +3,26 @@ using System; using System.Diagnostics; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - sealed partial class StorageSession : IDisposable { /// /// Adds all the element arguments to the HyperLogLog data structure stored at the variable name specified as key. /// - public unsafe GarnetStatus HyperLogLogAdd(ArgSlice key, string[] elements, out bool updated, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus HyperLogLogAdd(PinnedSpanByte key, string[] elements, out bool updated, ref TStringContext context) + where TStringContext : ITsavoriteContext { updated = false; parseState.Initialize(1); - var input = new RawStringInput(RespCommand.PFADD, ref parseState); + var input = new StringInput(RespCommand.PFADD, ref parseState); - var output = stackalloc byte[1]; + byte output = 0; byte pfaddUpdated = 0; foreach (var element in elements) @@ -32,19 +30,19 @@ public unsafe GarnetStatus HyperLogLogAdd(ArgSlice key, string[] eleme var elementSlice = scratchBufferBuilder.CreateArgSlice(element); parseState.SetArgument(0, elementSlice); - var o = new SpanByteAndMemory(output, 1); - var sbKey = key.SpanByte; - RMW_MainStore(ref sbKey, ref input, ref o, ref context); + var o = StringOutput.FromPinnedSpan(new Span(ref output)); + + _ = RMW_MainStore(key.ReadOnlySpan, ref input, ref o, ref context); - scratchBufferBuilder.RewindScratchBuffer(ref elementSlice); + scratchBufferBuilder.RewindScratchBuffer(elementSlice); //Invalid HLL Type - if (*output == (byte)0xFF) + if (output == (byte)0xFF) { pfaddUpdated = 0; break; } - pfaddUpdated |= *output; + pfaddUpdated |= output; } updated = pfaddUpdated > 0; @@ -54,18 +52,18 @@ public unsafe GarnetStatus HyperLogLogAdd(ArgSlice key, string[] eleme /// /// Adds one element to the HyperLogLog data structure stored at the variable name specified. /// - /// + /// /// /// /// /// /// - public GarnetStatus HyperLogLogAdd(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - => RMW_MainStore(ref key, ref input, ref output, ref context); + public GarnetStatus HyperLogLogAdd(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext + => RMW_MainStore(key.ReadOnlySpan, ref input, ref output, ref context); - public unsafe GarnetStatus HyperLogLogLength(Span keys, out long count, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus HyperLogLogLength(Span keys, out long count, ref TStringContext context) + where TStringContext : ITsavoriteContext { parseState.Initialize(keys.Length); for (var i = 0; i < keys.Length; i++) @@ -73,7 +71,7 @@ public unsafe GarnetStatus HyperLogLogLength(Span keys, out parseState.SetArgument(i, keys[i]); } - var input = new RawStringInput(RespCommand.PFCOUNT, ref parseState); + var input = new StringInput(RespCommand.PFCOUNT, ref parseState); return HyperLogLogLength(ref input, out count, out _, ref context); } @@ -87,8 +85,8 @@ public unsafe GarnetStatus HyperLogLogLength(Span keys, out /// /// /// - public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, out bool error, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus HyperLogLogLength(ref StringInput input, out long count, out bool error, ref TStringContext context) + where TStringContext : ITsavoriteContext { error = false; count = default; @@ -103,16 +101,17 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; var dstKey = input.parseState.GetArgSliceByRef(0); - txnManager.SaveKeyEntryToLock(dstKey, false, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main); + txnManager.SaveKeyEntryToLock(dstKey, LockType.Exclusive); for (var i = 1; i < input.parseState.Count; i++) { var currSrcKey = input.parseState.GetArgSliceByRef(i); - txnManager.SaveKeyEntryToLock(currSrcKey, false, LockType.Shared); + txnManager.SaveKeyEntryToLock(currSrcKey, LockType.Shared); } - txnManager.Run(true); + _ = txnManager.Run(true); } - var currLockableContext = txnManager.LockableContext; + var currTransactionalContext = txnManager.StringTransactionalContext; try { @@ -122,20 +121,24 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, sectorAlignedMemoryPoolAlignment); var srcReadBuffer = sectorAlignedMemoryHll1.GetValidPointer(); var dstReadBuffer = sectorAlignedMemoryHll2.GetValidPointer(); - var srcMergeBuffer = new SpanByteAndMemory(srcReadBuffer, hllBufferSize); - var dstMergeBuffer = new SpanByteAndMemory(dstReadBuffer, hllBufferSize); + var srcMergeBuffer = StringOutput.FromPinnedPointer(srcReadBuffer, hllBufferSize); + var dstMergeBuffer = StringOutput.FromPinnedPointer(dstReadBuffer, hllBufferSize); var isFirst = false; for (var i = 0; i < input.parseState.Count; i++) { - var currInput = new RawStringInput(RespCommand.PFCOUNT); + var currInput = new StringInput(RespCommand.PFCOUNT); - var srcKey = input.parseState.GetArgSliceByRef(i).SpanByte; + var srcKey = input.parseState.GetArgSliceByRef(i); - var status = GET(ref srcKey, ref currInput, ref srcMergeBuffer, ref currLockableContext); + var status = GET(srcKey, ref currInput, ref srcMergeBuffer, ref currTransactionalContext); // Handle case merging source key does not exist if (status == GarnetStatus.NOTFOUND) continue; + // The PFCOUNT/PFMERGE backend is contracted to populate SpanByte (sector-aligned native + // memory we passed in) and never overflow to heap Memory. Assert the contract so any + // future regression is caught immediately rather than silently returning garbage. + Debug.Assert(srcMergeBuffer.SpanByteAndMemory.IsSpanByte, "PFCOUNT backend must populate SpanByte"); // Invalid Type if (*(long*)srcReadBuffer == -1) { @@ -143,8 +146,8 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, break; } - var sbSrcHLL = srcMergeBuffer.SpanByte; - var sbDstHLL = dstMergeBuffer.SpanByte; + var sbSrcHLL = srcMergeBuffer.SpanByteAndMemory.SpanByte; + var sbDstHLL = dstMergeBuffer.SpanByteAndMemory.SpanByte; var srcHLL = sbSrcHLL.ToPointer(); var dstHLL = sbDstHLL.ToPointer(); @@ -153,13 +156,13 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, { isFirst = true; if (i == input.parseState.Count - 1) - count = HyperLogLog.DefaultHLL.Count(srcMergeBuffer.SpanByte.ToPointer()); + count = HyperLogLog.DefaultHLL.Count(srcMergeBuffer.SpanByteAndMemory.SpanByte.ToPointer()); else Buffer.MemoryCopy(srcHLL, dstHLL, sbSrcHLL.Length, sbSrcHLL.Length); continue; } - HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, sbDstHLL.Length); + _ = HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, sbDstHLL.Length); if (i == input.parseState.Count - 1) { @@ -182,7 +185,7 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, /// /// /// - public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool error) + public unsafe GarnetStatus HyperLogLogMerge(ref StringInput input, out bool error) { error = false; @@ -195,39 +198,46 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main); var dstKey = input.parseState.GetArgSliceByRef(0); - txnManager.SaveKeyEntryToLock(dstKey, false, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(dstKey, LockType.Exclusive); for (var i = 1; i < input.parseState.Count; i++) { var currSrcKey = input.parseState.GetArgSliceByRef(i); - txnManager.SaveKeyEntryToLock(currSrcKey, false, LockType.Shared); + txnManager.SaveKeyEntryToLock(currSrcKey, LockType.Shared); } - txnManager.Run(true); + _ = txnManager.Run(true); } - var currLockableContext = txnManager.LockableContext; + var currTransactionalContext = txnManager.StringTransactionalContext; try { sectorAlignedMemoryHll1 ??= new SectorAlignedMemory(hllBufferSize + sectorAlignedMemoryPoolAlignment, sectorAlignedMemoryPoolAlignment); var readBuffer = sectorAlignedMemoryHll1.GetValidPointer(); - var dstKey = input.parseState.GetArgSliceByRef(0).SpanByte; + var dstKey = input.parseState.GetArgSliceByRef(0); for (var i = 1; i < input.parseState.Count; i++) { #region readSrcHLL - var currInput = new RawStringInput(RespCommand.PFMERGE); + var currInput = new StringInput(RespCommand.PFMERGE); - var mergeBuffer = new SpanByteAndMemory(readBuffer, hllBufferSize); - var srcKey = input.parseState.GetArgSliceByRef(i).SpanByte; + StringOutput mergeBuffer = default; + mergeBuffer.SpanByteAndMemory = SpanByteAndMemory.FromPinnedPointer(readBuffer, hllBufferSize); + var srcKey = input.parseState.GetArgSliceByRef(i); - var status = GET(ref srcKey, ref currInput, ref mergeBuffer, ref currLockableContext); + var status = GET(srcKey, ref currInput, ref mergeBuffer, ref currTransactionalContext); // Handle case merging source key does not exist if (status == GarnetStatus.NOTFOUND) continue; + // The PFCOUNT/PFMERGE backend is contracted to populate SpanByte (sector-aligned native + // memory we passed in) and never overflow to heap Memory. Assert the contract so any + // future regression is caught immediately rather than silently returning garbage. + Debug.Assert(mergeBuffer.SpanByteAndMemory.IsSpanByte, "PFMERGE backend must populate SpanByte"); + // Invalid Type if (*(long*)readBuffer == -1) { @@ -239,12 +249,12 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e #region mergeToDst - var mergeSlice = new ArgSlice(ref mergeBuffer.SpanByte); + var mergeSlice = mergeBuffer.SpanByteAndMemory.SpanByte; parseState.InitializeWithArgument(mergeSlice); currInput.parseState = parseState; - SET_Conditional(ref dstKey, ref currInput, ref mergeBuffer, ref currLockableContext); + SET_Conditional(dstKey, ref currInput, ref mergeBuffer, ref currTransactionalContext); #endregion } diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index 68ebfca6c84..d66dd2d3cb2 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -10,19 +10,13 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { - public GarnetStatus GET(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus GET(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { long ctx = default; - var status = context.Read(ref key, ref input, ref output, ctx); + var status = context.Read((FixedSpanByteKey)key, ref input, ref output, ctx); if (status.IsPending) { @@ -36,7 +30,7 @@ public GarnetStatus GET(ref SpanByte key, ref RawStringInput input, re incr_session_found(); return GarnetStatus.OK; } - else if (status.IsCanceled) + else if (status.IsWrongType) { return GarnetStatus.WRONGTYPE; } @@ -47,14 +41,11 @@ public GarnetStatus GET(ref SpanByte key, ref RawStringInput input, re } } - public unsafe GarnetStatus ReadWithUnsafeContext(ArgSlice key, ref RawStringInput input, ref SpanByteAndMemory output, long localHeadAddress, long localReadCacheHeadAddress, out bool epochChanged, ref TContext context) - where TContext : ITsavoriteContext, IUnsafeContext + public unsafe GarnetStatus ReadWithUnsafeContext(PinnedSpanByte key, ref StringInput input, ref StringOutput output, long localHeadAddress, long localReadCacheHeadAddress, out bool epochChanged, ref TStringContext context) + where TStringContext : ITsavoriteContext, IUnsafeContext { - var _key = key.SpanByte; - long ctx = default; - epochChanged = false; - var status = context.Read(ref _key, ref Unsafe.AsRef(in input), ref output, ctx); + var status = context.Read((FixedSpanByteKey)key, ref Unsafe.AsRef(in input), ref output, userContext: default); if (status.IsPending) { @@ -87,49 +78,50 @@ public unsafe GarnetStatus ReadWithUnsafeContext(ArgSlice key, ref Raw return GarnetStatus.OK; } - public unsafe GarnetStatus GET(ArgSlice key, out ArgSlice value, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus GET(PinnedSpanByte key, out PinnedSpanByte value, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var input = new RawStringInput(RespCommand.GET); + var input = new StringInput(RespCommand.GET); value = default; - var _key = key.SpanByte; - var _output = new SpanByteAndMemory { SpanByte = scratchBufferBuilder.ViewRemainingArgSlice().SpanByte }; + var _output = new StringOutput(new SpanByteAndMemory { SpanByte = scratchBufferAllocator.ViewRemainingArgSlice() }); - var ret = GET(ref _key, ref input, ref _output, ref context); + var ret = GET(key, ref input, ref _output, ref context); if (ret == GarnetStatus.OK) { - if (!_output.IsSpanByte) + if (!_output.SpanByteAndMemory.IsSpanByte) { - value = scratchBufferBuilder.FormatScratch(0, _output.AsReadOnlySpan()); - _output.Memory.Dispose(); + // Output overflowed to heap Memory — copy to SBA and dispose + value = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.ReadOnlySpan); + _output.SpanByteAndMemory.Memory.Dispose(); } else { - value = scratchBufferBuilder.CreateArgSlice(_output.Length); + // Output fit in SBA's remaining space — just claim it (zero copy) + value = scratchBufferAllocator.CreateArgSlice(_output.SpanByteAndMemory.Length); } } return ret; } - public unsafe GarnetStatus GET(ArgSlice key, out MemoryResult value, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus GET(PinnedSpanByte key, out MemoryResult value, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var input = new RawStringInput(RespCommand.GET, arg1: -1); + var input = new StringInput(RespCommand.GET); - var _key = key.SpanByte; - var _output = new SpanByteAndMemory(); + var _output = new StringOutput(); - var ret = GET(ref _key, ref input, ref _output, ref context); - value = new MemoryResult(_output.Memory, _output.Length); + var ret = GET(key, ref input, ref _output, ref context); + value = new MemoryResult(_output.SpanByteAndMemory.Memory, _output.SpanByteAndMemory.Length); return ret; } - public GarnetStatus GET(byte[] key, out GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus GET(PinnedSpanByte key, out ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - long ctx = default; - var status = objectContext.Read(key, out output, ctx); + ObjectInput input = default; + output = default; + var status = objectContext.Read((FixedSpanByteKey)key, ref input, ref output, userContext: default); if (status.IsPending) { @@ -150,10 +142,10 @@ public GarnetStatus GET(byte[] key, out GarnetObjectStoreOutput } } - public unsafe GarnetStatus GETEX(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus GETEX(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.RMW(ref key, ref input, ref output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) { @@ -181,27 +173,13 @@ public unsafe GarnetStatus GETEX(ref SpanByte key, ref RawStringInput /// Span to allocate the output of the operation /// Basic Context of the store /// Operation status - public unsafe GarnetStatus GETDEL(ArgSlice key, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus GETDEL(PinnedSpanByte key, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var _key = key.SpanByte; - return GETDEL(ref _key, ref output, ref context); - } + var input = new StringInput(RespCommand.GETDEL); - /// - /// GETDEL command - Gets the value corresponding to the given key and deletes the key. - /// - /// The key to get the value for. - /// Span to allocate the output of the operation - /// Basic Context of the store - /// Operation status - public unsafe GarnetStatus GETDEL(ref SpanByte key, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext - { - var input = new RawStringInput(RespCommand.GETDEL); - - var status = context.RMW(ref key, ref input, ref output); - Debug.Assert(output.IsSpanByte); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); if (status.IsPending) CompletePendingForSession(ref status, ref output, ref context); @@ -209,10 +187,10 @@ public unsafe GarnetStatus GETDEL(ref SpanByte key, ref SpanByteAndMem return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } - public unsafe GarnetStatus GETRANGE(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus GETRANGE(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.Read(ref key, ref input, ref output); + var status = context.Read((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) { @@ -233,137 +211,28 @@ public unsafe GarnetStatus GETRANGE(ref SpanByte key, ref RawStringInp } } - - /// - /// Returns the remaining time to live of a key that has a timeout. - /// - /// - /// - /// The key to get the remaining time to live in the store. - /// The store to operate on - /// Span to allocate the output of the operation - /// Basic Context of the store - /// Object Context of the store - /// when true the command to execute is PTTL. - /// - public unsafe GarnetStatus TTL(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output, ref TContext context, ref TObjectContext objectContext, bool milliseconds = false) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext + public GarnetStatus SET(PinnedSpanByte key, PinnedSpanByte value, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var cmd = milliseconds ? RespCommand.PTTL : RespCommand.TTL; - var input = new RawStringInput(cmd); - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var status = context.Read(ref key, ref input, ref output); - - if (status.IsPending) - { - StartPendingMetrics(); - CompletePendingForSession(ref status, ref output, ref context); - StopPendingMetrics(); - } - - if (status.Found) return GarnetStatus.OK; - } - - if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - var header = new RespInputHeader(milliseconds ? GarnetObjectType.PTtl : GarnetObjectType.Ttl); - var objInput = new ObjectInput(header); - - var keyBA = key.ToByteArray(); - var objO = new GarnetObjectStoreOutput(output); - var status = objectContext.Read(ref keyBA, ref objInput, ref objO); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref objO, ref objectContext); - - if (status.Found) - { - output = objO.SpanByteAndMemory; - return GarnetStatus.OK; - } - } - return GarnetStatus.NOTFOUND; + var status = context.Upsert((FixedSpanByteKey)key, value.ReadOnlySpan); + return status.IsWrongType ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; } - /// - /// Get the absolute Unix timestamp at which the given key will expire. - /// - /// - /// - /// The key to get the Unix timestamp. - /// The store to operate on - /// Span to allocate the output of the operation - /// Basic Context of the store - /// Object Context of the store - /// when true the command to execute is PEXPIRETIME. - /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds or milliseconds at which the given key will expire. - public unsafe GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output, ref TContext context, ref TObjectContext objectContext, bool milliseconds = false) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext + public GarnetStatus SET(PinnedSpanByte key, ref StringInput input, PinnedSpanByte value, ref TStringContext context) + where TStringContext : ITsavoriteContext { - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var cmd = milliseconds ? RespCommand.PEXPIRETIME : RespCommand.EXPIRETIME; - var input = new RawStringInput(cmd); - var status = context.Read(ref key, ref input, ref output); - - if (status.IsPending) - { - StartPendingMetrics(); - CompletePendingForSession(ref status, ref output, ref context); - StopPendingMetrics(); - } - - if (status.Found) return GarnetStatus.OK; - } - - if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - var type = milliseconds ? GarnetObjectType.PExpireTime : GarnetObjectType.ExpireTime; - var header = new RespInputHeader(type); - var input = new ObjectInput(header); - - var keyBA = key.ToByteArray(); - var objO = new GarnetObjectStoreOutput(output); - var status = objectContext.Read(ref keyBA, ref input, ref objO); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref objO, ref objectContext); - - if (status.Found) - { - output = objO.SpanByteAndMemory; - return GarnetStatus.OK; - } - } - return GarnetStatus.NOTFOUND; - } - - public GarnetStatus SET(ref SpanByte key, ref SpanByte value, ref TContext context) - where TContext : ITsavoriteContext - { - context.Upsert(ref key, ref value); - return GarnetStatus.OK; - } - - public GarnetStatus SET(ref SpanByte key, ref RawStringInput input, ref SpanByte value, ref TContext context) - where TContext : ITsavoriteContext - { - var output = new SpanByteAndMemory(); - context.Upsert(ref key, ref input, ref value, ref output); - return GarnetStatus.OK; + var output = new StringOutput(); + var status = context.Upsert((FixedSpanByteKey)key, ref input, value.ReadOnlySpan, ref output); + return status.IsWrongType ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; } - public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input, ref TStringContext context) + where TStringContext : ITsavoriteContext { byte* pbOutput = stackalloc byte[8]; - var o = new SpanByteAndMemory(pbOutput, 8); + var o = StringOutput.FromPinnedPointer(pbOutput, 8); - var status = context.RMW(ref key, ref input, ref o); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref o); if (status.IsPending) { @@ -385,24 +254,24 @@ public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawSt } - public unsafe GarnetStatus DEL_Conditional(ref SpanByte key, ref RawStringInput input, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus DEL_Conditional(PinnedSpanByte key, ref StringInput input, ref TStringContext context) + where TStringContext : ITsavoriteContext { - Debug.Assert(input.header.cmd is RespCommand.DELIFGREATER or RespCommand.DELIFEXPIM); + Debug.Assert(input.header.cmd is RespCommand.DELIFGREATER); - byte* pbOutput = stackalloc byte[8]; - var o = new SpanByteAndMemory(pbOutput, 8); - var status = context.RMW(ref key, ref input, ref o); + Span outputSpan = stackalloc byte[8]; + var output = StringOutput.FromPinnedSpan(outputSpan); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) { StartPendingMetrics(); - CompletePendingForSession(ref status, ref o, ref context); + CompletePendingForSession(ref status, ref output, ref context); StopPendingMetrics(); } // Deletions in RMW are done by expiring the record, hence we use expiration as the indicator of success. - if (status.Expired) + if (status.IsExpired) { incr_session_found(); return GarnetStatus.OK; @@ -416,10 +285,10 @@ public unsafe GarnetStatus DEL_Conditional(ref SpanByte key, ref RawSt } } - public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus SET_Conditional(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.RMW(ref key, ref input, ref output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) { @@ -440,8 +309,8 @@ public unsafe GarnetStatus SET_Conditional(ref SpanByte key, ref RawSt } } - internal GarnetStatus MSET_Conditional(ref RawStringInput input, ref TContext ctx) - where TContext : ITsavoriteContext + internal GarnetStatus MSET_Conditional(ref StringInput input, ref TStringContext ctx) + where TStringContext : ITsavoriteContext { var error = false; var count = input.parseState.Count; @@ -453,21 +322,21 @@ internal GarnetStatus MSET_Conditional(ref RawStringInput input, ref T for (var i = 0; i < count; i += 2) { var srcKey = input.parseState.GetArgSliceByRef(i); - txnManager.SaveKeyEntryToLock(srcKey, false, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(srcKey, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(srcKey, LockType.Exclusive); } txnManager.Run(true); } - var context = txnManager.LockableContext; - var objContext = txnManager.ObjectStoreLockableContext; + var context = txnManager.StringTransactionalContext; + var unifiedContext = txnManager.UnifiedTransactionalContext; try { for (var i = 0; i < count; i += 2) { var srcKey = input.parseState.GetArgSliceByRef(i); - var status = EXISTS(srcKey, StoreType.All, ref context, ref objContext); + var status = EXISTS(srcKey, ref unifiedContext); if (status != GarnetStatus.NOTFOUND) { count = 0; @@ -491,83 +360,68 @@ internal GarnetStatus MSET_Conditional(ref RawStringInput input, ref T return error ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } - public GarnetStatus SET(ArgSlice key, ArgSlice value, ref TContext context) - where TContext : ITsavoriteContext - { - var _key = key.SpanByte; - var _value = value.SpanByte; - return SET(ref _key, ref _value, ref context); - } - - public GarnetStatus SET(byte[] key, IGarnetObject value, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SET(PinnedSpanByte key, IGarnetObject value, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - objectContext.Upsert(key, value); + objectContext.Upsert((FixedSpanByteKey)key, value); return GarnetStatus.OK; } - public GarnetStatus SET(ArgSlice key, Memory value, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus SET(PinnedSpanByte key, Memory value, ref TStringContext context) // TODO are memory overloads needed? + where TStringContext : ITsavoriteContext { - var _key = key.SpanByte; unsafe { fixed (byte* ptr = value.Span) - { - var _value = SpanByte.FromPinnedPointer(ptr, value.Length); - return SET(ref _key, ref _value, ref context); - } + context.Upsert((FixedSpanByteKey)key, new ReadOnlySpan(ptr, value.Length)); } + return GarnetStatus.OK; } - public unsafe GarnetStatus SETEX(ArgSlice key, ArgSlice value, ArgSlice expiryMs, ref TContext context) - where TContext : ITsavoriteContext - => SETEX(key, value, TimeSpan.FromMilliseconds(NumUtils.ReadInt64(expiryMs.Length, expiryMs.ptr)), ref context); + public unsafe GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, PinnedSpanByte expiryMs, ref TStringContext context) + where TStringContext : ITsavoriteContext + => SETEX(key, value, TimeSpan.FromMilliseconds(NumUtils.ReadInt64(expiryMs.Length, expiryMs.ToPointer())), ref context); - public GarnetStatus SETEX(ArgSlice key, ArgSlice value, TimeSpan expiry, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus SETEX(PinnedSpanByte key, PinnedSpanByte value, TimeSpan expiry, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var _key = key.SpanByte; - var valueSB = scratchBufferBuilder.FormatScratch(sizeof(long), value).SpanByte; - valueSB.ExtraMetadata = DateTimeOffset.UtcNow.Ticks + expiry.Ticks; - return SET(ref _key, ref valueSB, ref context); + var input = new StringInput(RespCommand.SETEX, ref parseState, arg1: DateTimeOffset.UtcNow.Ticks + expiry.Ticks); + return SET(key, ref input, value, ref context); } /// /// APPEND command - appends value at the end of existing string /// - /// Context type + /// Context type /// Key whose value is to be appended /// Value to be appended /// Length of updated value /// Store context /// Operation status - public unsafe GarnetStatus APPEND(ArgSlice key, ArgSlice value, ref ArgSlice output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus APPEND(PinnedSpanByte key, PinnedSpanByte value, ref PinnedSpanByte output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var _key = key.SpanByte; - var _output = new SpanByteAndMemory(output.SpanByte); + var _output = new StringOutput(new SpanByteAndMemory(output)); parseState.InitializeWithArgument(value); + var input = new StringInput(RespCommand.APPEND, ref parseState); - var input = new RawStringInput(RespCommand.APPEND, ref parseState); - - return APPEND(ref _key, ref input, ref _output, ref context); + return APPEND(key, ref input, ref _output, ref context); } /// /// APPEND command - appends value at the end of existing string /// - /// Context type + /// Context type /// Key whose value is to be appended /// Input for main store /// Length of updated value /// Store context /// Operation status - public unsafe GarnetStatus APPEND(ref SpanByte key, ref RawStringInput input, ref SpanByteAndMemory output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus APPEND(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var status = context.RMW(ref key, ref input, ref output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) { StartPendingMetrics(); @@ -575,623 +429,61 @@ public unsafe GarnetStatus APPEND(ref SpanByte key, ref RawStringInput StopPendingMetrics(); } - Debug.Assert(output.IsSpanByte); + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); return GarnetStatus.OK; } - public GarnetStatus DELETE(ArgSlice key, StoreType storeType, ref TContext context, ref TObjectContext objectContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - var _key = key.SpanByte; - return DELETE(ref _key, storeType, ref context, ref objectContext); - } - - public GarnetStatus DELETE(ref SpanByte key, StoreType storeType, ref TContext context, ref TObjectContext objectContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - var found = false; - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var status = context.Delete(ref key); - if (status.IsCanceled) - { - // Might be a vector set - status = vectorManager.TryDeleteVectorSet(this, ref key, out _); - } - - Debug.Assert(!status.IsPending); - if (status.Found) found = true; - } - - if (!objectStoreBasicContext.IsNull && (storeType == StoreType.Object || storeType == StoreType.All)) - { - var keyBA = key.ToByteArray(); - var status = objectContext.Delete(ref keyBA); - Debug.Assert(!status.IsPending); - if (status.Found) found = true; - } - - return found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; - } - - public unsafe GarnetStatus DELETE(byte[] key, StoreType storeType, ref TContext context, ref TObjectContext objectContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - bool found = false; - - if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - var status = objectContext.Delete(key); - if (status.IsCanceled) - { - // Might be a vector set - fixed (byte* keyPtr = key) - { - SpanByte keySpan = new(key.Length, (nint)keyPtr); - status = vectorManager.TryDeleteVectorSet(this, ref keySpan, out _); - } - - if (status.Found) found = true; - } - - Debug.Assert(!status.IsPending); - if (status.Found) found = true; - } - - if (!found && (storeType == StoreType.Main || storeType == StoreType.All)) - { - unsafe - { - fixed (byte* ptr = key) - { - var keySB = SpanByte.FromPinnedPointer(ptr, key.Length); - var status = context.Delete(ref keySB); - Debug.Assert(!status.IsPending); - if (status.Found) found = true; - } - } - } - return found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; - } - - public unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, bool withEtag) - { - return RENAME(oldKeySlice, newKeySlice, storeType, false, out _, withEtag); - } - - /// - /// Renames key to newkey if newkey does not yet exist. It returns an error when key does not exist. - /// - /// The old key to be renamed. - /// The new key name. - /// The type of store to perform the operation on. - /// - public unsafe GarnetStatus RENAMENX(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, out int result, bool withEtag) - { - return RENAME(oldKeySlice, newKeySlice, storeType, true, out result, withEtag); - } - - private unsafe GarnetStatus RENAME(ArgSlice oldKeySlice, ArgSlice newKeySlice, StoreType storeType, bool isNX, out int result, bool withEtag) - { - RawStringInput input = default; - var returnStatus = GarnetStatus.NOTFOUND; - result = -1; - - // If same name check return early. - if (oldKeySlice.ReadOnlySpan.SequenceEqual(newKeySlice.ReadOnlySpan)) - { - result = 1; - return GarnetStatus.OK; - } - - var createTransaction = false; - if (txnManager.state != TxnState.Running) - { - createTransaction = true; - txnManager.SaveKeyEntryToLock(oldKeySlice, false, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(newKeySlice, false, LockType.Exclusive); - txnManager.Run(true); - } - - var context = txnManager.LockableContext; - var objectContext = txnManager.ObjectStoreLockableContext; - var oldKey = oldKeySlice.SpanByte; - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - try - { - var newKey = newKeySlice.SpanByte; - - var o = new SpanByteAndMemory(); - var status = GET(ref oldKey, ref input, ref o, ref context); - - if (status == GarnetStatus.OK) - { - Debug.Assert(!o.IsSpanByte); - var memoryHandle = o.Memory.Memory.Pin(); - var ptrVal = (byte*)memoryHandle.Pointer; - - RespReadUtils.TryReadUnsignedLengthHeader(out var headerLength, ref ptrVal, ptrVal + o.Length); - - // Find expiration time of the old key - var expireSpan = new SpanByteAndMemory(); - var ttlStatus = TTL(ref oldKey, storeType, ref expireSpan, ref context, ref objectContext, true); - - if (ttlStatus == GarnetStatus.OK && !expireSpan.IsSpanByte) - { - var newValSlice = new ArgSlice(ptrVal, headerLength); - - using var expireMemoryHandle = expireSpan.Memory.Memory.Pin(); - var expirePtrVal = (byte*)expireMemoryHandle.Pointer; - RespReadUtils.TryReadInt64(out var expireTimeMs, ref expirePtrVal, expirePtrVal + expireSpan.Length, out var _); - - input = isNX ? new RawStringInput(RespCommand.SETEXNX) : new RawStringInput(RespCommand.SET); - - // If the key has an expiration, set the new key with the expiration - if (expireTimeMs > 0) - { - if (!withEtag && !isNX) - { - SETEX(newKeySlice, newValSlice, TimeSpan.FromMilliseconds(expireTimeMs), ref context); - } - else - { - // Move payload forward to make space for RespInputHeader and Metadata - parseState.InitializeWithArgument(newValSlice); - input.parseState = parseState; - input.arg1 = DateTimeOffset.UtcNow.Ticks + TimeSpan.FromMilliseconds(expireTimeMs).Ticks; - - if (withEtag) - { - input.header.SetWithEtagFlag(); - } - - var setStatus = SET_Conditional(ref newKey, ref input, ref context); - - if (isNX) - { - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; - returnStatus = GarnetStatus.OK; - } - } - } - else if (expireTimeMs == -1) // Its possible to have expireTimeMs as 0 (Key expired or will be expired now) or -2 (Key does not exist), in those cases we don't SET the new key - { - if (!withEtag && !isNX) - { - var value = newValSlice.SpanByte; - SET(ref newKey, ref value, ref context); - } - else - { - // Build parse state - parseState.InitializeWithArgument(newValSlice); - input.parseState = parseState; - - if (withEtag) - { - input.header.SetWithEtagFlag(); - } - - var setStatus = SET_Conditional(ref newKey, ref input, ref context); - - if (isNX) - { - // For SET NX `NOTFOUND` means the operation succeeded - result = setStatus == GarnetStatus.NOTFOUND ? 1 : 0; - returnStatus = GarnetStatus.OK; - } - } - } - - expireSpan.Memory.Dispose(); - memoryHandle.Dispose(); - o.Memory.Dispose(); - - // Delete the old key only when SET NX succeeded - if (isNX && result == 1) - { - DELETE(ref oldKey, StoreType.Main, ref context, ref objectContext); - } - else if (!isNX) - { - // Delete the old key - DELETE(ref oldKey, StoreType.Main, ref context, ref objectContext); - - returnStatus = GarnetStatus.OK; - } - } - } - } - finally - { - if (createTransaction) - txnManager.Commit(true); - } - } - - if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - createTransaction = false; - if (txnManager.state != TxnState.Running) - { - txnManager.SaveKeyEntryToLock(oldKeySlice, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(newKeySlice, true, LockType.Exclusive); - txnManager.Run(true); - createTransaction = true; - } - - try - { - byte[] oldKeyArray = oldKeySlice.ToArray(); - var status = GET(oldKeyArray, out var value, ref objectContext); - - if (status == GarnetStatus.OK) - { - var valObj = value.GarnetObject; - byte[] newKeyArray = newKeySlice.ToArray(); - - returnStatus = GarnetStatus.OK; - var canSetAndDelete = true; - if (isNX) - { - // Not using EXISTS method to avoid new allocation of Array for key - var getNewStatus = GET(newKeyArray, out _, ref objectContext); - canSetAndDelete = getNewStatus == GarnetStatus.NOTFOUND; - } - - if (canSetAndDelete) - { - // valObj already has expiration time, so no need to write expiration logic here - SET(newKeyArray, valObj, ref objectContext); - - // Delete the old key - DELETE(oldKeyArray, StoreType.Object, ref context, ref objectContext); - - result = 1; - } - else - { - result = 0; - } - } - } - finally - { - if (createTransaction) - txnManager.Commit(true); - } - } - return returnStatus; - } - /// - /// Returns if key is an existing one in the store. + /// Deletes a key from the main store context. /// - /// - /// /// The name of the key to use in the operation - /// The store to operate on. /// Basic context for the main store. - /// Object context for the object store. - /// - public GarnetStatus EXISTS(ArgSlice key, StoreType storeType, ref TContext context, ref TObjectContext objectContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - var status = GarnetStatus.NOTFOUND; - RawStringInput input = default; - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var _key = key.SpanByte; - var _output = new SpanByteAndMemory { SpanByte = scratchBufferBuilder.ViewRemainingArgSlice().SpanByte }; - status = GET(ref _key, ref input, ref _output, ref context); - - if (status == GarnetStatus.OK) - { - if (!_output.IsSpanByte) - _output.Memory.Dispose(); - return status; - } - } - - if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - status = GET(key.ToArray(), out _, ref objectContext); - } - - return status; - } - - /// - /// Set a timeout on key - /// - /// - /// - /// The key to set the timeout on. - /// Milliseconds value for the timeout. - /// True when the timeout was properly set. - /// The store to operate on. - /// >Flags to use for the operation. - /// Basic context for the main store. - /// Object context for the object store. - /// - public unsafe GarnetStatus EXPIRE(ArgSlice key, ArgSlice expiryMs, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - => EXPIRE(key, TimeSpan.FromMilliseconds(NumUtils.ReadInt64(expiryMs.Length, expiryMs.ptr)), out timeoutSet, storeType, expireOption, ref context, ref objectStoreContext); - - /// - /// Set a timeout on key. - /// - /// - /// - /// The key to set the timeout on. - /// Input for the main store - /// True when the timeout was properly set. - /// The store to operate on. - /// Basic context for the main store - /// Object context for the object store - /// - public unsafe GarnetStatus EXPIRE(ArgSlice key, ref RawStringInput input, out bool timeoutSet, StoreType storeType, ref TContext context, ref TObjectContext objectStoreContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - var rmwOutput = stackalloc byte[ObjectOutputHeader.Size]; - var output = new SpanByteAndMemory(SpanByte.FromPinnedPointer(rmwOutput, ObjectOutputHeader.Size)); - timeoutSet = false; - - var found = false; - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var _key = key.SpanByte; - var status = context.RMW(ref _key, ref input, ref output); - - if (status.IsPending) - CompletePendingForSession(ref status, ref output, ref context); - if (status.Found) found = true; - } - - if (!found && (storeType == StoreType.Object || storeType == StoreType.All) && - !objectStoreBasicContext.IsNull) - { - var header = new RespInputHeader(GarnetObjectType.Expire); - - // Re-encode expiration and expiration option as two integers instead of a long - var expirationWithOption = new ExpirationWithOption(input.arg1); - - var objInput = new ObjectInput(header, arg1: expirationWithOption.WordHead, arg2: expirationWithOption.WordTail); - - // Retry on object store - var objOutput = new GarnetObjectStoreOutput(output); - var keyBytes = key.ToArray(); - var status = objectStoreContext.RMW(ref keyBytes, ref objInput, ref objOutput); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref objOutput, ref objectStoreContext); - if (status.Found) found = true; - - output = objOutput.SpanByteAndMemory; - } - - Debug.Assert(output.IsSpanByte); - if (found) timeoutSet = ((ObjectOutputHeader*)output.SpanByte.ToPointer())->result1 == 1; - - return found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; - } - - - - /// - /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970). - /// - /// - /// - /// The key to set the timeout on. - /// Absolute Unix timestamp - /// True when the timeout was properly set. - /// The store to operate on. - /// Flags to use for the operation. - /// Basic context for the main store - /// Object context for the object store - /// When true, is treated as milliseconds else seconds - /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND - public unsafe GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, bool milliseconds = false) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - return EXPIRE(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectStoreContext, milliseconds ? RespCommand.PEXPIREAT : RespCommand.EXPIREAT); - } - - /// - /// Set a timeout on key. - /// - /// - /// - /// The key to set the timeout on. - /// The timespan value to set the expiration for. - /// True when the timeout was properly set. - /// The store to operate on. - /// Flags to use for the operation. - /// Basic context for the main store - /// Object context for the object store - /// When true the command executed is PEXPIRE, expire by default. - /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND - public unsafe GarnetStatus EXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, bool milliseconds = false) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - return EXPIRE(key, (long)(milliseconds ? expiry.TotalMilliseconds : expiry.TotalSeconds), out timeoutSet, storeType, expireOption, - ref context, ref objectStoreContext, milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE); - } - - /// - /// Set a timeout on key. - /// - /// - /// - /// The key to set the timeout on. - /// The timespan value to set the expiration for. - /// True when the timeout was properly set. - /// The store to operate on. - /// Flags to use for the operation. - /// Basic context for the main store - /// Object context for the object store - /// The current RESP command /// - public unsafe GarnetStatus EXPIRE(ArgSlice key, long expiration, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, RespCommand respCommand) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext + public GarnetStatus DELETE_MainStore(PinnedSpanByte key, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var rmwOutput = stackalloc byte[ObjectOutputHeader.Size]; - var output = new SpanByteAndMemory(SpanByte.FromPinnedPointer(rmwOutput, ObjectOutputHeader.Size)); - timeoutSet = false; - var found = false; - - // Convert to expiration time in ticks - var expirationTimeInTicks = respCommand switch - { - RespCommand.EXPIRE => DateTimeOffset.UtcNow.AddSeconds(expiration).UtcTicks, - RespCommand.PEXPIRE => DateTimeOffset.UtcNow.AddMilliseconds(expiration).UtcTicks, - RespCommand.EXPIREAT => ConvertUtils.UnixTimestampInSecondsToTicks(expiration), - _ => ConvertUtils.UnixTimestampInMillisecondsToTicks(expiration) - }; - - var expirationWithOption = new ExpirationWithOption(expirationTimeInTicks, expireOption); - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var input = new RawStringInput(RespCommand.EXPIRE, arg1: expirationWithOption.Word); - - var _key = key.SpanByte; - var status = context.RMW(ref _key, ref input, ref output); - - if (status.IsPending) - CompletePendingForSession(ref status, ref output, ref context); - if (status.Found) found = true; - } - - if (!found && (storeType == StoreType.Object || storeType == StoreType.All) && - !objectStoreBasicContext.IsNull) - { - var header = new RespInputHeader(GarnetObjectType.Expire); - var objInput = new ObjectInput(header, arg1: expirationWithOption.WordHead, arg2: expirationWithOption.WordTail); - - // Retry on object store - var objOutput = new GarnetObjectStoreOutput(output); - var keyBytes = key.ToArray(); - var status = objectStoreContext.RMW(ref keyBytes, ref objInput, ref objOutput); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref objOutput, ref objectStoreContext); - if (status.Found) found = true; - - output = objOutput.SpanByteAndMemory; - } - - Debug.Assert(output.IsSpanByte); - if (found) timeoutSet = ((ObjectOutputHeader*)output.SpanByte.ToPointer())->result1 == 1; - - return found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; - } - - public unsafe GarnetStatus PERSIST(ArgSlice key, StoreType storeType, ref TContext context, ref TObjectContext objectStoreContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - GarnetStatus status = GarnetStatus.NOTFOUND; - - var inputHeader = new RawStringInput(RespCommand.PERSIST); - - var pbOutput = stackalloc byte[8]; - var o = new SpanByteAndMemory(pbOutput, 8); - - if (storeType == StoreType.Main || storeType == StoreType.All) - { - var _key = key.SpanByte; - var _status = context.RMW(ref _key, ref inputHeader, ref o); - - if (_status.IsPending) - CompletePendingForSession(ref _status, ref o, ref context); - - Debug.Assert(o.IsSpanByte); - if (o.SpanByte.AsReadOnlySpan()[0] == 1) - status = GarnetStatus.OK; - } - - if (status == GarnetStatus.NOTFOUND && (storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) - { - // Retry on object store - var header = new RespInputHeader(GarnetObjectType.Persist); - var objInput = new ObjectInput(header); - - var objO = new GarnetObjectStoreOutput(o); - var _key = key.ToArray(); - var _status = objectStoreContext.RMW(ref _key, ref objInput, ref objO); - - if (_status.IsPending) - CompletePendingForObjectStoreSession(ref _status, ref objO, ref objectStoreContext); - - Debug.Assert(o.IsSpanByte); - if (o.SpanByte.AsReadOnlySpan().Slice(0, CmdStrings.RESP_RETURN_VAL_1.Length) - .SequenceEqual(CmdStrings.RESP_RETURN_VAL_1)) - status = GarnetStatus.OK; - } - - return status; + var status = context.Delete((FixedSpanByteKey)key); + Debug.Assert(!status.IsPending); + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } /// /// For existing keys - overwrites part of the value at a specified offset (in-place if possible) /// For non-existing keys - creates a new string with the value at a specified offset (padded with '\0's) /// - /// + /// /// The key for which to set the range /// Input for the main store /// The length of the updated string /// Basic context for the main store /// - public unsafe GarnetStatus SETRANGE(ArgSlice key, ref RawStringInput input, ref ArgSlice output, ref TContext context) - where TContext : ITsavoriteContext + public unsafe GarnetStatus SETRANGE(PinnedSpanByte key, ref StringInput input, ref PinnedSpanByte output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var sbKey = key.SpanByte; - SpanByteAndMemory sbmOut = new(output.SpanByte); + StringOutput sbmOut = new(new SpanByteAndMemory(output)); - var status = context.RMW(ref sbKey, ref input, ref sbmOut); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref sbmOut); if (status.IsPending) CompletePendingForSession(ref status, ref sbmOut, ref context); - Debug.Assert(sbmOut.IsSpanByte); - output.length = sbmOut.Length; + Debug.Assert(sbmOut.SpanByteAndMemory.IsSpanByte); + output.Length = sbmOut.SpanByteAndMemory.Length; return GarnetStatus.OK; } - public GarnetStatus Increment(ArgSlice key, ref RawStringInput input, ref ArgSlice output, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus Increment(PinnedSpanByte key, ref StringInput input, ref StringOutput output, ref TStringContext context) + where TStringContext : ITsavoriteContext { - var _key = key.SpanByte; - SpanByteAndMemory _output = new(output.SpanByte); - - var status = context.RMW(ref _key, ref input, ref _output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) - CompletePendingForSession(ref status, ref _output, ref context); - Debug.Assert(_output.IsSpanByte); - output.length = _output.Length; + CompletePendingForSession(ref status, ref output, ref context); return GarnetStatus.OK; } - public unsafe GarnetStatus Increment(ArgSlice key, out long output, long increment, ref TContext context) - where TContext : ITsavoriteContext + public GarnetStatus Increment(PinnedSpanByte key, out long output, long increment, ref TStringContext context) + where TStringContext : ITsavoriteContext { var cmd = RespCommand.INCRBY; if (increment < 0) @@ -1200,108 +492,28 @@ public unsafe GarnetStatus Increment(ArgSlice key, out long output, lo increment = -increment; } - var input = new RawStringInput(cmd, 0, increment); + var input = new StringInput(cmd, 0, increment); const int outputBufferLength = NumUtils.MaximumFormatInt64Length + 1; - var outputBuffer = stackalloc byte[outputBufferLength]; - - var _key = key.SpanByte; - var _output = new SpanByteAndMemory(outputBuffer, outputBufferLength); + var stringOutput = StringOutput.FromPinnedSpan(stackalloc byte[outputBufferLength]); - var status = context.RMW(ref _key, ref input, ref _output); + var status = context.RMW((FixedSpanByteKey)key, ref input, ref stringOutput); if (status.IsPending) - CompletePendingForSession(ref status, ref _output, ref context); + CompletePendingForSession(ref status, ref stringOutput, ref context); - Debug.Assert(_output.IsSpanByte); + Debug.Assert(stringOutput.SpanByteAndMemory.IsSpanByte); - output = NumUtils.ReadInt64(_output.Length, outputBuffer); + output = NumUtils.ReadInt64(stringOutput.SpanByteAndMemory.Span); return GarnetStatus.OK; } - public void WATCH(ArgSlice key, StoreType type) + public void WATCH(PinnedSpanByte key, StoreType type) { - txnManager.Watch(key, type); + txnManager.AddTransactionStoreType(type); + txnManager.Watch(key); } - public unsafe void WATCH(byte[] key, StoreType type) - { - fixed (byte* ptr = key) - { - WATCH(new ArgSlice(ptr, key.Length), type); - } - } - - public unsafe GarnetStatus SCAN(long cursor, ArgSlice match, long count, ref TContext context) - { - return GarnetStatus.OK; - } - - public GarnetStatus GetKeyType(ArgSlice key, out string keyType, ref TContext context, ref TObjectContext objectContext) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - keyType = "string"; - // Check if key exists in Main store - var status = EXISTS(key, StoreType.Main, ref context, ref objectContext); - - // If key was not found in the main store then it is an object - if (status != GarnetStatus.OK && !objectStoreBasicContext.IsNull) - { - status = GET(key.ToArray(), out GarnetObjectStoreOutput output, ref objectContext); - if (status == GarnetStatus.OK) - { - if ((output.GarnetObject as SortedSetObject) != null) - { - keyType = "zset"; - } - else if ((output.GarnetObject as ListObject) != null) - { - keyType = "list"; - } - else if ((output.GarnetObject as SetObject) != null) - { - keyType = "set"; - } - else if ((output.GarnetObject as HashObject) != null) - { - keyType = "hash"; - } - } - else - { - keyType = "none"; - status = GarnetStatus.NOTFOUND; - } - } - return status; - } - - public GarnetStatus MemoryUsageForKey(ArgSlice key, out long memoryUsage, ref TContext context, ref TObjectContext objectContext, int samples = 0) - where TContext : ITsavoriteContext - where TObjectContext : ITsavoriteContext - { - memoryUsage = -1; - - // Check if key exists in Main store - var status = GET(key, out ArgSlice keyValue, ref context); - - if (status == GarnetStatus.NOTFOUND) - { - status = GET(key.ToArray(), out GarnetObjectStoreOutput objectValue, ref objectContext); - if (status != GarnetStatus.NOTFOUND) - { - memoryUsage = RecordInfo.GetLength() + (2 * IntPtr.Size) + // Log record length - Utility.RoundUp(key.SpanByte.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead + // Key allocation in heap with overhead - objectValue.GarnetObject.Size; // Value allocation in heap - } - } - else - { - memoryUsage = RecordInfo.GetLength() + Utility.RoundUp(key.SpanByte.TotalSize, RecordInfo.GetLength()) + Utility.RoundUp(keyValue.SpanByte.TotalSize, RecordInfo.GetLength()); - } - - return status; - } + public unsafe GarnetStatus SCAN(long cursor, PinnedSpanByte match, long count, ref TStringContext context) => GarnetStatus.OK; /// /// Computes the Longest Common Subsequence (LCS) of two keys. @@ -1314,18 +526,19 @@ public GarnetStatus MemoryUsageForKey(ArgSlice key, ou /// If true, the length of each match is returned. /// The minimum length of a match to be considered. /// The status of the operation. - public unsafe GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) + public unsafe GarnetStatus LCS(PinnedSpanByte key1, PinnedSpanByte key2, ref StringOutput output, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) { var createTransaction = false; if (txnManager.state != TxnState.Running) { - txnManager.SaveKeyEntryToLock(key1, false, LockType.Shared); - txnManager.SaveKeyEntryToLock(key2, false, LockType.Shared); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main); + txnManager.SaveKeyEntryToLock(key1, LockType.Shared); + txnManager.SaveKeyEntryToLock(key2, LockType.Shared); txnManager.Run(true); createTransaction = true; } - var context = txnManager.LockableContext; + var context = txnManager.StringTransactionalContext; try { var status = LCSInternal(key1, key2, ref output, ref context, lenOnly, withIndices, withMatchLen, minMatchLen); @@ -1338,14 +551,14 @@ public unsafe GarnetStatus LCS(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemo } } - private unsafe GarnetStatus LCSInternal(ArgSlice key1, ArgSlice key2, ref SpanByteAndMemory output, ref TContext context, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) - where TContext : ITsavoriteContext + private unsafe GarnetStatus LCSInternal(PinnedSpanByte key1, PinnedSpanByte key2, ref StringOutput output, ref TStringContext context, bool lenOnly = false, bool withIndices = false, bool withMatchLen = false, int minMatchLen = 0) + where TStringContext : ITsavoriteContext { - ArgSlice val1, val2; + PinnedSpanByte val1, val2; var status1 = GET(key1, out val1, ref context); var status2 = GET(key2, out val2, ref context); - var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); + var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output.SpanByteAndMemory); try { diff --git a/libs/server/Storage/Session/MainStore/RangeIndexOps.cs b/libs/server/Storage/Session/MainStore/RangeIndexOps.cs new file mode 100644 index 00000000000..c9428ddd461 --- /dev/null +++ b/libs/server/Storage/Session/MainStore/RangeIndexOps.cs @@ -0,0 +1,934 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using Garnet.common; +using Garnet.server.BfTreeInterop; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Storage session methods for RangeIndex operations. + /// + /// Each method follows the same pattern: prepare a with + /// the appropriate , then either: + /// (a) Issue an RMW for lifecycle ops (CREATE, PROMOTE, RESTORE), or + /// (b) Acquire a shared lock via , + /// read the stub, operate on the native BfTree, and release the lock on dispose. + /// + /// WRONGTYPE handling: RI-specific commands on non-RI keys (and vice versa) are + /// rejected by ReadMethods.cs / RMWMethods.cs type-safety checks, which + /// return . + /// + sealed unsafe partial class StorageSession + { + /// + /// Create a new RangeIndex. Creates the BfTree via native interop, serializes the + /// configuration into a , and persists + /// it via RMW (which triggers InitialUpdater in RMWMethods.cs). + /// + /// + /// No external lock is needed — RMW is atomic at the store level. + /// Corner case: If the key already exists (duplicate RI.CREATE), the RMW + /// returns InPlaceUpdated (existing RI key) or Found (non-RI key). + /// In both cases, the freshly created BfTree is disposed and an error is returned. + /// + /// The Garnet key for the new index. + /// 0 for Disk, 1 for Memory. + /// BfTree circular buffer size in bytes. + /// Minimum record size. + /// Maximum record size. + /// Maximum key length. + /// Leaf page size (0 = auto-compute). + /// The operation result code. + /// Error message span if the operation failed. + /// on success (check for details). + public GarnetStatus RangeIndexCreate( + PinnedSpanByte key, + byte storageBackend, + ulong cacheSize, + uint minRecordSize, + uint maxRecordSize, + uint maxKeyLen, + uint leafPageSize, + out RangeIndexResult result, + out ReadOnlySpan errorMsg) + { + result = RangeIndexResult.Error; + errorMsg = default; + + var rangeIndexManager = functionsState.rangeIndexManager; + + // Auto-compute leaf page size if not specified + if (leafPageSize == 0) + leafPageSize = RangeIndexManager.ComputeLeafPageSize(maxRecordSize); + + // Create the BfTree instance via interop + BfTreeService bfTree; + try + { + bfTree = rangeIndexManager.CreateBfTree( + (StorageBackendType)storageBackend, key.ReadOnlySpan, cacheSize, + minRecordSize, maxRecordSize, maxKeyLen, leafPageSize); + } + catch (Exception ex) + { + errorMsg = System.Text.Encoding.UTF8.GetBytes($"ERR {ex.Message}"); + return GarnetStatus.OK; + } + + var treePtr = bfTree.NativePtr; + + var stub = new RangeIndexManager.RangeIndexStub + { + TreeHandle = treePtr, + CacheSize = cacheSize, + MinRecordSize = minRecordSize, + MaxRecordSize = maxRecordSize, + MaxKeyLen = maxKeyLen, + LeafPageSize = leafPageSize, + StorageBackend = storageBackend, + SerializationPhase = 0, + }; + + var psb = PinnedSpanByte.FromPinnedPointer((byte*)Unsafe.AsPointer(ref stub), RangeIndexManager.RangeIndexStub.Size); + + parseState.InitializeWithArgument(psb); + + var input = new StringInput(RespCommand.RICREATE, ref parseState); + var output = new StringOutput(); + + // RMW is atomic — no external lock needed + var status = stringBasicContext.RMW((FixedSpanByteKey)key, ref input, ref output); + + if (status.IsPending) + CompletePendingForSession(ref status, ref output, ref stringBasicContext); + + if (status.Record.Created) + { + // Register for lifecycle management (cold path) + var keyHash = stringBasicContext.GetKeyHash((FixedSpanByteKey)key); + rangeIndexManager.RegisterIndex(bfTree, keyHash, key.ReadOnlySpan); + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + + if (status.Record.InPlaceUpdated || status.Found) + { + // Key already existed — free the new tree and return error + bfTree.Dispose(); + result = RangeIndexResult.Error; + errorMsg = "ERR index already exists"u8; + return GarnetStatus.OK; + } + + // RMW failed - free the BfTree + bfTree.Dispose(); + errorMsg = "ERR failed to create range index"u8; + return GarnetStatus.OK; + } + + /// + /// Promote a RangeIndex stub from the read-only region to the mutable region (tail). + /// Issues an RMW with RIPROMOTE; CopyUpdater copies the stub bytes and clears the flushed flag. + /// Uses a local parseState to avoid corrupting the caller's parseState. + /// + internal void PromoteRangeIndexToTail(PinnedSpanByte key) + { + var promoteParseState = new SessionParseState(); + promoteParseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIPROMOTE, ref promoteParseState); + var output = new StringOutput(); + + var status = stringBasicContext.RMW((FixedSpanByteKey)key, ref input, ref output); + if (status.IsPending) + CompletePendingForSession(ref status, ref output, ref stringBasicContext); + } + + /// + /// Restore a BfTree stub by updating TreeHandle in the mutable record. + /// Issues an RMW with RIRESTORE; IPU sets the TreeHandle on the mutable stub. + /// + internal void RestoreRangeIndexStub(PinnedSpanByte key, nint newTreeHandle) + { + var restoreParseState = new SessionParseState(); + restoreParseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIRESTORE, ref restoreParseState, arg1: (long)newTreeHandle); + var output = new StringOutput(); + + var status = stringBasicContext.RMW((FixedSpanByteKey)key, ref input, ref output); + if (status.IsPending) + CompletePendingForSession(ref status, ref output, ref stringBasicContext); + } + + /// + /// RI.SET — insert or update a field in a range index. + /// + /// + /// Hot path: acquires shared lock via , + /// reads the stub to get the native BfTree pointer, calls + /// while the lock is held, then injects a synthetic RMW for AOF logging. + /// + /// Corner cases: + /// + /// Key doesn't exist → NOTFOUND from ReadRangeIndex → returns error. + /// Key is not an RI key → WRONGTYPE from ReadMethods type-safety. + /// Field/value exceeds configured limits → InvalidKV from native BfTree. + /// TreeHandle is zero (evicted/recovering) → ReadRangeIndex handles lazy restore internally. + /// + /// + /// The Garnet key of the RangeIndex. + /// The field (BfTree key) to set. + /// The value to associate with the field. + /// The operation result code. + /// Error message span if the operation failed. + /// if key exists but is not an RI key; otherwise. + public GarnetStatus RangeIndexSet( + PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg) + { + errorMsg = default; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RISET, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status == GarnetStatus.WRONGTYPE) + { + result = RangeIndexResult.Error; + return GarnetStatus.WRONGTYPE; + } + + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + errorMsg = "ERR no such range index"u8; + return GarnetStatus.OK; + } + + var treePtr = ExtractTreePtr(stubSpan); + if (treePtr == 0) + { + result = RangeIndexResult.Error; + errorMsg = "ERR no such range index"u8; + return GarnetStatus.OK; + } + + var insertResult = BfTreeService.InsertByPtr(treePtr, field, value); + if (insertResult == BfTreeInsertResult.InvalidKV) + { + ref readonly var stub = ref RangeIndexManager.ReadIndex(stubSpan); + result = RangeIndexResult.Error; + errorMsg = System.Text.Encoding.UTF8.GetBytes( + $"ERR key+value size must be between {stub.MinRecordSize} and {stub.MaxRecordSize} bytes (got {field.Length + value.Length}), max key length {stub.MaxKeyLen} (got {field.Length})"); + return GarnetStatus.OK; + } + + result = RangeIndexResult.OK; + + functionsState.rangeIndexManager.ReplicateRangeIndexSet( + key, field, value, functionsState.appendOnlyFile, + stringBasicContext.Session.Version, stringBasicContext.Session.ID, + functionsState.StoredProcMode); + + return GarnetStatus.OK; + } + } + + /// + /// RI.GET — read a field from a range index. + /// + /// + /// Hot path: acquires shared lock, reads stub, reads from BfTree, + /// and writes the value as a RESP bulk string directly into . + /// + /// Uses a two-path strategy to avoid allocations: + /// + /// Inline path: If the network buffer has enough space, reads the BfTree + /// value directly into the output buffer past a reserved RESP header, then backfills + /// the exact header and adjusts pointers. + /// Heap path: If the network buffer is too small, rents a pooled buffer + /// from memoryPool, reads into it, and sets output.SpanByteAndMemory.Memory. + /// + /// + /// The Garnet key of the RangeIndex. + /// The field (BfTree key) to read. + /// The output to write the RESP-formatted value into. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexGet( + PinnedSpanByte key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result) + { + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIGET, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + ref readonly var stub = ref RangeIndexManager.ReadIndex(stubSpan); + if (stub.TreeHandle == 0) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + Debug.Assert(output.SpanByteAndMemory.IsSpanByte); + + var bufLen = output.SpanByteAndMemory.Length; + var maxValueSize = (int)stub.MaxRecordSize; + + const int optimisticHeaderSize = 1 + 1 + 2; // $N\r\n + const int trailerSize = 2; // \r\n + const int maxHeaderSize = 1 + 10 + 2; // $\r\n + var minBufferNeeded = maxHeaderSize + maxValueSize + trailerSize; // $\r\n + value + \r\n + + if (bufLen >= minBufferNeeded) + { + // Read BfTree value directly into output buffer past the header reservation + var bufStart = output.SpanByteAndMemory.SpanByte.ToPointer(); + var valueStart = bufStart + optimisticHeaderSize; + var readResult = BfTreeService.ReadByPtrInto(stub.TreeHandle, field, valueStart, maxValueSize, out var bytesWritten); + + if (readResult != BfTreeReadResult.Found || bytesWritten < 0) + { + result = RangeIndexResult.NotFound; + return GarnetStatus.OK; + } + + // Backfill exact header, append trailer, shift to eliminate gap + var numLength = NumUtils.CountDigits(bytesWritten); + var actualHeaderSize = 1 + numLength + 2; // $N\r\n + var actualValueStart = bufStart + actualHeaderSize; + if (valueStart != actualValueStart) + Buffer.MemoryCopy(valueStart, actualValueStart, bytesWritten, bytesWritten); + + var tmp = bufStart; + *tmp++ = (byte)'$'; + NumUtils.WriteInt32(bytesWritten, numLength, ref tmp); + *tmp++ = (byte)'\r'; + *tmp = (byte)'\n'; + + var trailerPtr = actualValueStart + bytesWritten; + *trailerPtr++ = (byte)'\r'; + *trailerPtr = (byte)'\n'; + + var totalLen = actualHeaderSize + bytesWritten + trailerSize; + output.SpanByteAndMemory.Length = totalLen; + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + else + { + // Not enough space in network buffer — fall through to heap path + output.SpanByteAndMemory.ConvertToHeap(); + var heapMemory = functionsState.memoryPool.Rent(minBufferNeeded); + + fixed (byte* bufStart = heapMemory.Memory.Span) + { + var valueStart = bufStart + optimisticHeaderSize; + var readResult = BfTreeService.ReadByPtrInto(stub.TreeHandle, field, valueStart, maxValueSize, out var bytesWritten); + + if (readResult != BfTreeReadResult.Found || bytesWritten < 0) + { + heapMemory.Dispose(); + result = RangeIndexResult.NotFound; + return GarnetStatus.OK; + } + + // Backfill exact header, append trailer, shift to eliminate gap + var numLength = NumUtils.CountDigits(bytesWritten); + var actualHeaderSize = 1 + numLength + 2; // $N\r\n + var actualValueStart = bufStart + actualHeaderSize; + if (valueStart != actualValueStart) + Buffer.MemoryCopy(valueStart, actualValueStart, bytesWritten, bytesWritten); + + var tmp = bufStart; + *tmp++ = (byte)'$'; + NumUtils.WriteInt32(bytesWritten, numLength, ref tmp); + *tmp++ = (byte)'\r'; + *tmp = (byte)'\n'; + + var trailerPtr = actualValueStart + bytesWritten; + *trailerPtr++ = (byte)'\r'; + *trailerPtr = (byte)'\n'; + + var totalLen = actualHeaderSize + bytesWritten + trailerSize; + + output.SpanByteAndMemory.Length = totalLen; + output.SpanByteAndMemory.Memory = heapMemory; + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + } + } + } + + /// + /// RI.DEL — delete a field from a range index. + /// + /// + /// Hot path: acquires shared lock, reads stub, calls + /// while lock is held, then injects a synthetic RMW for AOF logging. + /// Note: BfTree delete is a logical tombstone; the field cannot be read after deletion. + /// + /// The Garnet key of the RangeIndex. + /// The field (BfTree key) to delete. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexDel( + PinnedSpanByte key, PinnedSpanByte field, + out RangeIndexResult result) + { + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIDEL, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + var treePtr = ExtractTreePtr(stubSpan); + if (treePtr == 0) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + BfTreeService.DeleteByPtr(treePtr, field); + result = RangeIndexResult.OK; + + functionsState.rangeIndexManager.ReplicateRangeIndexDel( + key, field, functionsState.appendOnlyFile, + stringBasicContext.Session.Version, stringBasicContext.Session.ID, + functionsState.StoredProcMode); + + return GarnetStatus.OK; + } + } + + /// + /// RI.SCAN — scan entries starting at a key with a count limit. + /// Acquires shared lock, reads stub, writes complete RESP array response into output while lock is held. + /// + /// The Garnet key of the RangeIndex. + /// The BfTree key to start scanning from (inclusive). + /// Maximum number of records to return. + /// Which fields to include in the response (Key, Value, or Both). + /// The output to write the RESP-formatted response into. + /// On return, the number of records written. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexScan( + PinnedSpanByte key, PinnedSpanByte startKey, int count, + ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result) + { + recordCount = 0; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RISCAN, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + var treePtr = ExtractTreePtr(stubSpan); + if (treePtr == 0) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + if (RangeIndexManager.ReadIndex(stubSpan).StorageBackend == (byte)StorageBackendType.Memory) + { + result = RangeIndexResult.MemoryModeNotSupported; + return GarnetStatus.OK; + } + + WriteScanToOutput(treePtr, startKey.ReadOnlySpan, count, returnField, + isScanWithCount: true, [], + ref output, out recordCount); + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + } + + /// + /// RI.RANGE — scan entries in the closed range [start, end] from a range index. + /// Acquires shared lock, reads stub, writes complete RESP array response into output while lock is held. + /// + /// The Garnet key of the RangeIndex. + /// The BfTree key to start scanning from (inclusive). + /// The BfTree key to stop scanning at (inclusive). + /// Which fields to include in the response (Key, Value, or Both). + /// The output to write the RESP-formatted response into. + /// On return, the number of records written. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexRange( + PinnedSpanByte key, PinnedSpanByte startKey, PinnedSpanByte endKey, + ScanReturnField returnField, ref StringOutput output, + out int recordCount, out RangeIndexResult result) + { + recordCount = 0; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIRANGE, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + var treePtr = ExtractTreePtr(stubSpan); + if (treePtr == 0) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + if (RangeIndexManager.ReadIndex(stubSpan).StorageBackend == (byte)StorageBackendType.Memory) + { + result = RangeIndexResult.MemoryModeNotSupported; + return GarnetStatus.OK; + } + + WriteScanToOutput(treePtr, startKey.ReadOnlySpan, 0, returnField, + isScanWithCount: false, endKey.ReadOnlySpan, + ref output, out recordCount); + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + } + + /// + /// RI.EXISTS — check whether a key exists and is a RangeIndex. + /// + /// + /// Uses with the tag. + /// Because RIEXISTS is in , + /// the MainStore Reader returns for non-RI keys, + /// which this method maps to exists = false (no error to the client). + /// Similarly, maps to exists = false. + /// + /// The Garnet key to check. + /// On return, true if the key exists and is a RangeIndex. + /// Always returns . + public GarnetStatus RangeIndexExists(PinnedSpanByte key, out bool exists) + { + exists = false; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIEXISTS, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + var output = StringOutput.FromPinnedSpan(stubSpan); + + var status = Read_RangeIndex(key.ReadOnlySpan, ref input, ref output, ref stringBasicContext); + + // OK means the key exists and IS a RangeIndex (WRONGTYPE/NOTFOUND for anything else) + exists = status == GarnetStatus.OK; + return GarnetStatus.OK; + } + + /// + /// RI.CONFIG — return the configuration stored in the RangeIndex stub. + /// Reads the stub under a shared lock (same pattern as other RI read commands) + /// and extracts all configuration fields. + /// + /// The Garnet key of the RangeIndex. + /// On return, the storage backend type (0=Disk, 1=Memory). + /// On return, the circular buffer size in bytes. + /// On return, the minimum record size. + /// On return, the maximum record size. + /// On return, the maximum key length. + /// On return, the leaf page size. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexConfig(PinnedSpanByte key, + out byte storageBackend, out ulong cacheSize, out uint minRecordSize, + out uint maxRecordSize, out uint maxKeyLen, out uint leafPageSize, + out RangeIndexResult result) + { + storageBackend = 0; + cacheSize = 0; + minRecordSize = 0; + maxRecordSize = 0; + maxKeyLen = 0; + leafPageSize = 0; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RICONFIG, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + if (stubSpan.Length != RangeIndexManager.IndexSizeBytes) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + ref readonly var stub = ref RangeIndexManager.ReadIndex(stubSpan); + storageBackend = stub.StorageBackend; + cacheSize = stub.CacheSize; + minRecordSize = stub.MinRecordSize; + maxRecordSize = stub.MaxRecordSize; + maxKeyLen = stub.MaxKeyLen; + leafPageSize = stub.LeafPageSize; + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + } + + /// + /// RI.METRICS — return runtime metrics from the stub and live index registry. + /// Reports the tree handle, live/flushed/recovered status for diagnostic purposes. + /// + /// The Garnet key of the RangeIndex. + /// On return, the native tree pointer (0 if not live). + /// On return, true if the tree is registered in the live index dictionary. + /// On return, true if the stub has the Flushed flag set. + /// On return, true if the stub has the Recovered flag set. + /// The operation result code. + /// if the key is not an RI key; otherwise. + public GarnetStatus RangeIndexMetrics(PinnedSpanByte key, + out nint treeHandle, out bool isLive, out bool isFlushed, out bool isRecovered, + out RangeIndexResult result) + { + treeHandle = nint.Zero; + isLive = false; + isFlushed = false; + isRecovered = false; + + parseState.InitializeWithArgument(key); + var input = new StringInput(RespCommand.RIMETRICS, ref parseState); + Span stubSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + + using (functionsState.rangeIndexManager.ReadRangeIndex(this, key, ref input, stubSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status == GarnetStatus.WRONGTYPE ? GarnetStatus.WRONGTYPE : GarnetStatus.OK; + } + + if (stubSpan.Length != RangeIndexManager.IndexSizeBytes) + { + result = RangeIndexResult.Error; + return GarnetStatus.OK; + } + + ref readonly var stub = ref RangeIndexManager.ReadIndex(stubSpan); + treeHandle = stub.TreeHandle; + isLive = stub.TreeHandle != nint.Zero; + isFlushed = stub.IsFlushed; + isRecovered = stub.IsRecovered; + result = RangeIndexResult.OK; + return GarnetStatus.OK; + } + } + + /// + /// Scan BfTree entries and write the complete RESP array response directly into . + /// + /// + /// Uses an optimistic header reservation strategy: + /// + /// Reserves 5 bytes for the array header (*NN\r\n, up to 99 results). + /// Writes record bodies sequentially after the reservation. + /// On buffer overflow, grows the buffer by renting from memoryPool and + /// copying existing data (scan is not restarted). + /// After scan completes, backfills the actual array count header. If the actual + /// header differs in size from the reservation, shifts data accordingly. + /// + /// + /// Buffer transitions: Starts in the inline (network) buffer. On overflow, + /// transitions to heap-allocated buffers from memoryPool. The final buffer + /// is either kept inline or transferred to output.SpanByteAndMemory.Memory. + /// + /// Native tree pointer for the BfTree. + /// Key to start scanning from (inclusive). + /// Max records for scan-with-count; 0 for range scan. + /// Which fields to include per record. + /// true for RI.SCAN; false for RI.RANGE. + /// End key for range scan (inclusive); ignored for scan-with-count. + /// The StringOutput to write the RESP response into. + /// On return, the number of records written. + private void WriteScanToOutput( + nint treePtr, ReadOnlySpan startKey, int count, + ScanReturnField returnField, bool isScanWithCount, + ReadOnlySpan endKey, ref StringOutput output, out int recordCount) + { + // Optimistically reserve for 2-digit count (*NN\r\n = 5 bytes). + // Most scans return < 100 results, so the shift is 0 bytes in the common case. + const int ReservedHeaderSize = 5; // *NN\r\n + + int recCount = 0; + var rf = returnField; + + // State for the grow-in-place callback + byte* curr; + byte* bufEnd; + byte* bufStart; + bool isHeap = false; + IMemoryOwner heapMemory = null; + MemoryHandle heapHandle = default; + var memoryPool = functionsState.memoryPool; + + if (output.SpanByteAndMemory.IsSpanByte) + { + bufStart = output.SpanByteAndMemory.SpanByte.ToPointer(); + bufEnd = bufStart + output.SpanByteAndMemory.Length; + curr = bufStart + ReservedHeaderSize; + } + else + { + // Already on heap (shouldn't normally happen, but handle it) + var heapSize = 64 * 1024; + heapMemory = memoryPool.Rent(heapSize); + heapHandle = heapMemory.Memory.Pin(); + bufStart = (byte*)heapHandle.Pointer; + bufEnd = bufStart + heapSize; + curr = bufStart + ReservedHeaderSize; + isHeap = true; + } + + ScanRecordAction callback = (k, v) => + { + recCount++; + + if (TryWriteRecordResp(rf, k, v, ref curr, bufEnd)) + return true; + + // Doesn't fit — grow the buffer and retry this record + var writtenBytes = (int)(curr - bufStart); + var newSize = Math.Max((int)(bufEnd - bufStart) * 2, writtenBytes + k.Length + v.Length + 256); + + var newMemory = memoryPool.Rent(newSize); + var newHandle = newMemory.Memory.Pin(); + var newStart = (byte*)newHandle.Pointer; + + // Copy partial data written so far + new Span(bufStart, writtenBytes).CopyTo(new Span(newStart, writtenBytes)); + + // Release old heap buffer if we were on heap + if (isHeap) + { + heapHandle.Dispose(); + heapMemory?.Dispose(); + } + + heapMemory = newMemory; + heapHandle = newHandle; + bufStart = newStart; + bufEnd = newStart + newSize; + curr = newStart + writtenBytes; + isHeap = true; + + // Retry the record write — guaranteed to succeed now + TryWriteRecordResp(rf, k, v, ref curr, bufEnd); + return true; + }; + + try + { + if (isScanWithCount) + BfTreeService.ScanWithCountByPtrCallback(treePtr, startKey, count, returnField, callback); + else + BfTreeService.ScanWithEndKeyByPtrCallback(treePtr, startKey, endKey, returnField, callback); + + // Backfill the array header + var actualHeaderSize = 1 + NumUtils.CountDigits(recCount) + 2; + var extraNeeded = actualHeaderSize - ReservedHeaderSize; + + if (!isHeap && extraNeeded > 0 && curr + extraNeeded > bufEnd) + { + // Need more header space than reserved but inline buffer is full — move to heap + var writtenBytes = (int)(curr - bufStart); + var newSize = writtenBytes + extraNeeded + 64; + var newMemory = memoryPool.Rent(newSize); + var newHandle = newMemory.Memory.Pin(); + var newStart = (byte*)newHandle.Pointer; + new Span(bufStart, writtenBytes).CopyTo(new Span(newStart, writtenBytes)); + + heapMemory = newMemory; + heapHandle = newHandle; + bufStart = newStart; + curr = newStart + writtenBytes; + isHeap = true; + } + + if (!isHeap) + { + BackfillArrayHeader(bufStart, curr, ReservedHeaderSize, recCount, ref output, out recordCount); + } + else + { + output.SpanByteAndMemory.ConvertToHeap(); + output.SpanByteAndMemory.Memory = heapMemory; + BackfillArrayHeader(bufStart, curr, ReservedHeaderSize, recCount, ref output, out recordCount); + heapHandle.Dispose(); + heapMemory = null; + } + } + finally + { + // Only dispose if we still own it (not transferred to output) + if (isHeap && heapMemory != null) + { + heapHandle.Dispose(); + heapMemory.Dispose(); + } + } + } + + /// + /// Try to write a single scan record as RESP directly into a buffer via pointer arithmetic. + /// + /// Which fields to write (Key, Value, or KeyAndValue). + /// The key bytes from the scan iterator. + /// The value bytes from the scan iterator. + /// Current write position in the buffer; advanced on success. + /// End-of-buffer pointer. + /// false if there isn't enough space; true on success. + private static bool TryWriteRecordResp(ScanReturnField returnField, + ReadOnlySpan keySpan, ReadOnlySpan valueSpan, + ref byte* curr, byte* end) + { + if (returnField == ScanReturnField.KeyAndValue) + { + if (!RespWriteUtils.TryWriteArrayLength(2, ref curr, end)) + return false; + if (!RespWriteUtils.TryWriteBulkString(keySpan, ref curr, end)) + return false; + if (!RespWriteUtils.TryWriteBulkString(valueSpan, ref curr, end)) + return false; + } + else if (returnField == ScanReturnField.Key) + { + if (!RespWriteUtils.TryWriteBulkString(keySpan, ref curr, end)) + return false; + } + else + { + if (!RespWriteUtils.TryWriteBulkString(valueSpan, ref curr, end)) + return false; + } + return true; + } + + /// + /// Backfill the RESP array header (*count\r\n) into the reserved space before the record bodies. + /// + /// + /// Handles three cases: + /// + /// headerGap > 0: Actual header is smaller than reserved space (common for ≤ 99 results). + /// Writes header at offset, then shifts all data left to eliminate the gap. + /// headerGap == 0: Exact fit — writes header at the start, no shift needed. + /// headerGap < 0: Actual header is larger than reserved (e.g., > 99 results with + /// 2-digit reservation). Shifts records right to make room, then writes header at start. + /// + /// + /// Start of the output buffer. + /// Current write position (past the last record body byte). + /// Number of bytes originally reserved for the header. + /// Actual number of records written. + /// The StringOutput to set the final length on. + /// On return, set to . + private static void BackfillArrayHeader(byte* bufStart, byte* curr, int reservedHeaderSize, int recCount, + ref StringOutput output, out int recordCount) + { + var countDigits = NumUtils.CountDigits(recCount); + var actualHeaderSize = 1 + countDigits + 2; // *N\r\n + var headerGap = reservedHeaderSize - actualHeaderSize; + var recordBytes = (int)(curr - (bufStart + reservedHeaderSize)); + + if (headerGap >= 0) + { + // Actual header fits in reserved space (or exact fit). Write header, shift left if needed. + var headerStart = bufStart + headerGap; + var tmp = headerStart; + *tmp++ = (byte)'*'; + NumUtils.WriteInt32(recCount, countDigits, ref tmp); + *tmp++ = (byte)'\r'; + *tmp++ = (byte)'\n'; + + var totalLen = actualHeaderSize + recordBytes; + if (headerGap > 0) + Buffer.MemoryCopy(headerStart, bufStart, totalLen, totalLen); + + output.SpanByteAndMemory.Length = totalLen; + } + else + { + // Actual header is larger than reserved (e.g., >99 results with 2-digit reservation). + // Shift records right to make room, then write header at the start. + var extraNeeded = -headerGap; + var totalLen = actualHeaderSize + recordBytes; + + // Shift records right by extraNeeded bytes (overlapping memmove) + var recordSrc = bufStart + reservedHeaderSize; + var recordDst = bufStart + actualHeaderSize; + Buffer.MemoryCopy(recordSrc, recordDst, recordBytes, recordBytes); + + // Write header at the start + var tmp = bufStart; + *tmp++ = (byte)'*'; + NumUtils.WriteInt32(recCount, countDigits, ref tmp); + *tmp++ = (byte)'\r'; + *tmp++ = (byte)'\n'; + + output.SpanByteAndMemory.Length = totalLen; + } + + recordCount = recCount; + } + + /// + /// Extract the native tree pointer from a stub span. + /// Returns if the span is too small to contain a valid stub. + /// + /// The span containing the serialized stub bytes. + /// The native tree pointer, or 0 if the stub is invalid or too small. + private static nint ExtractTreePtr(Span stubSpan) + { + if (stubSpan.Length != RangeIndexManager.IndexSizeBytes) + return 0; + + return RangeIndexManager.ReadIndex(stubSpan).TreeHandle; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Session/MainStore/VectorStoreOps.cs b/libs/server/Storage/Session/MainStore/VectorStoreOps.cs index 9073d73fa83..91cc6d88419 100644 --- a/libs/server/Storage/Session/MainStore/VectorStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/VectorStoreOps.cs @@ -110,6 +110,7 @@ public enum VectorDistanceMetricType : int XCosine_Normalized, } + /// /// Implementation of Vector Set operations. /// @@ -119,28 +120,28 @@ sealed partial class StorageSession : IDisposable /// Implement Vector Set Add - this may also create a Vector Set if one does not already exist. /// [SkipLocalsInit] - public unsafe GarnetStatus VectorSetAdd(SpanByte key, int reduceDims, VectorValueType valueType, ArgSlice values, ArgSlice element, VectorQuantType quantizer, int buildExplorationFactor, ArgSlice attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg) + public unsafe GarnetStatus VectorSetAdd(PinnedSpanByte key, int reduceDims, VectorValueType valueType, PinnedSpanByte values, PinnedSpanByte element, VectorQuantType quantizer, int buildExplorationFactor, PinnedSpanByte attributes, int numLinks, VectorDistanceMetricType distanceMetric, out VectorManagerResult result, out ReadOnlySpan errorMsg) { var dims = VectorManager.CalculateValueDimensions(valueType, values.ReadOnlySpan); - var dimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dims, 1))); - var reduceDimsArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); - var valueTypeArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref valueType, 1))); + var dimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref dims, 1))); + var reduceDimsArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref reduceDims, 1))); + var valueTypeArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref valueType, 1))); var valuesArg = values; var elementArg = element; - var quantizerArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantizer, 1))); - var buildExplorationFactorArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); + var quantizerArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref quantizer, 1))); + var buildExplorationFactorArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref buildExplorationFactor, 1))); var attributesArg = attributes; - var numLinksArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); - var distanceMetricArg = ArgSlice.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); + var numLinksArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref numLinks, 1))); + var distanceMetricArg = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref distanceMetric, 1))); parseState.InitializeWithArguments([dimsArg, reduceDimsArg, valueTypeArg, valuesArg, elementArg, quantizerArg, buildExplorationFactorArg, attributesArg, numLinksArg, distanceMetricArg]); - var input = new RawStringInput(RespCommand.VADD, ref parseState); + var input = new StringInput(RespCommand.VADD, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadOrCreateVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadOrCreateVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -156,7 +157,7 @@ public unsafe GarnetStatus VectorSetAdd(SpanByte key, int reduceDims, VectorValu if (result == VectorManagerResult.OK) { // On successful addition, we need to manually replicate the write - vectorManager.ReplicateVectorSetAdd(ref key, ref input, ref basicContext); + vectorManager.ReplicateVectorSetAdd(key, ref input, ref stringBasicContext); } return GarnetStatus.OK; @@ -167,13 +168,13 @@ public unsafe GarnetStatus VectorSetAdd(SpanByte key, int reduceDims, VectorValu /// Implement Vector Set Remove - returns not found if the element is not present, or the vector set does not exist. /// [SkipLocalsInit] - public unsafe GarnetStatus VectorSetRemove(SpanByte key, SpanByte element) + public unsafe GarnetStatus VectorSetRemove(PinnedSpanByte key, PinnedSpanByte element) { - var input = new RawStringInput(RespCommand.VREM, ref parseState); + var input = new StringInput(RespCommand.VREM, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -182,12 +183,12 @@ public unsafe GarnetStatus VectorSetRemove(SpanByte key, SpanByte element) // After a successful read we remove the vector while holding a shared lock // That lock prevents deletion, but everything else can proceed in parallel - var res = vectorManager.TryRemove(indexSpan, element.AsReadOnlySpan()); + var res = vectorManager.TryRemove(indexSpan, element.ReadOnlySpan); if (res == VectorManagerResult.OK) { // On successful removal, we need to manually replicate the write - vectorManager.ReplicateVectorSetRemove(ref key, ref element, ref input, ref basicContext); + vectorManager.ReplicateVectorSetRemove(key, element, ref input, ref stringBasicContext); return GarnetStatus.OK; } @@ -200,16 +201,16 @@ public unsafe GarnetStatus VectorSetRemove(SpanByte key, SpanByte element) /// Perform a similarity search on an existing Vector Set given a vector as a bunch of floats. /// [SkipLocalsInit] - public unsafe GarnetStatus VectorSetValueSimilarity(SpanByte key, VectorValueType valueType, ArgSlice values, int count, float delta, int searchExplorationFactor, ReadOnlySpan filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + public unsafe GarnetStatus VectorSetValueSimilarity(PinnedSpanByte key, VectorValueType valueType, PinnedSpanByte values, int count, float delta, int searchExplorationFactor, ReadOnlySpan filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) { - parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan())); + parseState.InitializeWithArgument(key); // Get the index - var input = new RawStringInput(RespCommand.VSIM, ref parseState); + var input = new StringInput(RespCommand.VSIM, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -228,15 +229,15 @@ public unsafe GarnetStatus VectorSetValueSimilarity(SpanByte key, VectorValueTyp /// Perform a similarity search on an existing Vector Set given an element that is already in the Vector Set. /// [SkipLocalsInit] - public unsafe GarnetStatus VectorSetElementSimilarity(SpanByte key, ReadOnlySpan element, int count, float delta, int searchExplorationFactor, ReadOnlySpan filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) + public unsafe GarnetStatus VectorSetElementSimilarity(PinnedSpanByte key, ReadOnlySpan element, int count, float delta, int searchExplorationFactor, ReadOnlySpan filter, int maxFilteringEffort, bool includeAttributes, ref SpanByteAndMemory outputIds, out VectorIdFormat outputIdFormat, ref SpanByteAndMemory outputDistances, ref SpanByteAndMemory outputAttributes, out VectorManagerResult result, ref SpanByteAndMemory filterBitmap) { - parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan())); + parseState.InitializeWithArgument(key); - var input = new RawStringInput(RespCommand.VSIM, ref parseState); + var input = new StringInput(RespCommand.VSIM, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -254,15 +255,15 @@ public unsafe GarnetStatus VectorSetElementSimilarity(SpanByte key, ReadOnlySpan /// Get the approximate vector associated with an element, after (approximately) reversing any transformation. /// [SkipLocalsInit] - public unsafe GarnetStatus VectorSetEmbedding(SpanByte key, ReadOnlySpan element, ref SpanByteAndMemory outputDistances) + public unsafe GarnetStatus VectorSetEmbedding(PinnedSpanByte key, ReadOnlySpan element, ref SpanByteAndMemory outputDistances) { - parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan())); + parseState.InitializeWithArgument(key); - var input = new RawStringInput(RespCommand.VEMB, ref parseState); + var input = new StringInput(RespCommand.VEMB, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -279,15 +280,15 @@ public unsafe GarnetStatus VectorSetEmbedding(SpanByte key, ReadOnlySpan e } [SkipLocalsInit] - internal unsafe GarnetStatus VectorSetDimensions(SpanByte key, out int dimensions) + internal unsafe GarnetStatus VectorSetDimensions(PinnedSpanByte key, out int dimensions) { - parseState.InitializeWithArgument(ArgSlice.FromPinnedSpan(key.AsReadOnlySpan())); + parseState.InitializeWithArgument(key); - var input = new RawStringInput(RespCommand.VDIM, ref parseState); + var input = new StringInput(RespCommand.VDIM, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -296,7 +297,7 @@ internal unsafe GarnetStatus VectorSetDimensions(SpanByte key, out int dimension } // After a successful read we extract metadata - VectorManager.ReadIndex(indexSpan, out _, out var dimensionsUS, out var reducedDimensionsUS, out _, out _, out _, out _, out _, out _); + VectorManager.ReadIndex(indexSpan, out _, out var dimensionsUS, out var reducedDimensionsUS, out _, out _, out _, out _, out _); dimensions = (int)(reducedDimensionsUS == 0 ? dimensionsUS : reducedDimensionsUS); @@ -308,7 +309,7 @@ internal unsafe GarnetStatus VectorSetDimensions(SpanByte key, out int dimension /// Get debugging information about the VectorSet /// [SkipLocalsInit] - internal unsafe GarnetStatus VectorSetInfo(SpanByte key, + internal unsafe GarnetStatus VectorSetInfo(PinnedSpanByte key, out VectorQuantType quantType, out VectorDistanceMetricType distanceMetricType, out uint vectorDimensions, @@ -317,11 +318,11 @@ internal unsafe GarnetStatus VectorSetInfo(SpanByte key, out uint numberOfLinks, out long size) { - parseState.InitializeWithArgument(new(ref key)); + parseState.InitializeWithArgument(key); - var input = new RawStringInput(RespCommand.VINFO, ref parseState); + var input = new StringInput(RespCommand.VINFO, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { @@ -336,7 +337,7 @@ internal unsafe GarnetStatus VectorSetInfo(SpanByte key, } // After a successful read we extract metadata - VectorManager.ReadIndex(indexSpan, out var context, out vectorDimensions, out reducedDimensions, out quantType, out buildExplorationFactor, out numberOfLinks, out distanceMetricType, out var indexPtr, out _); + VectorManager.ReadIndex(indexSpan, out var context, out vectorDimensions, out reducedDimensions, out quantType, out buildExplorationFactor, out numberOfLinks, out distanceMetricType, out var indexPtr); size = (long)NativeDiskANNMethods.card(context, indexPtr); return GarnetStatus.OK; @@ -347,21 +348,21 @@ internal unsafe GarnetStatus VectorSetInfo(SpanByte key, /// Get the attributes associated with an element in the VectorSet /// [SkipLocalsInit] - internal unsafe GarnetStatus VectorSetGetAttribute(SpanByte key, ArgSlice elementId, ref SpanByteAndMemory outputAttributes) + internal unsafe GarnetStatus VectorSetGetAttribute(PinnedSpanByte key, PinnedSpanByte elementId, ref SpanByteAndMemory outputAttributes) { - parseState.InitializeWithArgument(new(ref key)); + parseState.InitializeWithArgument(key); // Get the index - var input = new RawStringInput(RespCommand.VGETATTR, ref parseState); + var input = new StringInput(RespCommand.VGETATTR, ref parseState); Span indexSpan = stackalloc byte[VectorManager.IndexSizeBytes]; - using (vectorManager.ReadVectorIndex(this, ref key, ref input, indexSpan, out var status)) + using (vectorManager.ReadVectorIndex(this, key, ref input, indexSpan, out var status)) { if (status != GarnetStatus.OK) { return status; } - var result = vectorManager.FetchSingleVectorElementAttributes(indexSpan, elementId.SpanByte, ref outputAttributes); + var result = vectorManager.FetchSingleVectorElementAttributes(indexSpan, elementId, ref outputAttributes); return result == VectorManagerResult.OK ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } } diff --git a/libs/server/Storage/Session/ObjectStore/AdvancedOps.cs b/libs/server/Storage/Session/ObjectStore/AdvancedOps.cs index 2d98a5ae71e..a4770f8173e 100644 --- a/libs/server/Storage/Session/ObjectStore/AdvancedOps.cs +++ b/libs/server/Storage/Session/ObjectStore/AdvancedOps.cs @@ -2,47 +2,43 @@ // Licensed under the MIT license. using System; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { - public GarnetStatus RMW_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus RMW_ObjectStore(ReadOnlySpan key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = objectStoreContext.RMW(ref key, ref input, ref output); + var status = objectContext.RMW((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref output, ref objectStoreContext); + CompletePendingForObjectStoreSession(ref status, ref output, ref objectContext); if (status.Found) { if (output.HasWrongType) return GarnetStatus.WRONGTYPE; - return GarnetStatus.OK; } return GarnetStatus.NOTFOUND; } - public GarnetStatus Read_ObjectStore(ref byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus Read_ObjectStore(ReadOnlySpan key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = objectStoreContext.Read(ref key, ref input, ref output); + var status = objectContext.Read((FixedSpanByteKey)key, ref input, ref output); if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref output, ref objectStoreContext); + CompletePendingForObjectStoreSession(ref status, ref output, ref objectContext); if (status.Found) { if (output.HasWrongType) return GarnetStatus.WRONGTYPE; - return GarnetStatus.OK; } diff --git a/libs/server/Storage/Session/ObjectStore/Common.cs b/libs/server/Storage/Session/ObjectStore/Common.cs index 5e5a69ad82e..b96d5eec70c 100644 --- a/libs/server/Storage/Session/ObjectStore/Common.cs +++ b/libs/server/Storage/Session/ObjectStore/Common.cs @@ -12,105 +12,50 @@ namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { #region Common ObjectStore Methods - unsafe GarnetStatus RMWObjectStoreOperation(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - var objStoreOutput = new GarnetObjectStoreOutput(); - - // Perform RMW on object store - var status = objectStoreContext.RMW(ref key, ref input, ref objStoreOutput); - - output = objStoreOutput.Header; - - return CompletePendingAndGetGarnetStatus(status, ref objectStoreContext, ref objStoreOutput); - } - - unsafe GarnetStatus RMWObjectStoreOperation(byte[] key, ArgSlice input, - out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - ref var objInput = ref Unsafe.AsRef(input.ptr); - - return RMWObjectStoreOperation(key, ref objInput, out output, ref objectStoreContext); - } - /// /// Perform RMW operation in object store - /// use this method in commands that return an array /// /// /// /// - /// + /// /// /// - GarnetStatus RMWObjectStoreOperationWithOutput(byte[] key, ref ObjectInput input, ref TObjectContext objectStoreContext, ref GarnetObjectStoreOutput output) - where TObjectContext : ITsavoriteContext + GarnetStatus RMWObjectStoreOperation(ReadOnlySpan key, ref ObjectInput input, ref TObjectContext objectContext, ref ObjectOutput output) + where TObjectContext : ITsavoriteContext { - if (objectStoreContext.Session is null) + if (objectContext.Session is null) ThrowObjectStoreUninitializedException(); // Perform RMW on object store - var status = objectStoreContext.RMW(ref key, ref input, ref output); + var status = objectContext.RMW((FixedSpanByteKey)key, ref input, ref output); - return CompletePendingAndGetGarnetStatus(status, ref objectStoreContext, ref output); + return CompletePendingAndGetGarnetStatus(status, ref objectContext, ref output); } /// /// Perform Read operation in object store - /// use this method in commands that return an array /// /// /// /// - /// + /// /// /// - GarnetStatus ReadObjectStoreOperationWithOutput(byte[] key, ref ObjectInput input, ref TObjectContext objectStoreContext, ref GarnetObjectStoreOutput output) - where TObjectContext : ITsavoriteContext + GarnetStatus ReadObjectStoreOperation(ReadOnlySpan key, ref ObjectInput input, ref TObjectContext objectContext, ref ObjectOutput output) + where TObjectContext : ITsavoriteContext { - if (objectStoreContext.Session is null) + if (objectContext.Session is null) ThrowObjectStoreUninitializedException(); // Perform read on object store - var status = objectStoreContext.Read(ref key, ref input, ref output); + var status = objectContext.Read((FixedSpanByteKey)key, ref input, ref output); - return CompletePendingAndGetGarnetStatus(status, ref objectStoreContext, ref output); - } - - /// - /// Perform Read operation in object store - /// use this method in commands that return an array - /// - /// - /// - /// - /// - /// - /// - unsafe GarnetStatus ReadObjectStoreOperationWithOutput(byte[] key, ArgSlice input, - ref TObjectContext objectStoreContext, ref GarnetObjectStoreOutput output) - where TObjectContext : ITsavoriteContext - { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - ref var objInput = ref Unsafe.AsRef(input.ptr); - - return ReadObjectStoreOperationWithOutput(key, ref objInput, ref objectStoreContext, ref output); + return CompletePendingAndGetGarnetStatus(status, ref objectContext, ref output); } /// @@ -122,9 +67,9 @@ unsafe GarnetStatus ReadObjectStoreOperationWithOutput(byte[] ke /// The pattern to match /// Limit number for the response /// The list of items for the response - /// - public unsafe GarnetStatus ObjectScan(GarnetObjectType objectType, ArgSlice key, long cursor, string match, int count, out ArgSlice[] items, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + /// + public unsafe GarnetStatus ObjectScan(GarnetObjectType objectType, PinnedSpanByte key, long cursor, string match, int count, out PinnedSpanByte[] items, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { Debug.Assert(objectType is GarnetObjectType.Hash or GarnetObjectType.Set or GarnetObjectType.SortedSet); @@ -159,30 +104,30 @@ public unsafe GarnetStatus ObjectScan(GarnetObjectType objectTyp var cursorSpan = paramsSpan.Slice(paramsSpanOffset, cursorLength); NumUtils.WriteInt64(cursor, cursorSpan); paramsSpanOffset += cursorLength; - var cursorSlice = ArgSlice.FromPinnedSpan(cursorSpan); + var cursorSlice = PinnedSpanByte.FromPinnedSpan(cursorSpan); // MATCH var matchSpan = paramsSpan.Slice(paramsSpanOffset, CmdStrings.MATCH.Length); CmdStrings.MATCH.CopyTo(matchSpan); paramsSpanOffset += CmdStrings.MATCH.Length; - var matchSlice = ArgSlice.FromPinnedSpan(matchSpan); + var matchSlice = PinnedSpanByte.FromPinnedSpan(matchSpan); // Pattern var patternSpan = paramsSpan.Slice(paramsSpanOffset, matchPattern.Length); - Encoding.ASCII.GetBytes(matchPattern, patternSpan); + _ = Encoding.ASCII.GetBytes(matchPattern, patternSpan); paramsSpanOffset += matchPattern.Length; - var matchPatternSlice = ArgSlice.FromPinnedSpan(patternSpan); + var matchPatternSlice = PinnedSpanByte.FromPinnedSpan(patternSpan); // COUNT var countSpan = paramsSpan.Slice(paramsSpanOffset, CmdStrings.COUNT.Length); CmdStrings.COUNT.CopyTo(countSpan); paramsSpanOffset += CmdStrings.COUNT.Length; - var countSlice = ArgSlice.FromPinnedSpan(countSpan); + var countSlice = PinnedSpanByte.FromPinnedSpan(countSpan); // Value var countValueSpan = paramsSpan.Slice(paramsSpanOffset, countLength); - NumUtils.WriteInt64(count, countValueSpan); - var countValueSlice = ArgSlice.FromPinnedSpan(countValueSpan); + _ = NumUtils.WriteInt64(count, countValueSpan); + var countValueSlice = PinnedSpanByte.FromPinnedSpan(countValueSpan); parseState.InitializeWithArguments(cursorSlice, matchSlice, matchPatternSlice, countSlice, countValueSlice); @@ -204,10 +149,10 @@ public unsafe GarnetStatus ObjectScan(GarnetObjectType objectTyp break; } - var output = new GarnetObjectStoreOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var output = new ObjectOutput(); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); - scratchBufferBuilder.RewindScratchBuffer(ref paramsSlice); + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); items = default; if (status == GarnetStatus.OK) @@ -226,7 +171,7 @@ public unsafe GarnetStatus ObjectScan(GarnetObjectType objectTyp /// /// An RESP3 array in array will be flattened into the return array. RESP3 map/set types will be returned as arrays. /// "*2\r\n*2\r\n$1\r\na\r\n,0\r\n*2\r\n$1\r\nb\r\n,1\r\n" will return [a, 0, b, 1] - unsafe ArgSlice[] ProcessRespArrayOutput(GarnetObjectStoreOutput output, out string error, bool isScanOutput = false) + unsafe PinnedSpanByte[] ProcessRespArrayOutput(ObjectOutput output, out string error, bool isScanOutput = false) { if (functionsState.respProtocolVersion >= 3) return ProcessResp3ArrayOutput(output, out error, isScanOutput); @@ -237,21 +182,20 @@ unsafe ArgSlice[] ProcessRespArrayOutput(GarnetObjectStoreOutput output, out str /// /// Converts an array of elements in RESP format to ArgSlice[] type /// - /// The RESP format output object + /// The RESP format output object /// A description of the error, if there is any /// True when the output comes from HSCAN, ZSCAN OR SSCAN command /// - private unsafe ArgSlice[] ProcessResp2ArrayOutput(GarnetObjectStoreOutput outputFooter, out string error, bool isScanOutput) + private unsafe PinnedSpanByte[] ProcessResp2ArrayOutput(ObjectOutput output, out string error, bool isScanOutput) { - ArgSlice[] elements = default; + PinnedSpanByte[] elements = default; error = default; // For reading the elements in the output byte* element = null; var len = 0; - var outputSpan = outputFooter.SpanByteAndMemory.IsSpanByte ? - outputFooter.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : outputFooter.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { @@ -285,11 +229,11 @@ private unsafe ArgSlice[] ProcessResp2ArrayOutput(GarnetObjectStoreOutput output return default; // Create the argslice[] - elements = new ArgSlice[isScanOutput ? arraySize + 1 : arraySize]; + elements = new PinnedSpanByte[isScanOutput ? arraySize + 1 : arraySize]; var i = 0; if (isScanOutput) - elements[i++] = new ArgSlice(element, len); + elements[i++] = PinnedSpanByte.FromPinnedPointer(element, len); for (; i < elements.Length; i++) { @@ -297,7 +241,7 @@ private unsafe ArgSlice[] ProcessResp2ArrayOutput(GarnetObjectStoreOutput output len = 0; if (RespReadUtils.TryReadPtrWithLengthHeader(ref element, ref len, ref refPtr, end)) { - elements[i] = new ArgSlice(element, len); + elements[i] = PinnedSpanByte.FromPinnedPointer(element, len); } } } @@ -307,20 +251,22 @@ private unsafe ArgSlice[] ProcessResp2ArrayOutput(GarnetObjectStoreOutput output len = 0; if (!RespReadUtils.TryReadPtrWithLengthHeader(ref result, ref len, ref refPtr, end)) return default; - elements = [new ArgSlice(result, len)]; + elements = [PinnedSpanByte.FromPinnedPointer(result, len)]; } + + CopyPinnedSpanByteArrayToScratchBuffer(elements, ref output); } } finally { - if (!outputFooter.SpanByteAndMemory.IsSpanByte) - outputFooter.SpanByteAndMemory.Memory.Dispose(); + if (!output.SpanByteAndMemory.IsSpanByte) + output.SpanByteAndMemory.Memory.Dispose(); } return elements; } - private unsafe ArgSlice[] ProcessResp3ArrayOutput(GarnetObjectStoreOutput output, out string error, bool isScanOutput) + private unsafe PinnedSpanByte[] ProcessResp3ArrayOutput(ObjectOutput output, out string error, bool isScanOutput) { // We support arrays ('*'), RSEP3 sets ('~') and RESP3 maps ('%'). // All are returned as arrays. @@ -331,15 +277,14 @@ static bool IsSupportedArrayType(char c) return c is '*' or '~' or '%'; } - ArgSlice[] elements = default; + PinnedSpanByte[] elements = default; error = default; // For reading the elements in the output byte* element = null; var len = 0; - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { @@ -401,11 +346,11 @@ static bool IsSupportedArrayType(char c) } // Create the argslice[] - elements = new ArgSlice[(arraySize * innerLen) + (isScanOutput ? 1 : 0)]; + elements = new PinnedSpanByte[(arraySize * innerLen) + (isScanOutput ? 1 : 0)]; var i = 0; if (isScanOutput) - elements[i++] = new ArgSlice(element, len); + elements[i++] = PinnedSpanByte.FromPinnedPointer(element, len); for (; i < elements.Length; i += innerLen) { @@ -428,7 +373,7 @@ static bool IsSupportedArrayType(char c) { if (RespReadUtils.TryReadPtrWithLengthHeader(ref element, ref len, ref refPtr, end)) { - elements[i + j] = new ArgSlice(element, len); + elements[i + j] = PinnedSpanByte.FromPinnedPointer(element, len); } } } @@ -440,8 +385,10 @@ static bool IsSupportedArrayType(char c) len = 0; if (!RespReadUtils.TryReadPtrWithLengthHeader(ref result, ref len, ref refPtr, end)) return default; - elements = [new ArgSlice(result, len)]; + elements = [PinnedSpanByte.FromPinnedPointer(result, len)]; } + + CopyPinnedSpanByteArrayToScratchBuffer(elements, ref output); } } finally @@ -459,7 +406,7 @@ static bool IsSupportedArrayType(char c) /// The RESP format output object /// A description of the error, if there is any /// - unsafe int[] ProcessRespIntegerArrayOutput(GarnetObjectStoreOutput output, out string error) + unsafe int[] ProcessRespIntegerArrayOutput(ObjectOutput output, out string error) { int[] elements = default; error = default; @@ -467,8 +414,7 @@ unsafe int[] ProcessRespIntegerArrayOutput(GarnetObjectStoreOutput output, out s // For reading the elements in the output byte* element = null; - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { @@ -489,7 +435,7 @@ unsafe int[] ProcessRespIntegerArrayOutput(GarnetObjectStoreOutput output, out s // Create the argslice[] elements = new int[arraySize]; - for (int i = 0; i < elements.Length; i++) + for (var i = 0; i < elements.Length; i++) { if (*refPtr != ':') { @@ -528,7 +474,7 @@ unsafe int[] ProcessRespIntegerArrayOutput(GarnetObjectStoreOutput output, out s /// The RESP format output object /// A description of the error, if there is any /// - unsafe long[] ProcessRespInt64ArrayOutput(GarnetObjectStoreOutput output, out string error) + unsafe long[] ProcessRespInt64ArrayOutput(ObjectOutput output, out string error) { long[] elements = default; error = default; @@ -536,8 +482,7 @@ unsafe long[] ProcessRespInt64ArrayOutput(GarnetObjectStoreOutput output, out st // For reading the elements in the output byte* element = null; - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { @@ -591,17 +536,88 @@ unsafe long[] ProcessRespInt64ArrayOutput(GarnetObjectStoreOutput output, out st return elements; } + /// + /// When output uses heap Memory (which is not pinned), copies PinnedSpanByte array data + /// to the scratch buffer (which uses pinned arrays) so that pointers remain valid. + /// Must be called inside the fixed block while source memory is still pinned. + /// Uses a single allocation to avoid scratch buffer reallocation during the copy. + /// + unsafe void CopyPinnedSpanByteArrayToScratchBuffer(PinnedSpanByte[] elements, ref ObjectOutput output) + { + if (elements == null || elements.Length == 0 || output.SpanByteAndMemory.IsSpanByte) + return; + + var totalSize = 0; + for (var i = 0; i < elements.Length; i++) + totalSize += elements[i].Length; + + if (totalSize == 0) + return; + + var buf = scratchBufferAllocator.CreateArgSlice(totalSize); + var bufPtr = buf.ptr; + for (var i = 0; i < elements.Length; i++) + { + var elemLen = elements[i].Length; + if (elemLen > 0) + { + elements[i].ReadOnlySpan.CopyTo(new Span(bufPtr, elemLen)); + elements[i] = PinnedSpanByte.FromPinnedPointer(bufPtr, elemLen); + bufPtr += elemLen; + } + } + } + + /// + /// When output uses heap Memory (which is not pinned), copies PinnedSpanByte pair data + /// to the scratch buffer (which uses pinned arrays) so that pointers remain valid. + /// Must be called inside the fixed block while source memory is still pinned. + /// Uses a single allocation to avoid scratch buffer reallocation during the copy. + /// + unsafe void CopyPinnedSpanBytePairsToScratchBuffer((PinnedSpanByte member, PinnedSpanByte score)[] pairs, ref ObjectOutput output) + { + if (pairs == null || pairs.Length == 0 || output.SpanByteAndMemory.IsSpanByte) + return; + + var totalSize = 0; + for (var i = 0; i < pairs.Length; i++) + totalSize += pairs[i].member.Length + pairs[i].score.Length; + + if (totalSize == 0) + return; + + var buf = scratchBufferAllocator.CreateArgSlice(totalSize); + var bufPtr = buf.ptr; + for (var i = 0; i < pairs.Length; i++) + { + var memberLen = pairs[i].member.Length; + if (memberLen > 0) + { + pairs[i].member.ReadOnlySpan.CopyTo(new Span(bufPtr, memberLen)); + pairs[i].member = PinnedSpanByte.FromPinnedPointer(bufPtr, memberLen); + bufPtr += memberLen; + } + + var scoreLen = pairs[i].score.Length; + if (scoreLen > 0) + { + pairs[i].score.ReadOnlySpan.CopyTo(new Span(bufPtr, scoreLen)); + pairs[i].score = PinnedSpanByte.FromPinnedPointer(bufPtr, scoreLen); + bufPtr += scoreLen; + } + } + } + /// /// Processes RESP output as pairs of score and member. /// - unsafe (ArgSlice member, ArgSlice score)[] ProcessRespArrayOutputAsPairs(GarnetObjectStoreOutput output, out string error) + unsafe (PinnedSpanByte member, PinnedSpanByte score)[] ProcessRespArrayOutputAsPairs(ObjectOutput output, out string error) { - (ArgSlice member, ArgSlice score)[] result = default; + (PinnedSpanByte member, PinnedSpanByte score)[] result = default; error = default; byte* element = null; var len = 0; - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { @@ -622,21 +638,23 @@ unsafe long[] ProcessRespInt64ArrayOutput(GarnetObjectStoreOutput output, out st Debug.Assert(arraySize % 2 == 0, "Array elements are expected to be in pairs"); arraySize /= 2; // Halve the array size to hold items as pairs - result = new (ArgSlice member, ArgSlice score)[arraySize]; + result = new (PinnedSpanByte member, PinnedSpanByte score)[arraySize]; for (var i = 0; i < result.Length; i++) { if (!RespReadUtils.TryReadPtrWithLengthHeader(ref element, ref len, ref refPtr, outputPtr + outputSpan.Length)) return default; - result[i].member = new ArgSlice(element, len); + result[i].member = PinnedSpanByte.FromPinnedPointer(element, len); if (!RespReadUtils.TryReadPtrWithLengthHeader(ref element, ref len, ref refPtr, outputPtr + outputSpan.Length)) return default; - result[i].score = new ArgSlice(element, len); + result[i].score = PinnedSpanByte.FromPinnedPointer(element, len); } } + + CopyPinnedSpanBytePairsToScratchBuffer(result, ref output); } } finally @@ -653,14 +671,13 @@ unsafe long[] ProcessRespInt64ArrayOutput(GarnetObjectStoreOutput output, out st /// /// The RESP format output object /// - unsafe ArgSlice ProcessRespSingleTokenOutput(GarnetObjectStoreOutput output) + unsafe PinnedSpanByte ProcessRespSingleTokenOutput(ObjectOutput output) { byte* element = null; var len = 0; - ArgSlice result; + PinnedSpanByte result; - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { fixed (byte* outputPtr = outputSpan) @@ -671,8 +688,13 @@ unsafe ArgSlice ProcessRespSingleTokenOutput(GarnetObjectStoreOutput output) if (!RespReadUtils.TryReadPtrWithSignedLengthHeader(ref element, ref len, ref refPtr, end) || len < 0) return default; + result = PinnedSpanByte.FromPinnedPointer(element, len); - result = new ArgSlice(element, len); + // When output uses heap Memory, the finally block will dispose it, invalidating + // the PinnedSpanByte pointer. Copy data to scratch buffer while still pinned. + // CreateArgSlice allocates pinned space and copies the data into it. + if (result.Length > 0 && !output.SpanByteAndMemory.IsSpanByte) + result = scratchBufferAllocator.CreateArgSlice(result.ReadOnlySpan); } } finally @@ -690,10 +712,9 @@ unsafe ArgSlice ProcessRespSingleTokenOutput(GarnetObjectStoreOutput output) /// The RESP format output object /// /// integer - unsafe bool TryProcessRespSimple64IntOutput(GarnetObjectStoreOutput output, out long value) + unsafe bool TryProcessRespSimple64IntOutput(ObjectOutput output, out long value) { - var outputSpan = output.SpanByteAndMemory.IsSpanByte ? - output.SpanByteAndMemory.SpanByte.AsReadOnlySpan() : output.SpanByteAndMemory.AsMemoryReadOnlySpan(); + var outputSpan = output.SpanByteAndMemory.ReadOnlySpan; try { fixed (byte* outputPtr = outputSpan) @@ -714,108 +735,17 @@ unsafe bool TryProcessRespSimple64IntOutput(GarnetObjectStoreOutput output, out } /// - /// Gets the value of the key store in the Object Store + /// Deletes a key from the object store context. /// - /// - /// - /// - /// - /// + /// The name of the key to use in the operation + /// Basic context for the object store. /// - unsafe GarnetStatus ReadObjectStoreOperation(byte[] key, ArgSlice input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus DELETE_ObjectStore(PinnedSpanByte key, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - ref var _input = ref Unsafe.AsRef(input.ptr); - - var _output = new GarnetObjectStoreOutput(); - - // Perform Read on object store - var status = objectStoreContext.Read(ref key, ref _input, ref _output); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref _output, ref objectStoreContext); - - output = _output.Header; - - if (_output.HasWrongType) - return GarnetStatus.WRONGTYPE; - - if (status.Found && (!status.Record.Created && !status.Record.CopyUpdated && !status.Record.InPlaceUpdated)) - return GarnetStatus.OK; - - return GarnetStatus.NOTFOUND; - } - - /// - /// Gets the value of the key store in the Object Store - /// - /// - /// - /// - /// - /// - /// - unsafe GarnetStatus ReadObjectStoreOperation(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - var _output = new GarnetObjectStoreOutput(); - - // Perform Read on object store - var status = objectStoreContext.Read(ref key, ref input, ref _output); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref _output, ref objectStoreContext); - - output = _output.Header; - - if (_output.HasWrongType) - return GarnetStatus.WRONGTYPE; - - if (status.Found && (!status.Record.Created && !status.Record.CopyUpdated && !status.Record.InPlaceUpdated)) - return GarnetStatus.OK; - - return GarnetStatus.NOTFOUND; - } - - /// - /// Gets the value of the key store in the Object Store - /// - unsafe GarnetStatus ReadObjectStoreOperationWithObject(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, out IGarnetObject garnetObject, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - { - if (objectStoreContext.Session is null) - ThrowObjectStoreUninitializedException(); - - var _output = new GarnetObjectStoreOutput(); - - // Perform Read on object store - var status = objectStoreContext.Read(ref key, ref input, ref _output); - - if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref _output, ref objectStoreContext); - - output = _output.Header; - - if (_output.HasWrongType) - { - garnetObject = null; - return GarnetStatus.WRONGTYPE; - } - - if (status.Found && (!status.Record.Created && !status.Record.CopyUpdated && !status.Record.InPlaceUpdated)) - { - garnetObject = _output.GarnetObject; - return GarnetStatus.OK; - } - - garnetObject = null; - return GarnetStatus.NOTFOUND; + var status = objectContext.Delete((FixedSpanByteKey)key); + Debug.Assert(!status.IsPending); + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; } /// @@ -825,10 +755,10 @@ unsafe GarnetStatus ReadObjectStoreOperationWithObject(byte[] ke /// The key of the sorted set /// /// - /// - public GarnetStatus ObjectScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + /// + public GarnetStatus ObjectScan(ReadOnlySpan key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key, ref input, ref objectContext, ref output); [MethodImpl(MethodImplOptions.NoInlining)] static void ThrowObjectStoreUninitializedException() @@ -841,14 +771,14 @@ static void ThrowObjectStoreUninitializedException() /// /// /// - /// + /// /// /// - private GarnetStatus CompletePendingAndGetGarnetStatus(Status status, ref TObjectContext objectStoreContext, ref GarnetObjectStoreOutput output) - where TObjectContext : ITsavoriteContext + private GarnetStatus CompletePendingAndGetGarnetStatus(Status status, ref TObjectContext objectContext, ref ObjectOutput output) + where TObjectContext : ITsavoriteContext { if (status.IsPending) - CompletePendingForObjectStoreSession(ref status, ref output, ref objectStoreContext); + CompletePendingForObjectStoreSession(ref status, ref output, ref objectContext); if (status.NotFound && !status.Record.Created) return GarnetStatus.NOTFOUND; @@ -869,13 +799,11 @@ private GarnetStatus CompletePendingAndGetGarnetStatus(Status st /// The input object for the operation. /// The context of the object store. /// The status of the operation. - private GarnetStatus ObjectCollect(ArgSlice searchKey, ReadOnlySpan typeObject, SingleWriterMultiReaderLock collectLock, ref ObjectInput input, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + private GarnetStatus ObjectCollect(PinnedSpanByte searchKey, ReadOnlySpan typeObject, SingleWriterMultiReaderLock collectLock, ref ObjectInput input, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { if (!collectLock.TryWriteLock()) - { return GarnetStatus.NOTFOUND; - } try { @@ -885,14 +813,11 @@ private GarnetStatus ObjectCollect(ArgSlice searchKey, ReadOnlyS do { if (!DbScan(searchKey, true, cursor, out storeCursor, out var hashKeys, 100, typeObject)) - { return GarnetStatus.OK; - } + var output = new ObjectOutput(); foreach (var hashKey in hashKeys) - { - RMWObjectStoreOperation(hashKey, ref input, out _, ref objectContext); - } + RMWObjectStoreOperation(hashKey, ref input, ref objectContext, ref output); cursor = storeCursor; } while (storeCursor != 0); diff --git a/libs/server/Storage/Session/ObjectStore/CompletePending.cs b/libs/server/Storage/Session/ObjectStore/CompletePending.cs index 35297661122..b00b7c6c432 100644 --- a/libs/server/Storage/Session/ObjectStore/CompletePending.cs +++ b/libs/server/Storage/Session/ObjectStore/CompletePending.cs @@ -2,13 +2,11 @@ // Licensed under the MIT license. using System.Diagnostics; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession { /// @@ -17,8 +15,8 @@ sealed partial class StorageSession /// /// /// - static void CompletePendingForObjectStoreSession(ref Status status, ref GarnetObjectStoreOutput output, ref TContext objectContext) - where TContext : ITsavoriteContext + internal static void CompletePendingForObjectStoreSession(ref Status status, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { objectContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); var more = completedOutputs.Next(); diff --git a/libs/server/Storage/Session/ObjectStore/HashOps.cs b/libs/server/Storage/Session/ObjectStore/HashOps.cs index a3fd6dad320..43f350864b6 100644 --- a/libs/server/Storage/Session/ObjectStore/HashOps.cs +++ b/libs/server/Storage/Session/ObjectStore/HashOps.cs @@ -7,9 +7,6 @@ namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Server API methods - HASH /// @@ -29,11 +26,11 @@ sealed partial class StorageSession : IDisposable /// /// /// - /// + /// /// /// - public unsafe GarnetStatus HashSet(ArgSlice key, ArgSlice field, ArgSlice value, out int itemsDoneCount, ref TObjectContext objectStoreContext, bool nx = false) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, out int itemsDoneCount, ref TObjectContext objectContext, bool nx = false) + where TObjectContext : ITsavoriteContext { itemsDoneCount = 0; @@ -46,8 +43,9 @@ public unsafe GarnetStatus HashSet(ArgSlice key, ArgSlice field, // Prepare the input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HSET }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); itemsDoneCount = output.result1; return status; @@ -62,10 +60,10 @@ public unsafe GarnetStatus HashSet(ArgSlice key, ArgSlice field, /// /// /// - /// + /// /// - public unsafe GarnetStatus HashSet(ArgSlice key, (ArgSlice field, ArgSlice value)[] elements, out int itemsDoneCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashSet(PinnedSpanByte key, (PinnedSpanByte field, PinnedSpanByte value)[] elements, out int itemsDoneCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { itemsDoneCount = 0; @@ -83,8 +81,9 @@ public unsafe GarnetStatus HashSet(ArgSlice key, (ArgSlice field // Prepare the input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HSET }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); itemsDoneCount = output.result1; return status; @@ -97,12 +96,12 @@ public unsafe GarnetStatus HashSet(ArgSlice key, (ArgSlice field /// /// /// - /// + /// /// /// - public GarnetStatus HashDelete(ArgSlice key, ArgSlice field, out int itemsDoneCount, ref TObjectContext objectStoreContext, bool nx = false) - where TObjectContext : ITsavoriteContext - => HashDelete(key, [field], out itemsDoneCount, ref objectStoreContext); + public GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte field, out int itemsDoneCount, ref TObjectContext objectContext, bool nx = false) + where TObjectContext : ITsavoriteContext + => HashDelete(key, [field], out itemsDoneCount, ref objectContext); /// /// Removes the specified fields from the hash key. @@ -111,10 +110,10 @@ public GarnetStatus HashDelete(ArgSlice key, ArgSlice field, out /// /// /// - /// + /// /// - public unsafe GarnetStatus HashDelete(ArgSlice key, ArgSlice[] fields, out int itemsDoneCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashDelete(PinnedSpanByte key, PinnedSpanByte[] fields, out int itemsDoneCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { itemsDoneCount = 0; @@ -127,8 +126,9 @@ public unsafe GarnetStatus HashDelete(ArgSlice key, ArgSlice[] f // Prepare the input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HDEL }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); itemsDoneCount = output.result1; return status; @@ -141,10 +141,10 @@ public unsafe GarnetStatus HashDelete(ArgSlice key, ArgSlice[] f /// /// /// - /// + /// /// - public unsafe GarnetStatus HashGet(ArgSlice key, ArgSlice field, out ArgSlice value, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashGet(PinnedSpanByte key, PinnedSpanByte field, out PinnedSpanByte value, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { value = default; @@ -158,9 +158,9 @@ public unsafe GarnetStatus HashGet(ArgSlice key, ArgSlice field, var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HGET }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); value = default; if (status == GarnetStatus.OK) @@ -176,10 +176,10 @@ public unsafe GarnetStatus HashGet(ArgSlice key, ArgSlice field, /// /// /// - /// + /// /// - public unsafe GarnetStatus HashGetMultiple(ArgSlice key, ArgSlice[] fields, out ArgSlice[] values, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashGetMultiple(PinnedSpanByte key, PinnedSpanByte[] fields, out PinnedSpanByte[] values, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { values = default; @@ -193,9 +193,9 @@ public unsafe GarnetStatus HashGetMultiple(ArgSlice key, ArgSlic var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HMGET }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); values = default; if (status == GarnetStatus.OK) @@ -210,10 +210,10 @@ public unsafe GarnetStatus HashGetMultiple(ArgSlice key, ArgSlic /// /// /// - /// + /// /// - public unsafe GarnetStatus HashGetAll(ArgSlice key, out ArgSlice[] values, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashGetAll(PinnedSpanByte key, out PinnedSpanByte[] values, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { values = default; @@ -224,9 +224,9 @@ public unsafe GarnetStatus HashGetAll(ArgSlice key, out ArgSlice var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HGETALL }; var input = new ObjectInput(header); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); values = default; if (status == GarnetStatus.OK) @@ -241,11 +241,11 @@ public unsafe GarnetStatus HashGetAll(ArgSlice key, out ArgSlice /// /// /// - /// + /// /// /// - public unsafe GarnetStatus HashLength(ArgSlice key, out int items, ref TObjectContext objectStoreContext, bool nx = false) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashLength(PinnedSpanByte key, out int items, ref TObjectContext objectContext, bool nx = false) + where TObjectContext : ITsavoriteContext { items = 0; @@ -255,8 +255,9 @@ public unsafe GarnetStatus HashLength(ArgSlice key, out int item // Prepare the input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HLEN }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); items = output.result1; @@ -270,10 +271,10 @@ public unsafe GarnetStatus HashLength(ArgSlice key, out int item /// /// /// - /// + /// /// - public unsafe GarnetStatus HashExists(ArgSlice key, ArgSlice field, out bool exists, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashExists(PinnedSpanByte key, PinnedSpanByte field, out bool exists, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { exists = false; if (key.Length == 0) @@ -285,8 +286,9 @@ public unsafe GarnetStatus HashExists(ArgSlice key, ArgSlice fie // Prepare the input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HEXISTS }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); exists = output.result1 == 1; @@ -299,10 +301,10 @@ public unsafe GarnetStatus HashExists(ArgSlice key, ArgSlice fie /// /// /// - /// + /// /// - public unsafe GarnetStatus HashRandomField(ArgSlice key, out ArgSlice field, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashRandomField(PinnedSpanByte key, out PinnedSpanByte field, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { field = default; @@ -316,9 +318,9 @@ public unsafe GarnetStatus HashRandomField(ArgSlice key, out Arg var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HRANDFIELD }; var input = new ObjectInput(header, 1 << 2, seed); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); // Process output if (status == GarnetStatus.OK) @@ -337,10 +339,10 @@ public unsafe GarnetStatus HashRandomField(ArgSlice key, out Arg /// /// /// - /// + /// /// - public unsafe GarnetStatus HashRandomField(ArgSlice key, int count, bool withValues, out ArgSlice[] fields, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashRandomField(PinnedSpanByte key, int count, bool withValues, out PinnedSpanByte[] fields, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { fields = default; @@ -355,8 +357,8 @@ public unsafe GarnetStatus HashRandomField(ArgSlice key, int cou var inputArg = (((count << 1) | 1) << 1) | (withValues ? 1 : 0); var input = new ObjectInput(header, inputArg, seed); - var output = new GarnetObjectStoreOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var output = new ObjectOutput(); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); fields = default; if (status == GarnetStatus.OK) @@ -374,11 +376,11 @@ public unsafe GarnetStatus HashRandomField(ArgSlice key, int cou /// /// /// - /// + /// /// - public GarnetStatus HashSet(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus HashSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// HashGet: Returns the value associated with field in the hash stored at key. @@ -390,11 +392,11 @@ public GarnetStatus HashSet(byte[] key, ref ObjectInput input, o /// /// /// - /// + /// /// - public GarnetStatus HashGet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus HashGet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns all fields and values of the hash stored at key. @@ -403,11 +405,11 @@ public GarnetStatus HashGet(byte[] key, ref ObjectInput input, r /// /// /// - /// + /// /// - public GarnetStatus HashGetAll(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus HashGetAll(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the values associated with the specified fields in the hash stored at key. @@ -416,11 +418,11 @@ public GarnetStatus HashGetAll(byte[] key, ref ObjectInput input /// /// /// - /// + /// /// - public GarnetStatus HashGetMultiple(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus HashGetMultiple(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns a random field from the hash value stored at key. @@ -429,11 +431,11 @@ public GarnetStatus HashGetMultiple(byte[] key, ref ObjectInput /// /// /// - /// + /// /// - public GarnetStatus HashRandomField(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus HashRandomField(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the number of fields contained in the hash key. @@ -442,11 +444,11 @@ public GarnetStatus HashRandomField(byte[] key, ref ObjectInput /// /// /// - /// + /// /// - public GarnetStatus HashLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus HashLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the string length of the value associated with field in the hash stored at key. If the key or the field do not exist, 0 is returned. @@ -454,12 +456,12 @@ public GarnetStatus HashLength(byte[] key, ref ObjectInput input /// /// /// - /// + /// /// /// - public GarnetStatus HashStrLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus HashStrLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes the specified fields from the hash key. @@ -468,11 +470,11 @@ public GarnetStatus HashStrLength(byte[] key, ref ObjectInput in /// /// /// - /// + /// /// - public GarnetStatus HashDelete(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus HashDelete(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns if field exists in the hash stored at key. @@ -481,11 +483,11 @@ public GarnetStatus HashDelete(byte[] key, ref ObjectInput input /// /// /// - /// + /// /// - public GarnetStatus HashExists(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus HashExists(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns all field names in the hash key. @@ -496,9 +498,9 @@ public GarnetStatus HashExists(byte[] key, ref ObjectInput input /// /// /// - public GarnetStatus HashKeys(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus HashKeys(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns all values in the hash key. @@ -509,22 +511,9 @@ public GarnetStatus HashKeys(byte[] key, ref ObjectInput input, /// /// /// - public GarnetStatus HashVals(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); - - /// - /// Increments the number stored at field in the hash stored at key by increment. - /// - /// - /// - /// - /// - /// - /// - public GarnetStatus HashIncrement(byte[] key, ArgSlice input, out ObjectOutputHeader output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, input, out output, ref objectContext); + public GarnetStatus HashVals(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// HashIncrementByFloat: Increment the specified field of a hash stored at key, @@ -536,9 +525,9 @@ public GarnetStatus HashIncrement(byte[] key, ArgSlice input, ou /// /// /// - public GarnetStatus HashIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus HashIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Sets the expiration time for the specified key. @@ -546,12 +535,12 @@ public GarnetStatus HashIncrement(byte[] key, ref ObjectInput in /// The type of the object context. /// The key for which to set the expiration time. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus HashExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + public GarnetStatus HashExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the time-to-live (TTL) of a hash key. @@ -561,15 +550,15 @@ public GarnetStatus HashExpire(ArgSlice key, ref ObjectInput inp /// Indicates whether the TTL is in milliseconds. /// Indicates whether the TTL is a timestamp. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus HashTimeToLive(ArgSlice key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus HashTimeToLive(PinnedSpanByte key, bool isMilliseconds, bool isTimestamp, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { var innerInput = new ObjectInput(input.header, ref input.parseState, arg1: isMilliseconds ? 1 : 0, arg2: isTimestamp ? 1 : 0); - return ReadObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref output); + return ReadObjectStoreOperation(key.ReadOnlySpan, ref innerInput, ref objectContext, ref output); } /// @@ -578,12 +567,12 @@ public GarnetStatus HashTimeToLive(ArgSlice key, bool isMillisec /// The type of the object context. /// The key of the hash. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus HashPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + public GarnetStatus HashPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Collects hash keys and performs a specified operation on them. @@ -597,18 +586,16 @@ public GarnetStatus HashPersist(ArgSlice key, ref ObjectInput in /// If the first key is "*", all hash keys are scanned in batches and the operation is performed on each key. /// Otherwise, the operation is performed on the specified keys. /// - public GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus HashCollect(ReadOnlySpan keys, ref ObjectInput input, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { if (keys[0].ReadOnlySpan.SequenceEqual("*"u8)) - { return ObjectCollect(keys[0], CmdStrings.HASH, _hcollectTaskLock, ref input, ref objectContext); - } + + var output = new ObjectOutput(); foreach (var key in keys) - { - RMWObjectStoreOperation(key.ToArray(), ref input, out _, ref objectContext); - } + RMWObjectStoreOperation(key, ref input, ref objectContext, ref output); return GarnetStatus.OK; } diff --git a/libs/server/Storage/Session/ObjectStore/ListOps.cs b/libs/server/Storage/Session/ObjectStore/ListOps.cs index d8a8e2e6779..d3d4ee0c469 100644 --- a/libs/server/Storage/Session/ObjectStore/ListOps.cs +++ b/libs/server/Storage/Session/ObjectStore/ListOps.cs @@ -3,13 +3,11 @@ using System; using System.Linq; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { /// @@ -23,10 +21,10 @@ sealed partial class StorageSession : IDisposable /// The elements to be added at the left or the righ of the list /// The Right or Left modifier of the operation to perform /// The length of the list after the push operations. - /// + /// /// - public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice[] elements, ListOperation lop, out int itemsDoneCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus ListPush(PinnedSpanByte key, PinnedSpanByte[] elements, ListOperation lop, out int itemsDoneCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { itemsDoneCount = 0; @@ -39,12 +37,12 @@ public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice[] ele // Prepare the input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var arrKey = key.ToArray(); - var status = RMWObjectStoreOperation(arrKey, ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); itemsDoneCount = output.result1; - itemBroker.HandleCollectionUpdate(arrKey); + itemBroker?.HandleCollectionUpdate(key.ToArray()); return status; } @@ -59,10 +57,10 @@ public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice[] ele /// /// /// - /// + /// /// - public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice element, ListOperation lop, out int itemsDoneCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus ListPush(PinnedSpanByte key, PinnedSpanByte element, ListOperation lop, out int itemsDoneCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { itemsDoneCount = 0; @@ -72,11 +70,12 @@ public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice eleme // Prepare the input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); itemsDoneCount = output.result1; - itemBroker.HandleCollectionUpdate(key.Span.ToArray()); + itemBroker?.HandleCollectionUpdate(key.ToArray()); return status; } @@ -87,13 +86,13 @@ public unsafe GarnetStatus ListPush(ArgSlice key, ArgSlice eleme /// /// /// - /// + /// /// /// The popped element - public GarnetStatus ListPop(ArgSlice key, ListOperation lop, ref TObjectContext objectStoreContext, out ArgSlice element) - where TObjectContext : ITsavoriteContext + public GarnetStatus ListPop(PinnedSpanByte key, ListOperation lop, ref TObjectContext objectContext, out PinnedSpanByte element) + where TObjectContext : ITsavoriteContext { - var status = ListPop(key, 1, lop, ref objectStoreContext, out var elements); + var status = ListPop(key, 1, lop, ref objectContext, out var elements); element = status == GarnetStatus.OK ? elements.FirstOrDefault() : default; return status; } @@ -106,19 +105,19 @@ public GarnetStatus ListPop(ArgSlice key, ListOperation lop, ref /// /// /// - /// + /// /// /// The count elements popped from the list - public unsafe GarnetStatus ListPop(ArgSlice key, int count, ListOperation lop, ref TObjectContext objectStoreContext, out ArgSlice[] elements) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus ListPop(PinnedSpanByte key, int count, ListOperation lop, ref TObjectContext objectContext, out PinnedSpanByte[] elements) + where TObjectContext : ITsavoriteContext { // Prepare the input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; var input = new ObjectInput(header, count); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); //process output elements = default; @@ -140,8 +139,8 @@ public unsafe GarnetStatus ListPop(ArgSlice key, int count, List /// /// /// The count elements popped from the list - public unsafe GarnetStatus ListPopMultiple(ArgSlice[] keys, OperationDirection direction, int count, ref TObjectContext objectContext, out ArgSlice key, out ArgSlice[] elements) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus ListPopMultiple(PinnedSpanByte[] keys, OperationDirection direction, int count, ref TObjectContext objectContext, out PinnedSpanByte key, out PinnedSpanByte[] elements) + where TObjectContext : ITsavoriteContext { foreach (var k in keys) { @@ -172,11 +171,11 @@ public unsafe GarnetStatus ListPopMultiple(ArgSlice[] keys, Oper /// /// /// - /// + /// /// /// - public unsafe GarnetStatus ListLength(ArgSlice key, ref TObjectContext objectStoreContext, out int count) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus ListLength(PinnedSpanByte key, ref TObjectContext objectContext, out int count) + where TObjectContext : ITsavoriteContext { count = 0; @@ -186,8 +185,9 @@ public unsafe GarnetStatus ListLength(ArgSlice key, ref TObjectC // Prepare the input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LLEN }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); count = output.result1; return status; @@ -203,33 +203,35 @@ public unsafe GarnetStatus ListLength(ArgSlice key, ref TObjectC /// /// out parameter, The element being popped and pushed /// GarnetStatus - public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element) + public GarnetStatus ListMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, OperationDirection sourceDirection, OperationDirection destinationDirection, out byte[] element) { element = default; - var objectLockableContext = txnManager.ObjectStoreLockableContext; + var objectTransactionalContext = txnManager.ObjectTransactionalContext; - if (objectLockableContext.Session is null) + if (objectTransactionalContext.Session is null) ThrowObjectStoreUninitializedException(); // If source and destination are the same, the operation is equivalent to removing the last element from the list // and pushing it as first element of the list, so it can be considered as a list rotation command. - bool sameKey = sourceKey.ReadOnlySpan.SequenceEqual(destinationKey.ReadOnlySpan); + var sameKey = sourceKey.ReadOnlySpan.SequenceEqual(destinationKey.ReadOnlySpan); - bool createTransaction = false; + var createTransaction = false; if (txnManager.state != TxnState.Running) { createTransaction = true; - txnManager.SaveKeyEntryToLock(sourceKey, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(destinationKey, true, LockType.Exclusive); - txnManager.Run(true); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(sourceKey, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(destinationKey, LockType.Exclusive); + _ = txnManager.Run(true); } - var objectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; + var unifiedContext = txnManager.UnifiedTransactionalContext; try { // Get the source key - var statusOp = GET(sourceKey.ToArray(), out var sourceList, ref objectLockableContext); + var statusOp = GET(sourceKey, out var sourceList, ref objectTransactionalContext); if (statusOp == GarnetStatus.NOTFOUND) { @@ -247,8 +249,7 @@ public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, Operat if (!sameKey) { // Read destination key - var arrDestKey = destinationKey.ToArray(); - statusOp = GET(arrDestKey, out var destinationList, ref objectStoreLockableContext); + statusOp = GET(destinationKey, out var destinationList, ref objectContext); if (statusOp == GarnetStatus.NOTFOUND) { @@ -289,29 +290,28 @@ public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, Operat { if (srcListObject.LnkList.Count == 0) { - _ = EXPIRE(sourceKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectLockableContext); + _ = EXPIRE(sourceKey, TimeSpan.Zero, out _, ExpireOption.None, ref unifiedContext); } // Left push (addfirst) to destination if (destinationDirection == OperationDirection.Left) - dstListObject.LnkList.AddFirst(element); + _ = dstListObject.LnkList.AddFirst(element); else - dstListObject.LnkList.AddLast(element); + _ = dstListObject.LnkList.AddLast(element); dstListObject.UpdateSize(element); - newListValue = new ListObject(dstListObject.LnkList, dstListObject.Expiration, dstListObject.Size); + newListValue = new ListObject(dstListObject.LnkList, dstListObject.HeapMemorySize); // Upsert - SET(destinationKey.ToArray(), newListValue, ref objectStoreLockableContext); + _ = SET(destinationKey, newListValue, ref objectContext); } else { // When the source and the destination key is the same the operation is done only in the sourceList if (destinationDirection == OperationDirection.Left) - srcListObject.LnkList.AddFirst(element); + _ = srcListObject.LnkList.AddFirst(element); else if (destinationDirection == OperationDirection.Right) - srcListObject.LnkList.AddLast(element); + _ = srcListObject.LnkList.AddLast(element); newListValue = srcListObject; ((ListObject)newListValue).UpdateSize(element); } @@ -323,7 +323,7 @@ public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, Operat txnManager.Commit(true); } - itemBroker.HandleCollectionUpdate(destinationKey.Span.ToArray()); + itemBroker?.HandleCollectionUpdate(destinationKey.Span.ToArray()); return GarnetStatus.OK; } @@ -334,16 +334,17 @@ public GarnetStatus ListMove(ArgSlice sourceKey, ArgSlice destinationKey, Operat /// /// /// - /// + /// /// true when successful - public unsafe bool ListTrim(ArgSlice key, int start, int stop, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe bool ListTrim(PinnedSpanByte key, int start, int stop, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { // Prepare the input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LTRIM }; var input = new ObjectInput(header, start, stop); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out _, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); return status == GarnetStatus.OK; } @@ -355,13 +356,13 @@ public unsafe bool ListTrim(ArgSlice key, int start, int stop, r /// /// /// - /// + /// /// - public GarnetStatus ListPush(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus ListPush(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); - itemBroker.HandleCollectionUpdate(key); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + itemBroker?.HandleCollectionUpdate(key.ToArray()); return status; } @@ -373,12 +374,12 @@ public GarnetStatus ListPush(byte[] key, ref ObjectInput input, /// /// /// - /// + /// /// - public GarnetStatus ListPosition(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus ListPosition(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - return ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + return ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); } /// @@ -387,11 +388,14 @@ public GarnetStatus ListPosition(byte[] key, ref ObjectInput inp /// /// /// - /// + /// /// - public GarnetStatus ListTrim(byte[] key, ref ObjectInput input, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out _, ref objectStoreContext); + public GarnetStatus ListTrim(PinnedSpanByte key, ref ObjectInput input, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + var output = new ObjectOutput(); + return RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + } /// /// Gets the specified elements of the list stored at key. @@ -400,11 +404,11 @@ public GarnetStatus ListTrim(byte[] key, ref ObjectInput input, /// /// /// - /// + /// /// - public GarnetStatus ListRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus ListRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Inserts a new element in the list stored at key either before or after a value pivot @@ -413,13 +417,13 @@ public GarnetStatus ListRange(byte[] key, ref ObjectInput input, /// /// /// - /// + /// /// - public GarnetStatus ListInsert(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus ListInsert(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); - itemBroker.HandleCollectionUpdate(key); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + itemBroker?.HandleCollectionUpdate(key.ToArray()); return status; } @@ -430,11 +434,11 @@ public GarnetStatus ListInsert(byte[] key, ref ObjectInput input /// /// /// - /// + /// /// - public GarnetStatus ListIndex(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus ListIndex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes the first count occurrences of elements equal to element from the list. @@ -444,11 +448,11 @@ public GarnetStatus ListIndex(byte[] key, ref ObjectInput input, /// /// /// - /// + /// /// - public GarnetStatus ListRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus ListRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes the count elements from the head(left) or tail(right) of the list stored at key. @@ -458,11 +462,11 @@ public GarnetStatus ListRemove(byte[] key, ref ObjectInput input /// /// /// - /// + /// /// - public unsafe GarnetStatus ListPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public unsafe GarnetStatus ListPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes the count elements from the head(left) or tail(right) of the list stored at key. @@ -472,11 +476,11 @@ public unsafe GarnetStatus ListPop(byte[] key, ref ObjectInput i /// /// /// - /// + /// /// - public unsafe GarnetStatus ListLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public unsafe GarnetStatus ListLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Sets the list element at index to element. @@ -485,10 +489,10 @@ public unsafe GarnetStatus ListLength(byte[] key, ref ObjectInpu /// /// /// - /// + /// /// - public unsafe GarnetStatus ListSet(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public unsafe GarnetStatus ListSet(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); } } \ No newline at end of file diff --git a/libs/server/Storage/Session/ObjectStore/SetOps.cs b/libs/server/Storage/Session/ObjectStore/SetOps.cs index c1c6c9c0b89..f4e245269cb 100644 --- a/libs/server/Storage/Session/ObjectStore/SetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SetOps.cs @@ -4,13 +4,11 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Server session for RESP protocol - SET /// @@ -25,10 +23,10 @@ sealed partial class StorageSession : IDisposable /// ArgSlice with key /// /// - /// + /// /// - internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice member, out int saddCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte member, out int saddCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { saddCount = 0; @@ -38,8 +36,9 @@ internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice membe // Prepare the input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SADD }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); saddCount = output.result1; return status; @@ -54,10 +53,10 @@ internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice membe /// ArgSlice with key /// /// - /// + /// /// - internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice[] members, out int saddCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetAdd(PinnedSpanByte key, PinnedSpanByte[] members, out int saddCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { saddCount = 0; @@ -70,11 +69,10 @@ internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice[] mem // Prepare the input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SADD }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); // Iterate through all inputs and add them to the scratch buffer in RESP format - - - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); saddCount = output.result1; return status; @@ -88,10 +86,10 @@ internal unsafe GarnetStatus SetAdd(ArgSlice key, ArgSlice[] mem /// ArgSlice with key /// /// - /// + /// /// - internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice member, out int sremCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte member, out int sremCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { sremCount = 0; @@ -101,8 +99,9 @@ internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice me // Prepare the input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SREM }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); sremCount = output.result1; return status; @@ -118,10 +117,10 @@ internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice me /// ArgSlice with key /// /// - /// + /// /// - internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice[] members, out int sremCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int sremCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { sremCount = 0; @@ -134,8 +133,9 @@ internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice[] // Prepare the input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SREM }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); sremCount = output.result1; return status; @@ -147,10 +147,10 @@ internal unsafe GarnetStatus SetRemove(ArgSlice key, ArgSlice[] /// /// /// - /// + /// /// - internal unsafe GarnetStatus SetLength(ArgSlice key, out int count, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetLength(PinnedSpanByte key, out int count, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { count = 0; @@ -160,8 +160,9 @@ internal unsafe GarnetStatus SetLength(ArgSlice key, out int cou // Prepare the input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SCARD }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); count = output.result1; return status; @@ -173,10 +174,10 @@ internal unsafe GarnetStatus SetLength(ArgSlice key, out int cou /// /// /// - /// + /// /// - internal unsafe GarnetStatus SetMembers(ArgSlice key, out ArgSlice[] members, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetMembers(PinnedSpanByte key, out PinnedSpanByte[] members, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { members = default; @@ -187,9 +188,9 @@ internal unsafe GarnetStatus SetMembers(ArgSlice key, out ArgSli var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SMEMBERS }; var input = new ObjectInput(header); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); if (status == GarnetStatus.OK) members = ProcessRespArrayOutput(output, out _); @@ -203,12 +204,12 @@ internal unsafe GarnetStatus SetMembers(ArgSlice key, out ArgSli /// /// /// - /// + /// /// - internal GarnetStatus SetPop(ArgSlice key, out ArgSlice element, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal GarnetStatus SetPop(PinnedSpanByte key, out PinnedSpanByte element, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = SetPop(key, int.MinValue, out var elements, ref objectStoreContext); + var status = SetPop(key, int.MinValue, out var elements, ref objectContext); element = default; if (status == GarnetStatus.OK && elements != default) element = elements[0]; @@ -223,10 +224,10 @@ internal GarnetStatus SetPop(ArgSlice key, out ArgSlice element, /// /// /// - /// + /// /// - internal unsafe GarnetStatus SetPop(ArgSlice key, int count, out ArgSlice[] elements, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + internal unsafe GarnetStatus SetPop(PinnedSpanByte key, int count, out PinnedSpanByte[] elements, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { elements = default; @@ -237,9 +238,9 @@ internal unsafe GarnetStatus SetPop(ArgSlice key, int count, out var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SPOP }; var input = new ObjectInput(header, count); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); if (status != GarnetStatus.OK) return status; @@ -259,7 +260,7 @@ internal unsafe GarnetStatus SetPop(ArgSlice key, int count, out /// /// /// - internal unsafe GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey, ArgSlice member, out int smoveResult) + internal unsafe GarnetStatus SetMove(PinnedSpanByte sourceKey, PinnedSpanByte destinationKey, PinnedSpanByte member, out int smoveResult) { smoveResult = 0; @@ -267,19 +268,18 @@ internal unsafe GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey if (txnManager.state != TxnState.Running) { createTransaction = true; - txnManager.SaveKeyEntryToLock(sourceKey, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(destinationKey, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(sourceKey, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(destinationKey, LockType.Exclusive); _ = txnManager.Run(true); } - var objectLockableContext = txnManager.ObjectStoreLockableContext; + var objectTransactionalContext = txnManager.ObjectTransactionalContext; + var unifiedTransactionalContext = txnManager.UnifiedTransactionalContext; try { - var arrDstKey = destinationKey.ToArray(); - var arrSrcKey = sourceKey.ToArray(); - - var srcGetStatus = GET(arrSrcKey, out var srcObject, ref objectLockableContext); + var srcGetStatus = GET(sourceKey, out var srcObject, ref objectTransactionalContext); if (srcGetStatus == GarnetStatus.NOTFOUND) return GarnetStatus.NOTFOUND; @@ -292,7 +292,7 @@ internal unsafe GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey if (sameKey) return GarnetStatus.OK; - var dstGetStatus = GET(arrDstKey, out var dstObject, ref objectLockableContext); + var dstGetStatus = GET(destinationKey, out var dstObject, ref objectTransactionalContext); SetObject dstSetObject; if (dstGetStatus == GarnetStatus.OK) @@ -316,16 +316,15 @@ internal unsafe GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey if (srcSetObject.Set.Count == 0) { - _ = EXPIRE(sourceKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectLockableContext); + _ = EXPIRE(sourceKey, TimeSpan.Zero, out _, ExpireOption.None, ref unifiedTransactionalContext); } - dstSetObject.Set.Add(arrMember); + _ = dstSetObject.Set.Add(arrMember); dstSetObject.UpdateSize(arrMember); if (dstGetStatus == GarnetStatus.NOTFOUND) { - var setStatus = SET(arrDstKey, dstSetObject, ref objectLockableContext); + var setStatus = SET(destinationKey, dstSetObject, ref objectTransactionalContext); if (setStatus == GarnetStatus.OK) smoveResult = 1; } @@ -351,7 +350,7 @@ internal unsafe GarnetStatus SetMove(ArgSlice sourceKey, ArgSlice destinationKey /// /// /// - public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetIntersect(PinnedSpanByte[] keys, out HashSet output) { output = default; @@ -364,17 +363,18 @@ public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; try { - return SetIntersect(keys, ref setObjectStoreLockableContext, out output); + return SetIntersect(keys, ref setObjectTransactionalContext, out output); } finally { @@ -391,7 +391,7 @@ public GarnetStatus SetIntersect(ArgSlice[] keys, out HashSet output) /// /// /// - public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetIntersectStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) { count = default; @@ -400,26 +400,26 @@ public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count return GarnetStatus.OK; } - var destination = scratchBufferBuilder.CreateArgSlice(key); - var createTransaction = false; if (txnManager.state != TxnState.Running) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(key, LockType.Exclusive); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; + var setUnifiedTransactionalContext = txnManager.UnifiedTransactionalContext; try { - var status = SetIntersect(keys, ref setObjectStoreLockableContext, out var members); + var status = SetIntersect(keys, ref setObjectTransactionalContext, out var members); if (status == GarnetStatus.OK) { @@ -432,12 +432,11 @@ public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count newSetObject.UpdateSize(item); } - _ = SET(key, newSetObject, ref setObjectStoreLockableContext); + _ = SET(key, newSetObject, ref setObjectTransactionalContext); } else { - _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref setObjectStoreLockableContext); + _ = EXPIRE(key, TimeSpan.Zero, out _, ExpireOption.None, ref setUnifiedTransactionalContext); } count = members.Count; @@ -453,8 +452,8 @@ public GarnetStatus SetIntersectStore(byte[] key, ArgSlice[] keys, out int count } - private GarnetStatus SetIntersect(ReadOnlySpan keys, ref TObjectContext objectContext, out HashSet output) - where TObjectContext : ITsavoriteContext + private GarnetStatus SetIntersect(ReadOnlySpan keys, ref TObjectContext objectContext, out HashSet output) + where TObjectContext : ITsavoriteContext { output = new HashSet(ByteArrayComparer.Instance); @@ -463,17 +462,11 @@ private GarnetStatus SetIntersect(ReadOnlySpan keys, r return GarnetStatus.OK; } - var status = GET(keys[0].ToArray(), out var first, ref objectContext); - + var status = GET(keys[0], out var first, ref objectContext); if (status == GarnetStatus.NOTFOUND) - { return GarnetStatus.OK; - } - if (status == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } if (status == GarnetStatus.OK) { @@ -500,7 +493,7 @@ private GarnetStatus SetIntersect(ReadOnlySpan keys, r return GarnetStatus.OK; } - status = GET(keys[i].ToArray(), out var next, ref objectContext); + status = GET(keys[i], out var next, ref objectContext); if (status == GarnetStatus.WRONGTYPE) return GarnetStatus.WRONGTYPE; if (status == GarnetStatus.OK) @@ -530,7 +523,7 @@ private GarnetStatus SetIntersect(ReadOnlySpan keys, r /// /// /// - public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) + public GarnetStatus SetUnion(PinnedSpanByte[] keys, out HashSet output) { output = new HashSet(ByteArrayComparer.Instance); @@ -543,17 +536,18 @@ public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; try { - return SetUnion(keys, ref setObjectStoreLockableContext, out output); + return SetUnion(keys, ref setObjectTransactionalContext, out output); } finally { @@ -570,33 +564,33 @@ public GarnetStatus SetUnion(ArgSlice[] keys, out HashSet output) /// /// /// - public GarnetStatus SetUnionStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetUnionStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) { count = default; if (keys.Length == 0) return GarnetStatus.OK; - var destination = scratchBufferBuilder.CreateArgSlice(key); - var createTransaction = false; if (txnManager.state != TxnState.Running) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(key, LockType.Exclusive); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; + var setUnifiedTransactionalContext = txnManager.UnifiedTransactionalContext; try { - var status = SetUnion(keys, ref setObjectStoreLockableContext, out var members); + var status = SetUnion(keys, ref setObjectTransactionalContext, out var members); if (status == GarnetStatus.OK) { @@ -609,12 +603,11 @@ public GarnetStatus SetUnionStore(byte[] key, ArgSlice[] keys, out int count) newSetObject.UpdateSize(item); } - _ = SET(key, newSetObject, ref setObjectStoreLockableContext); + _ = SET(key, newSetObject, ref setObjectTransactionalContext); } else { - _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref setObjectStoreLockableContext); + _ = EXPIRE(key, TimeSpan.Zero, out _, ExpireOption.None, ref setUnifiedTransactionalContext); } count = members.Count; @@ -629,18 +622,16 @@ public GarnetStatus SetUnionStore(byte[] key, ArgSlice[] keys, out int count) } } - private GarnetStatus SetUnion(ArgSlice[] keys, ref TObjectContext objectContext, out HashSet output) - where TObjectContext : ITsavoriteContext + private GarnetStatus SetUnion(PinnedSpanByte[] keys, ref TObjectContext objectContext, out HashSet output) + where TObjectContext : ITsavoriteContext { output = new HashSet(ByteArrayComparer.Instance); if (keys.Length == 0) - { return GarnetStatus.OK; - } foreach (var item in keys) { - if (GET(item.ToArray(), out var currObject, ref objectContext) == GarnetStatus.OK) + if (GET(item, out var currObject, ref objectContext) == GarnetStatus.OK) { if (currObject.GarnetObject is not SetObject setObject) { @@ -666,9 +657,9 @@ private GarnetStatus SetUnion(ArgSlice[] keys, ref TObjectContex /// /// /// - public GarnetStatus SetAdd(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectContext); + public GarnetStatus SetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes the specified members from the set. @@ -681,9 +672,9 @@ public GarnetStatus SetAdd(byte[] key, ref ObjectInput input, ou /// /// /// - public GarnetStatus SetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectContext); + public GarnetStatus SetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the number of elements of the set. @@ -694,9 +685,9 @@ public GarnetStatus SetRemove(byte[] key, ref ObjectInput input, /// /// /// - public GarnetStatus SetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectContext); + public GarnetStatus SetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns all members of the set at key. @@ -707,9 +698,9 @@ public GarnetStatus SetLength(byte[] key, ref ObjectInput input, /// /// /// - public GarnetStatus SetMembers(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SetMembers(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns if member is a member of the set stored at key. @@ -720,9 +711,9 @@ public GarnetStatus SetMembers(byte[] key, ref ObjectInput input /// /// /// - public GarnetStatus SetIsMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SetIsMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns whether each member is a member of the set stored at key. @@ -732,8 +723,8 @@ public GarnetStatus SetIsMember(byte[] key, ref ObjectInput inpu /// /// /// - public unsafe GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] members, out int[] result, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SetIsMember(PinnedSpanByte key, PinnedSpanByte[] members, out int[] result, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { result = default; @@ -749,8 +740,8 @@ public unsafe GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] SetOp = SetOperation.SMISMEMBER, }, ref parseState); - var output = new GarnetObjectStoreOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + var output = new ObjectOutput { SpanByteAndMemory = new SpanByteAndMemory(null) }; + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); if (status == GarnetStatus.OK) result = ProcessRespIntegerArrayOutput(output, out _); @@ -767,9 +758,9 @@ public unsafe GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] /// /// /// - public GarnetStatus SetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// When called with just the key argument, return a random element from the set value stored at key. @@ -784,9 +775,9 @@ public GarnetStatus SetPop(byte[] key, ref ObjectInput input, re /// /// /// - public GarnetStatus SetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the members of the set resulting from the difference between the first set at key and all the successive sets at keys. @@ -794,7 +785,7 @@ public GarnetStatus SetRandomMember(byte[] key, ref ObjectInput /// /// /// - public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet members) + public GarnetStatus SetDiff(PinnedSpanByte[] keys, out HashSet members) { members = default; @@ -807,17 +798,18 @@ public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet members) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; try { - return SetDiff(keys, ref setObjectStoreLockableContext, out members); + return SetDiff(keys, ref setObjectTransactionalContext, out members); } finally { @@ -834,33 +826,33 @@ public GarnetStatus SetDiff(ArgSlice[] keys, out HashSet members) /// /// /// - public GarnetStatus SetDiffStore(byte[] key, ArgSlice[] keys, out int count) + public GarnetStatus SetDiffStore(PinnedSpanByte key, PinnedSpanByte[] keys, out int count) { count = default; if (keys.Length == 0) return GarnetStatus.OK; - var destination = scratchBufferBuilder.CreateArgSlice(key); - var createTransaction = false; if (txnManager.state != TxnState.Running) { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(key, LockType.Exclusive); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; + var setUnifiedTransactionalContext = txnManager.UnifiedTransactionalContext; try { - var status = SetDiff(keys, ref setObjectStoreLockableContext, out var diffSet); + var status = SetDiff(keys, ref setObjectTransactionalContext, out var diffSet); if (status == GarnetStatus.OK) { @@ -872,12 +864,11 @@ public GarnetStatus SetDiffStore(byte[] key, ArgSlice[] keys, out int count) _ = newSetObject.Set.Add(item); newSetObject.UpdateSize(item); } - _ = SET(key, newSetObject, ref setObjectStoreLockableContext); + _ = SET(key, newSetObject, ref setObjectTransactionalContext); } else { - _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref setObjectStoreLockableContext); + _ = EXPIRE(key, TimeSpan.Zero, out _, ExpireOption.None, ref setUnifiedTransactionalContext); } count = diffSet.Count; @@ -892,8 +883,8 @@ public GarnetStatus SetDiffStore(byte[] key, ArgSlice[] keys, out int count) } } - private GarnetStatus SetDiff(ArgSlice[] keys, ref TObjectContext objectContext, out HashSet output) - where TObjectContext : ITsavoriteContext + private GarnetStatus SetDiff(PinnedSpanByte[] keys, ref TObjectContext objectContext, out HashSet output) + where TObjectContext : ITsavoriteContext { output = new HashSet(); if (keys.Length == 0) @@ -902,17 +893,11 @@ private GarnetStatus SetDiff(ArgSlice[] keys, ref TObjectContext } // first SetObject - var status = GET(keys[0].ToArray(), out var first, ref objectContext); - + var status = GET(keys[0], out var first, ref objectContext); if (status == GarnetStatus.NOTFOUND) - { return GarnetStatus.OK; - } - if (status == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } if (status == GarnetStatus.OK) { @@ -932,7 +917,7 @@ private GarnetStatus SetDiff(ArgSlice[] keys, ref TObjectContext // after SetObjects for (var i = 1; i < keys.Length; i++) { - status = GET(keys[i].ToArray(), out var next, ref objectContext); + status = GET(keys[i], out var next, ref objectContext); if (status == GarnetStatus.WRONGTYPE) return GarnetStatus.WRONGTYPE; if (status == GarnetStatus.OK) @@ -957,9 +942,9 @@ private GarnetStatus SetDiff(ArgSlice[] keys, ref TObjectContext /// Optional limit for stopping early when reaching this size /// /// - public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, out int count) { - if (txnManager.ObjectStoreLockableContext.Session is null) + if (txnManager.ObjectTransactionalContext.Session is null) ThrowObjectStoreUninitializedException(); count = 0; @@ -973,16 +958,17 @@ public GarnetStatus SetIntersectLength(ReadOnlySpan keys, int? limit, { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } - var setObjectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var setObjectTransactionalContext = txnManager.ObjectTransactionalContext; try { - var status = SetIntersect(keys, ref setObjectStoreLockableContext, out var result); + var status = SetIntersect(keys, ref setObjectTransactionalContext, out var result); if (status == GarnetStatus.OK && result != null) { count = limit > 0 ? Math.Min(result.Count, limit.Value) : result.Count; diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs index 7168e54df46..7febc81f28c 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs @@ -8,9 +8,6 @@ namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { /// @@ -23,9 +20,9 @@ sealed partial class StorageSession : IDisposable /// /// /// - public GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus GeoAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// GEOHASH: Returns valid Geohash strings representing the position of one or more elements in a geospatial data of the sorted set. @@ -38,9 +35,9 @@ public GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, re /// /// /// - public GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus GeoCommands(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Geospatial search and return result.. @@ -57,11 +54,11 @@ public GarnetStatus GeoCommands(byte[] key, ref ObjectInput inpu /// /// /// - public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearchOptions opts, + public GarnetStatus GeoSearchReadOnly(PinnedSpanByte key, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext { var createTransaction = false; @@ -69,14 +66,15 @@ public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearc { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(key, true, LockType.Shared); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); + txnManager.SaveKeyEntryToLock(key, LockType.Shared); txnManager.Run(true); } try { // Can we optimize more when ANY is used? - var statusOp = GET(key.ToArray(), out var firstObj, ref objectContext); + var statusOp = GET(key, out var firstObj, ref objectContext); if (statusOp == GarnetStatus.OK) { if (firstObj.GarnetObject is not SortedSetObject firstSortedSet) @@ -113,12 +111,12 @@ public GarnetStatus GeoSearchReadOnly(ArgSlice key, ref GeoSearc /// /// /// - public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destination, + public unsafe GarnetStatus GeoSearchStore(PinnedSpanByte key, PinnedSpanByte destination, ref GeoSearchOptions opts, ref ObjectInput input, ref SpanByteAndMemory output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext { var createTransaction = false; @@ -126,11 +124,13 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(key, true, LockType.Shared); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(destination, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(key, LockType.Shared); _ = txnManager.Run(true); } - var objectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var geoObjectTransactionalContext = txnManager.ObjectTransactionalContext; + var geoUnifiedTransactionalContext = txnManager.UnifiedTransactionalContext; using var writer = new RespMemoryWriter(functionsState.respProtocolVersion, ref output); @@ -138,18 +138,13 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice { SpanByteAndMemory searchOutMem = default; - var status = GET(key.ToArray(), out var firstObj, ref objectStoreLockableContext); + var status = GET(key, out var firstObj, ref geoObjectTransactionalContext); if (status == GarnetStatus.OK) { if (firstObj.GarnetObject is SortedSetObject firstSortedSet) - { - firstSortedSet.GeoSearch(ref input, ref searchOutMem, functionsState.respProtocolVersion, - ref opts, false); - } + firstSortedSet.GeoSearch(ref input, ref searchOutMem, functionsState.respProtocolVersion, ref opts, false); else - { status = GarnetStatus.WRONGTYPE; - } } if (status == GarnetStatus.WRONGTYPE) @@ -160,7 +155,7 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice if (status == GarnetStatus.NOTFOUND) { // Expire/Delete the destination key if the source key is not found - _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, ref lockableContext, ref objectStoreLockableContext); + _ = EXPIRE(destination, TimeSpan.Zero, out _, ExpireOption.None, ref geoUnifiedTransactionalContext); writer.WriteInt32(0); return GarnetStatus.OK; } @@ -180,23 +175,22 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice return GarnetStatus.OK; } - var destinationKey = destination.ToArray(); - objectStoreLockableContext.Delete(ref destinationKey); + _ = geoObjectTransactionalContext.Delete((FixedSpanByteKey)destination); - RespReadUtils.TryReadUnsignedArrayLength(out var foundItems, ref currOutPtr, endOutPtr); + _ = RespReadUtils.TryReadUnsignedArrayLength(out var foundItems, ref currOutPtr, endOutPtr); // Prepare the parse state for sorted set add parseState.Initialize(foundItems * 2); for (var j = 0; j < foundItems; j++) { - RespReadUtils.TryReadUnsignedArrayLength(out var innerLength, ref currOutPtr, endOutPtr); + _ = RespReadUtils.TryReadUnsignedArrayLength(out var innerLength, ref currOutPtr, endOutPtr); Debug.Assert(innerLength == 2, "Should always has location and hash or distance"); // Read location into parse state - parseState.Read((2 * j) + 1, ref currOutPtr, endOutPtr); + _ = parseState.Read((2 * j) + 1, ref currOutPtr, endOutPtr); // Read score into parse state - parseState.Read(2 * j, ref currOutPtr, endOutPtr); + _ = parseState.Read(2 * j, ref currOutPtr, endOutPtr); } // Prepare the input @@ -206,10 +200,10 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice SortedSetOp = SortedSetOperation.ZADD, }, ref parseState); - var zAddOutput = new GarnetObjectStoreOutput(); + var zAddOutput = new ObjectOutput(); try { - RMWObjectStoreOperationWithOutput(destinationKey, ref zAddInput, ref objectStoreLockableContext, ref zAddOutput); + RMWObjectStoreOperation(destination, ref zAddInput, ref geoObjectTransactionalContext, ref zAddOutput); writer.WriteInt32(foundItems); } @@ -225,9 +219,7 @@ public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice finally { searchOutHandler.Dispose(); - // GeoSearch writes via RespMemoryWriter, which (with a default SpanByte) rents a - // MemoryPool buffer and assigns it here. Dispose to release it back to the pool. - searchOutMem.Memory?.Dispose(); + searchOutMem.Dispose(); } return GarnetStatus.OK; diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index e4188af7e0c..44291f4b063 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -12,9 +12,6 @@ namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - sealed partial class StorageSession : IDisposable { private SingleWriterMultiReaderLock _zcollectTaskLock; @@ -27,10 +24,10 @@ sealed partial class StorageSession : IDisposable /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetAdd(ArgSlice key, ArgSlice score, ArgSlice member, out int zaddCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetAdd(PinnedSpanByte key, PinnedSpanByte score, PinnedSpanByte member, out int zaddCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { zaddCount = 0; if (key.Length == 0) @@ -43,16 +40,13 @@ public unsafe GarnetStatus SortedSetAdd(ArgSlice key, ArgSlice s var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZADD }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var keyBytes = key.ToArray(); - var status = RMWObjectStoreOperationWithOutput(keyBytes, ref input, ref objectStoreContext, ref output); - itemBroker.HandleCollectionUpdate(keyBytes); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + itemBroker?.HandleCollectionUpdate(key.ToArray()); if (status == GarnetStatus.OK) - { zaddCount = TryProcessRespSimple64IntOutput(output, out var value) ? (int)value : default; - } return status; } @@ -65,10 +59,10 @@ public unsafe GarnetStatus SortedSetAdd(ArgSlice key, ArgSlice s /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetAdd(ArgSlice key, (ArgSlice score, ArgSlice member)[] inputs, out int zaddCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetAdd(PinnedSpanByte key, (PinnedSpanByte score, PinnedSpanByte member)[] inputs, out int zaddCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { zaddCount = 0; @@ -86,11 +80,10 @@ public unsafe GarnetStatus SortedSetAdd(ArgSlice key, (ArgSlice var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZADD }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var keyBytes = key.ToArray(); - var status = RMWObjectStoreOperationWithOutput(keyBytes, ref input, ref objectStoreContext, ref output); - itemBroker.HandleCollectionUpdate(keyBytes); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + itemBroker?.HandleCollectionUpdate(key.ToArray()); if (status == GarnetStatus.OK) { @@ -108,12 +101,11 @@ public unsafe GarnetStatus SortedSetAdd(ArgSlice key, (ArgSlice /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice member, out int zremCount, - ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte member, out int zremCount, + ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { zremCount = 0; @@ -126,8 +118,9 @@ public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice // Prepare the input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREM }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key, ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); zremCount = output.result1; return status; @@ -141,10 +134,10 @@ public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice[] members, out int zremCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRemove(PinnedSpanByte key, PinnedSpanByte[] members, out int zremCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { zremCount = 0; @@ -156,8 +149,9 @@ public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice[ // Prepare the input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREM }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key, ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); zremCount = output.result1; return status; @@ -171,12 +165,11 @@ public unsafe GarnetStatus SortedSetRemove(byte[] key, ArgSlice[ /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetRemoveRangeByLex(ArgSlice key, string min, string max, - out int countRemoved, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, string min, string max, + out int countRemoved, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { countRemoved = 0; @@ -190,11 +183,11 @@ public unsafe GarnetStatus SortedSetRemoveRangeByLex(ArgSlice ke // Store parameters to buffer var minSpan = paramsSpan.Slice(0, min.Length); Encoding.UTF8.GetBytes(min, minSpan); - var minSlice = ArgSlice.FromPinnedSpan(minSpan); + var minSlice = PinnedSpanByte.FromPinnedSpan(minSpan); var maxSpan = paramsSpan.Slice(min.Length, max.Length); Encoding.UTF8.GetBytes(max, maxSpan); - var maxSlice = ArgSlice.FromPinnedSpan(maxSpan); + var maxSlice = PinnedSpanByte.FromPinnedSpan(maxSpan); // Prepare the parse state parseState.InitializeWithArguments(minSlice, maxSlice); @@ -202,11 +195,12 @@ public unsafe GarnetStatus SortedSetRemoveRangeByLex(ArgSlice ke // Prepare the input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREMRANGEBYLEX }; var input = new ObjectInput(header, ref parseState); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); countRemoved = output.result1; - scratchBufferBuilder.RewindScratchBuffer(ref paramsSlice); + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); return status; } @@ -219,12 +213,11 @@ public unsafe GarnetStatus SortedSetRemoveRangeByLex(ArgSlice ke /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetRemoveRangeByScore(ArgSlice key, string min, string max, - out int countRemoved, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRemoveRangeByScore(PinnedSpanByte key, string min, string max, + out int countRemoved, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { countRemoved = 0; @@ -238,11 +231,11 @@ public unsafe GarnetStatus SortedSetRemoveRangeByScore(ArgSlice // Store parameters to buffer var minSpan = paramsSpan.Slice(0, min.Length); Encoding.UTF8.GetBytes(min, minSpan); - var minSlice = ArgSlice.FromPinnedSpan(minSpan); + var minSlice = PinnedSpanByte.FromPinnedSpan(minSpan); var maxSpan = paramsSpan.Slice(min.Length, max.Length); Encoding.UTF8.GetBytes(max, maxSpan); - var maxSlice = ArgSlice.FromPinnedSpan(maxSpan); + var maxSlice = PinnedSpanByte.FromPinnedSpan(maxSpan); // Prepare the parse state parseState.InitializeWithArguments(minSlice, maxSlice); @@ -251,12 +244,12 @@ public unsafe GarnetStatus SortedSetRemoveRangeByScore(ArgSlice var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREMRANGEBYSCORE }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); - scratchBufferBuilder.RewindScratchBuffer(ref paramsSlice); + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); if (status == GarnetStatus.OK) { @@ -276,12 +269,11 @@ public unsafe GarnetStatus SortedSetRemoveRangeByScore(ArgSlice /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetRemoveRangeByRank(ArgSlice key, int start, int stop, - out int countRemoved, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRemoveRangeByRank(PinnedSpanByte key, int start, int stop, + out int countRemoved, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { countRemoved = 0; @@ -300,11 +292,11 @@ public unsafe GarnetStatus SortedSetRemoveRangeByRank(ArgSlice k // Store parameters to buffer var startSpan = paramsSpan.Slice(0, startLen); NumUtils.WriteInt64(start, startSpan); - var startSlice = ArgSlice.FromPinnedSpan(startSpan); + var startSlice = PinnedSpanByte.FromPinnedSpan(startSpan); var stopSpan = paramsSpan.Slice(startLen, stopLen); NumUtils.WriteInt64(stop, stopSpan); - var stopSlice = ArgSlice.FromPinnedSpan(stopSpan); + var stopSlice = PinnedSpanByte.FromPinnedSpan(stopSpan); parseState.InitializeWithArguments(startSlice, stopSlice); @@ -312,12 +304,12 @@ public unsafe GarnetStatus SortedSetRemoveRangeByRank(ArgSlice k var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREMRANGEBYRANK }; var input = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, + status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); - scratchBufferBuilder.RewindScratchBuffer(ref paramsSlice); + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); if (status == GarnetStatus.OK) { @@ -335,10 +327,10 @@ public unsafe GarnetStatus SortedSetRemoveRangeByRank(ArgSlice k /// /// When true return the lowest scores, otherwise the highest. /// - /// + /// /// - public unsafe GarnetStatus SortedSetPop(ArgSlice key, int count, bool lowScoresFirst, out (ArgSlice member, ArgSlice score)[] pairs, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetPop(PinnedSpanByte key, int count, bool lowScoresFirst, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { pairs = default; if (key.Length == 0) @@ -349,9 +341,9 @@ public unsafe GarnetStatus SortedSetPop(ArgSlice key, int count, var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; var input = new ObjectInput(header, count, 2); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); //process output if (status == GarnetStatus.OK) @@ -370,12 +362,11 @@ public unsafe GarnetStatus SortedSetPop(ArgSlice key, int count, /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetIncrement(ArgSlice key, double increment, ArgSlice member, - out double newScore, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetIncrement(PinnedSpanByte key, double increment, PinnedSpanByte member, + out double newScore, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { newScore = 0; @@ -393,10 +384,12 @@ public unsafe GarnetStatus SortedSetIncrement(ArgSlice key, doub var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZINCRBY }; var input = new ObjectInput(header, ref parseState, arg2: 2); - var output = new GarnetObjectStoreOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, + var output = new ObjectOutput(); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + scratchBufferBuilder.RewindScratchBuffer(incrSlice); + // Process output if (status == GarnetStatus.OK) { @@ -418,10 +411,10 @@ public unsafe GarnetStatus SortedSetIncrement(ArgSlice key, doub /// /// /// - /// + /// /// - public unsafe GarnetStatus SortedSetLength(ArgSlice key, out int zcardCount, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetLength(PinnedSpanByte key, out int zcardCount, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { zcardCount = 0; @@ -431,8 +424,9 @@ public unsafe GarnetStatus SortedSetLength(ArgSlice key, out int // Prepare the input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCARD }; var input = new ObjectInput(header); + var output = new ObjectOutput(); - var status = ReadObjectStoreOperation(key.ToArray(), ref input, out var output, ref objectStoreContext); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); zcardCount = output.result1; return status; @@ -455,8 +449,8 @@ public unsafe GarnetStatus SortedSetLength(ArgSlice key, out int /// /// /// - public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice min, ArgSlice max, SortedSetOrderOperation sortedSetOrderOperation, ref TObjectContext objectContext, out ArgSlice[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRange(PinnedSpanByte key, PinnedSpanByte min, PinnedSpanByte max, SortedSetOrderOperation sortedSetOrderOperation, ref TObjectContext objectContext, out PinnedSpanByte[] elements, out string error, bool withScores = false, bool reverse = false, (string, int) limit = default) + where TObjectContext : ITsavoriteContext { elements = default; error = default; @@ -476,26 +470,39 @@ public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice _ => SortedSetRangeOpts.None }; - var arguments = new List { min, max }; + var arguments = new List { min, max }; if (reverse) { rangeOpts |= SortedSetRangeOpts.Reverse; } - // Limit parameter + // Limit parameter — single contiguous allocation to avoid + // ScratchBufferBuilder reallocation invalidating earlier pointers. + PinnedSpanByte paramsSlice = default; if (limit != default && (sortedSetOrderOperation == SortedSetOrderOperation.ByScore || sortedSetOrderOperation == SortedSetOrderOperation.ByLex)) { - arguments.Add(scratchBufferBuilder.CreateArgSlice("LIMIT"u8)); + var limitKeywordBytes = "LIMIT"u8; + var offsetLength = Encoding.UTF8.GetByteCount(limit.Item1); + var countLength = NumUtils.CountDigits(limit.Item2); + var totalSize = limitKeywordBytes.Length + offsetLength + countLength; + + paramsSlice = scratchBufferBuilder.CreateArgSlice(totalSize); + var ptr = paramsSlice.ptr; + + // LIMIT keyword + limitKeywordBytes.CopyTo(new Span(ptr, limitKeywordBytes.Length)); + arguments.Add(PinnedSpanByte.FromPinnedPointer(ptr, limitKeywordBytes.Length)); + ptr += limitKeywordBytes.Length; // Offset - arguments.Add(scratchBufferBuilder.CreateArgSlice(limit.Item1)); + Encoding.UTF8.GetBytes(limit.Item1, new Span(ptr, offsetLength)); + arguments.Add(PinnedSpanByte.FromPinnedPointer(ptr, offsetLength)); + ptr += offsetLength; // Count - var limitCountLength = NumUtils.CountDigits(limit.Item2); - var limitCountSlice = scratchBufferBuilder.CreateArgSlice(limitCountLength); - NumUtils.WriteInt64(limit.Item2, limitCountSlice.Span); - arguments.Add(limitCountSlice); + NumUtils.WriteInt64(limit.Item2, new Span(ptr, countLength)); + arguments.Add(PinnedSpanByte.FromPinnedPointer(ptr, countLength)); } parseState.InitializeWithArguments([.. arguments]); @@ -504,14 +511,11 @@ public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZRANGE }; var input = new ObjectInput(header, ref parseState, arg2: (int)rangeOpts); - var output = new GarnetObjectStoreOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + var output = new ObjectOutput(); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); - for (var i = arguments.Count - 1; i > 1; i--) - { - var currSlice = arguments[i]; - scratchBufferBuilder.RewindScratchBuffer(ref currSlice); - } + if (paramsSlice.Length > 0) + scratchBufferBuilder.RewindScratchBuffer(paramsSlice); if (status == GarnetStatus.OK) elements = ProcessRespArrayOutput(output, out error); @@ -526,7 +530,7 @@ public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice /// /// /// - public unsafe GarnetStatus SortedSetDifference(ReadOnlySpan keys, out SortedSet<(double, byte[])> pairs) + public unsafe GarnetStatus SortedSetDifference(ReadOnlySpan keys, out SortedSet<(double, byte[])> pairs) { pairs = default; @@ -539,12 +543,13 @@ public unsafe GarnetStatus SortedSetDifference(ReadOnlySpan keys, out { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); txnManager.Run(true); } - var objectContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; try { @@ -574,7 +579,7 @@ public unsafe GarnetStatus SortedSetDifference(ReadOnlySpan keys, out /// /// /// - public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count) + public GarnetStatus SortedSetDifferenceStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, out int count) { count = default; @@ -587,13 +592,15 @@ public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySp { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destinationKey, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(destinationKey, LockType.Exclusive); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } - var objectContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; + var unifiedContext = txnManager.UnifiedTransactionalContext; try { @@ -613,14 +620,12 @@ public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySp newSetObject.Add(element, score); } - var destinationKeyBytes = destinationKey.ToArray(); - _ = SET(destinationKeyBytes, newSetObject, ref objectContext); - itemBroker.HandleCollectionUpdate(destinationKeyBytes); + _ = SET(destinationKey, newSetObject, ref objectContext); + itemBroker?.HandleCollectionUpdate(destinationKey.ToArray()); } else { - _ = EXPIRE(destinationKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectContext); + _ = EXPIRE(destinationKey, TimeSpan.Zero, out _, ExpireOption.None, ref unifiedContext); } return status; @@ -638,10 +643,10 @@ public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySp /// The member to get the rank /// If true, the rank is calculated from low to high /// The rank of the member (null if the member does not exist) - /// + /// /// - public unsafe GarnetStatus SortedSetRank(ArgSlice key, ArgSlice member, bool reverse, out long? rank, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRank(PinnedSpanByte key, PinnedSpanByte member, bool reverse, out long? rank, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { rank = null; if (key.Length == 0) @@ -657,9 +662,9 @@ public unsafe GarnetStatus SortedSetRank(ArgSlice key, ArgSlice const int outputContainerSize = 32; // 3 for HEADER + CRLF + 20 for ascii long var outputContainer = stackalloc byte[outputContainerSize]; - var output = new GarnetObjectStoreOutput(new(outputContainer, outputContainerSize)); + var output = ObjectOutput.FromPinnedPointer(outputContainer, outputContainerSize); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectStoreContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); if (status == GarnetStatus.OK) { @@ -685,13 +690,13 @@ public unsafe GarnetStatus SortedSetRank(ArgSlice key, ArgSlice /// /// /// - /// + /// /// - public GarnetStatus SortedSetAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetAdd(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - var status = RMWObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); - itemBroker.HandleCollectionUpdate(key); + var status = RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); + itemBroker?.HandleCollectionUpdate(key.ToArray()); return status; } @@ -703,12 +708,12 @@ public GarnetStatus SortedSetAdd(byte[] key, ref ObjectInput inp /// The source key from which the range will be taken. /// The input object containing range parameters. /// The result of the operation, indicating the number of elements stored. - /// The context of the object store. + /// The context of the object store. /// Returns a GarnetStatus indicating the success or failure of the operation. - public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, ArgSlice srcKey, ref ObjectInput input, out int result, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetRangeStore(PinnedSpanByte dstKey, PinnedSpanByte srcKey, ref ObjectInput input, out int result, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - if (txnManager.ObjectStoreLockableContext.Session is null) + if (txnManager.ObjectTransactionalContext.Session is null) ThrowObjectStoreUninitializedException(); result = 0; @@ -722,31 +727,30 @@ public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(dstKey, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(srcKey, true, LockType.Shared); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(dstKey, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(srcKey, LockType.Shared); _ = txnManager.Run(true); } // SetObject - var objectStoreLockableContext = txnManager.ObjectStoreLockableContext; + var ssObjectTransactionalContext = txnManager.ObjectTransactionalContext; + var ssUnifiedTransactionalContext = txnManager.UnifiedTransactionalContext; try { SpanByteAndMemory rangeOutputMem = default; - var rangeOutput = new GarnetObjectStoreOutput(rangeOutputMem); - - var status = SortedSetRange(srcKey.ToArray(), ref input, ref rangeOutput, ref objectStoreLockableContext); + var rangeOutput = new ObjectOutput(rangeOutputMem); + var status = SortedSetRange(srcKey, ref input, ref rangeOutput, ref ssObjectTransactionalContext); rangeOutputMem = rangeOutput.SpanByteAndMemory; if (status == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } if (status == GarnetStatus.NOTFOUND) { // Expire/Delete the destination key if the source key is not found - _ = EXPIRE(dstKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, ref lockableContext, ref objectStoreLockableContext); + _ = EXPIRE(dstKey, TimeSpan.Zero, out _, ExpireOption.None, ref ssUnifiedTransactionalContext); return GarnetStatus.OK; } @@ -759,8 +763,8 @@ public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, ref var currOutPtr = ref rangeOutPtr; var endOutPtr = rangeOutPtr + rangeOutputMem.Length; - var destinationKey = dstKey.ToArray(); - objectStoreLockableContext.Delete(ref destinationKey); + var destinationKey = dstKey.ReadOnlySpan; + ssUnifiedTransactionalContext.Delete((FixedSpanByteKey)destinationKey); RespReadUtils.TryReadUnsignedArrayLength(out var arrayLen, ref currOutPtr, endOutPtr); Debug.Assert(arrayLen % 2 == 0, "Should always contain element and its score"); @@ -784,11 +788,11 @@ public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, SortedSetOp = SortedSetOperation.ZADD, }, ref parseState); - var zAddOutput = new GarnetObjectStoreOutput(); + var zAddOutput = new ObjectOutput(); try { - RMWObjectStoreOperationWithOutput(destinationKey, ref zAddInput, ref objectStoreLockableContext, ref zAddOutput); - itemBroker.HandleCollectionUpdate(destinationKey); + RMWObjectStoreOperation(destinationKey, ref zAddInput, ref ssObjectTransactionalContext, ref zAddOutput); + itemBroker?.HandleCollectionUpdate(destinationKey.ToArray()); } finally { @@ -805,7 +809,7 @@ public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, rangeOutputHandler.Dispose(); // SortedSetRange writes via RespMemoryWriter, which (with a default SpanByte) rents // a MemoryPool buffer and assigns it here. Dispose to release it back to the pool. - rangeOutputMem.Memory?.Dispose(); + rangeOutputMem.Dispose(); } return status; } @@ -824,11 +828,11 @@ public unsafe GarnetStatus SortedSetRangeStore(ArgSlice dstKey, /// /// /// - /// + /// /// - public GarnetStatus SortedSetRemove(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus SortedSetRemove(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the number of members of the sorted set. @@ -837,11 +841,11 @@ public GarnetStatus SortedSetRemove(byte[] key, ref ObjectInput /// /// /// - /// + /// /// - public GarnetStatus SortedSetLength(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus SortedSetLength(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the specified range of elements in the sorted set stored at key. @@ -852,11 +856,11 @@ public GarnetStatus SortedSetLength(byte[] key, ref ObjectInput /// /// /// - /// + /// /// - public GarnetStatus SortedSetRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the score of member in the sorted set at key. @@ -866,11 +870,11 @@ public GarnetStatus SortedSetRange(byte[] key, ref ObjectInput i /// /// /// - /// + /// /// - public GarnetStatus SortedSetScore(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetScore(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the scores of members in the sorted set at key. @@ -880,11 +884,11 @@ public GarnetStatus SortedSetScore(byte[] key, ref ObjectInput i /// /// /// - /// + /// /// - public GarnetStatus SortedSetScores(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetScores(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes and returns the first element from the sorted set stored at key, @@ -894,11 +898,11 @@ public GarnetStatus SortedSetScores(byte[] key, ref ObjectInput /// /// /// - /// + /// /// - public GarnetStatus SortedSetPop(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetPop(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the number of elements in the sorted set at key with a score between min and max. @@ -910,8 +914,8 @@ public GarnetStatus SortedSetPop(byte[] key, ref ObjectInput inp /// /// /// - public unsafe GarnetStatus SortedSetCount(ArgSlice key, ArgSlice minScore, ArgSlice maxScore, out int numElements, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public unsafe GarnetStatus SortedSetCount(PinnedSpanByte key, PinnedSpanByte minScore, PinnedSpanByte maxScore, out int numElements, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { numElements = 0; if (key.Length == 0) @@ -926,9 +930,9 @@ public unsafe GarnetStatus SortedSetCount(ArgSlice key, ArgSlice const int outputContainerSize = 32; // 3 for HEADER + CRLF + 20 for ascii long var outputContainer = stackalloc byte[outputContainerSize]; - var output = new GarnetObjectStoreOutput(new(outputContainer, outputContainerSize)); + var output = ObjectOutput.FromPinnedPointer(outputContainer, outputContainerSize); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + var status = ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); if (status == GarnetStatus.OK) { @@ -949,9 +953,9 @@ public unsafe GarnetStatus SortedSetCount(ArgSlice key, ArgSlice /// /// /// - public GarnetStatus SortedSetCount(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SortedSetCount(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Removes all elements in the sorted set between the @@ -963,9 +967,9 @@ public GarnetStatus SortedSetCount(byte[] key, ref ObjectInput i /// /// /// - public GarnetStatus SortedSetRemoveRangeByLex(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperation(key, ref input, out output, ref objectContext); + public GarnetStatus SortedSetRemoveRangeByLex(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the number of elements in the sorted set with a value between min and max. @@ -976,11 +980,11 @@ public GarnetStatus SortedSetRemoveRangeByLex(byte[] key, ref Ob /// /// /// - /// + /// /// - public GarnetStatus SortedSetLengthByValue(byte[] key, ref ObjectInput input, out ObjectOutputHeader output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperation(key, ref input, out output, ref objectStoreContext); + public GarnetStatus SortedSetLengthByValue(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Increments the score of member in the sorted set stored at key by increment. @@ -990,11 +994,11 @@ public GarnetStatus SortedSetLengthByValue(byte[] key, ref Objec /// /// /// - /// + /// /// - public GarnetStatus SortedSetIncrement(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetIncrement(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// ZREMRANGEBYRANK: Removes all elements in the sorted set stored at key with rank between start and stop. @@ -1006,9 +1010,9 @@ public GarnetStatus SortedSetIncrement(byte[] key, ref ObjectInp /// /// /// - public GarnetStatus SortedSetRemoveRange(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SortedSetRemoveRange(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns the rank of member in the sorted set, the scores in the sorted set are ordered from low to high @@ -1019,9 +1023,9 @@ public GarnetStatus SortedSetRemoveRange(byte[] key, ref ObjectI /// /// /// - public GarnetStatus SortedSetRank(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SortedSetRank(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Returns a random member from the sorted set key. @@ -1032,9 +1036,9 @@ public GarnetStatus SortedSetRank(byte[] key, ref ObjectInput in /// /// /// - public GarnetStatus SortedSetRandomMember(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref output); + public GarnetStatus SortedSetRandomMember(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); /// /// Iterates members of SortedSet key and their associated scores using a cursor, @@ -1044,13 +1048,13 @@ public GarnetStatus SortedSetRandomMember(byte[] key, ref Object /// /// /// - /// + /// /// - public GarnetStatus SortedSetScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectStoreContext) - where TObjectContext : ITsavoriteContext - => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref output); + public GarnetStatus SortedSetScan(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ReadOnlySpan, ref input, ref objectContext, ref output); - public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) { pairs = default; @@ -1063,12 +1067,13 @@ public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); txnManager.Run(true); } - var objectContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; try { @@ -1092,7 +1097,7 @@ public GarnetStatus SortedSetUnion(ReadOnlySpan keys, double[] weights } } - public GarnetStatus SortedSetUnionStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) + public GarnetStatus SortedSetUnionStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) { count = default; @@ -1105,13 +1110,15 @@ public GarnetStatus SortedSetUnionStore(ArgSlice destinationKey, ReadOnlySpan(ReadOnlySpan keys, ref TObjectContext objectContext, + private GarnetStatus SortedSetUnion(ReadOnlySpan keys, ref TObjectContext objectContext, out Dictionary pairs, double[] weights = null, SortedSetAggregateType aggregateType = SortedSetAggregateType.Sum) - where TObjectContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext { pairs = default; @@ -1161,12 +1166,9 @@ private GarnetStatus SortedSetUnion(ReadOnlySpan keys, return GarnetStatus.OK; // Get the first sorted set - var status = GET(keys[0].ToArray(), out var firstObj, ref objectContext); - + var status = GET(keys[0], out var firstObj, ref objectContext); if (status == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } Dictionary sortedSetDictionary = null; @@ -1199,7 +1201,7 @@ private GarnetStatus SortedSetUnion(ReadOnlySpan keys, // Process remaining sets for (var i = 1; i < keys.Length; i++) { - status = GET(keys[i].ToArray(), out var nextObj, ref objectContext); + status = GET(keys[i], out var nextObj, ref objectContext); if (status == GarnetStatus.WRONGTYPE) return GarnetStatus.WRONGTYPE; if (status != GarnetStatus.OK) @@ -1234,16 +1236,14 @@ private GarnetStatus SortedSetUnion(ReadOnlySpan keys, return GarnetStatus.OK; } - private GarnetStatus SortedSetDifference(ReadOnlySpan keys, ref TObjectContext objectContext, out Dictionary pairs) - where TObjectContext : ITsavoriteContext + private GarnetStatus SortedSetDifference(ReadOnlySpan keys, ref TObjectContext objectContext, out Dictionary pairs) + where TObjectContext : ITsavoriteContext { pairs = default; - var statusOp = GET(keys[0].ToArray(), out var firstObj, ref objectContext); + var statusOp = GET(keys[0], out var firstObj, ref objectContext); if (statusOp == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } if (statusOp == GarnetStatus.NOTFOUND) { @@ -1252,9 +1252,7 @@ private GarnetStatus SortedSetDifference(ReadOnlySpan } if (firstObj.GarnetObject is not SortedSetObject firstSortedSet) - { return GarnetStatus.WRONGTYPE; - } pairs = SortedSetObject.CopyDiff(firstSortedSet, null); if (keys.Length == 1) @@ -1265,7 +1263,7 @@ private GarnetStatus SortedSetDifference(ReadOnlySpan // read the rest of the keys for (var item = 1; item < keys.Length; item++) { - statusOp = GET(keys[item].ToArray(), out var nextObj, ref objectContext); + statusOp = GET(keys[item], out var nextObj, ref objectContext); if (statusOp == GarnetStatus.WRONGTYPE) return GarnetStatus.WRONGTYPE; if (statusOp != GarnetStatus.OK) @@ -1286,9 +1284,9 @@ private GarnetStatus SortedSetDifference(ReadOnlySpan /// /// Removes and returns up to count members and their scores from the first sorted set that contains a member. /// - public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out ArgSlice poppedKey, out (ArgSlice member, ArgSlice score)[] pairs) + public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, bool lowScoresFirst, out PinnedSpanByte poppedKey, out (PinnedSpanByte member, PinnedSpanByte score)[] pairs) { - if (txnManager.ObjectStoreLockableContext.Session is null) + if (txnManager.ObjectTransactionalContext.Session is null) ThrowObjectStoreUninitializedException(); pairs = default; @@ -1303,12 +1301,13 @@ public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var key in keys) - txnManager.SaveKeyEntryToLock(key, true, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(key, LockType.Exclusive); txnManager.Run(true); } - var storeLockableContext = txnManager.ObjectStoreLockableContext; + var transactionalContext = txnManager.ObjectTransactionalContext; try { @@ -1317,7 +1316,7 @@ public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, { if (key.Length == 0) continue; - var status = SortedSetPop(key, count, lowScoresFirst, out pairs, ref storeLockableContext); + var status = SortedSetPop(key, count, lowScoresFirst, out pairs, ref transactionalContext); if (status == GarnetStatus.OK && pairs != null && pairs.Length > 0) { poppedKey = key; @@ -1325,9 +1324,7 @@ public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, } if (status != GarnetStatus.OK && status != GarnetStatus.NOTFOUND) - { return status; - } } return GarnetStatus.OK; @@ -1342,7 +1339,7 @@ public unsafe GarnetStatus SortedSetMPop(ReadOnlySpan keys, int count, /// /// Computes the cardinality of the intersection of multiple sorted sets. /// - public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) + public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? limit, out int count) { count = 0; @@ -1358,7 +1355,7 @@ public GarnetStatus SortedSetIntersectLength(ReadOnlySpan keys, int? l /// /// Computes the intersection of multiple sorted sets and stores the resulting sorted set at destinationKey. /// - public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) + public GarnetStatus SortedSetIntersectStore(PinnedSpanByte destinationKey, ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out int count) { count = default; @@ -1371,13 +1368,15 @@ public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpa { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - txnManager.SaveKeyEntryToLock(destinationKey, true, LockType.Exclusive); + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object | TransactionStoreTypes.Unified); + txnManager.SaveKeyEntryToLock(destinationKey, LockType.Exclusive); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); _ = txnManager.Run(true); } - var objectContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; + var unifiedContext = txnManager.UnifiedTransactionalContext; try { @@ -1398,14 +1397,12 @@ public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpa newSortedSetObject.Add(element, score); } - var destinationKeyBytes = destinationKey.ToArray(); - _ = SET(destinationKeyBytes, newSortedSetObject, ref objectContext); - itemBroker.HandleCollectionUpdate(destinationKeyBytes); + _ = SET(destinationKey, newSortedSetObject, ref objectContext); + itemBroker?.HandleCollectionUpdate(destinationKey.ToArray()); } else { - _ = EXPIRE(destinationKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, - ref lockableContext, ref objectContext); + _ = EXPIRE(destinationKey, TimeSpan.Zero, out _, ExpireOption.None, ref unifiedContext); } return status; @@ -1420,7 +1417,7 @@ public GarnetStatus SortedSetIntersectStore(ArgSlice destinationKey, ReadOnlySpa /// /// Computes the intersection of multiple sorted sets and returns the result with optional weights and aggregate type. /// - public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) + public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, out SortedSet<(double, byte[])> pairs) { pairs = default; @@ -1433,12 +1430,13 @@ public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] wei { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Object); foreach (var item in keys) - txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(item, LockType.Shared); txnManager.Run(true); } - var objectContext = txnManager.ObjectStoreLockableContext; + var objectContext = txnManager.ObjectTransactionalContext; try { @@ -1474,17 +1472,14 @@ public GarnetStatus SortedSetIntersect(ReadOnlySpan keys, double[] wei /// The object context. /// The resulting dictionary of intersected elements and their scores. /// - private GarnetStatus SortedSetIntersection(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, ref TObjectContext objectContext, out Dictionary pairs) - where TObjectContext : ITsavoriteContext + private GarnetStatus SortedSetIntersection(ReadOnlySpan keys, double[] weights, SortedSetAggregateType aggregateType, ref TObjectContext objectContext, out Dictionary pairs) + where TObjectContext : ITsavoriteContext { pairs = default; - var statusOp = GET(keys[0].ToArray(), out var firstObj, ref objectContext); - + var statusOp = GET(keys[0], out var firstObj, ref objectContext); if (statusOp == GarnetStatus.WRONGTYPE) - { return GarnetStatus.WRONGTYPE; - } if (statusOp == GarnetStatus.NOTFOUND) { @@ -1512,14 +1507,12 @@ private GarnetStatus SortedSetIntersection(ReadOnlySpan(ReadOnlySpanThe type of the object context. /// The key for which to set the expiration time. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus SortedSetExpire(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetExpire(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { - return RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + return RMWObjectStoreOperation(key.ToArray(), ref input, ref objectContext, ref output); } /// @@ -1594,8 +1587,8 @@ public GarnetStatus SortedSetExpire(ArgSlice key, ref ObjectInpu /// The results of the operation, indicating the number of fields that were successfully set to expire. /// The context of the object store. /// Returns a GarnetStatus indicating the success or failure of the operation. - public GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetExpire(PinnedSpanByte key, ReadOnlySpan members, DateTimeOffset expireAt, ExpireOption expireOption, out int[] results, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { results = default; var expirationTimeInTicks = expireAt.UtcTicks; @@ -1607,8 +1600,8 @@ public GarnetStatus SortedSetExpire(ArgSlice key, ReadOnlySpan(ArgSlice key, ReadOnlySpanThe type of the object context. /// The key of the hash. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - { - return ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); - } + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => ReadObjectStoreOperation(key.ToArray(), ref input, ref objectContext, ref output); /// /// Returns the time-to-live (TTL) of a SortedSet member. @@ -1642,8 +1633,8 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, ref Object /// The array of TimeSpan representing the TTL for each member. /// The context of the object store. /// Returns a GarnetStatus indicating the success or failure of the operation. - public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySpan members, out TimeSpan[] expireIn, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetTimeToLive(PinnedSpanByte key, ReadOnlySpan members, out TimeSpan[] expireIn, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { expireIn = default; var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZTTL }; @@ -1653,14 +1644,11 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySp var isTimestamp = 0; var innerInput = new ObjectInput(header, ref parseState, arg1: isMilliseconds, arg2: isTimestamp); - var output = new GarnetObjectStoreOutput(); - var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref output); + var output = new ObjectOutput(); + var status = ReadObjectStoreOperation(key.ToArray(), ref innerInput, ref objectContext, ref output); if (status == GarnetStatus.OK) - { - expireIn = ProcessRespInt64ArrayOutput(output, out _).Select(x => TimeSpan.FromMilliseconds(x < 0 ? 0 : x)).ToArray(); - } - + expireIn = [.. ProcessRespInt64ArrayOutput(output, out _).Select(x => TimeSpan.FromMilliseconds(x < 0 ? 0 : x))]; return status; } @@ -1670,12 +1658,12 @@ public GarnetStatus SortedSetTimeToLive(ArgSlice key, ReadOnlySp /// The type of the object context. /// The key of the SortedSet. /// The input object containing the operation details. - /// The output footer object to store the result. + /// The output object to store the result. /// The object context for the operation. /// The status of the operation. - public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInput input, ref GarnetObjectStoreOutput output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - => RMWObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref output); + public GarnetStatus SortedSetPersist(PinnedSpanByte key, ref ObjectInput input, ref ObjectOutput output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + => RMWObjectStoreOperation(key.ToArray(), ref input, ref objectContext, ref output); /// /// Removes the expiration time from the specified members in the sorted set stored at the given key. @@ -1686,17 +1674,17 @@ public GarnetStatus SortedSetPersist(ArgSlice key, ref ObjectInp /// The results of the operation, indicating the number of members whose expiration time was successfully removed. /// The context of the object store. /// Returns a GarnetStatus indicating the success or failure of the operation. - public GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan members, out int[] results, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetPersist(PinnedSpanByte key, ReadOnlySpan members, out int[] results, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { results = default; var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZPERSIST }; parseState.Initialize(members.Length); parseState.SetArguments(0, members); var innerInput = new ObjectInput(header, ref parseState); - var output = new GarnetObjectStoreOutput(); + var output = new ObjectOutput(); - var status = RMWObjectStoreOperationWithOutput(key.ToArray(), ref innerInput, ref objectContext, ref output); + var status = RMWObjectStoreOperation(key.ToArray(), ref innerInput, ref objectContext, ref output); if (status == GarnetStatus.OK) { @@ -1718,17 +1706,18 @@ public GarnetStatus SortedSetPersist(ArgSlice key, ReadOnlySpan< /// If the first key is "*", all SortedSet keys are scanned in batches and the operation is performed on each key. /// Otherwise, the operation is performed on the specified keys. /// - public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref ObjectInput input, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { if (keys[0].ReadOnlySpan.SequenceEqual("*"u8)) { return ObjectCollect(keys[0], CmdStrings.ZSET, _zcollectTaskLock, ref input, ref objectContext); } + var output = new ObjectOutput(); foreach (var key in keys) { - RMWObjectStoreOperation(key.ToArray(), ref input, out _, ref objectContext); + RMWObjectStoreOperation(key.ToArray(), ref input, ref objectContext, ref output); } return GarnetStatus.OK; @@ -1745,7 +1734,7 @@ public GarnetStatus SortedSetCollect(ReadOnlySpan keys /// Otherwise, the operation is performed on the specified keys. /// public GarnetStatus SortedSetCollect(ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext { return SortedSetCollect([], ref objectContext); } @@ -1761,15 +1750,15 @@ public GarnetStatus SortedSetCollect(ref TObjectContext objectCo /// If the first key is "*", all SortedSet keys are scanned in batches and the operation is performed on each key. /// Otherwise, the operation is performed on the specified keys. /// - public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext + public GarnetStatus SortedSetCollect(ReadOnlySpan keys, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext { var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOLLECT }; var innerInput = new ObjectInput(header); if (keys.IsEmpty) { - return SortedSetCollect([ArgSlice.FromPinnedSpan("*"u8)], ref innerInput, ref objectContext); + return SortedSetCollect([PinnedSpanByte.FromPinnedSpan("*"u8)], ref innerInput, ref objectContext); } return SortedSetCollect(keys, ref innerInput, ref objectContext); diff --git a/libs/server/Storage/Session/StorageSession.cs b/libs/server/Storage/Session/StorageSession.cs index 9b12be286b5..68f83197aa7 100644 --- a/libs/server/Storage/Session/StorageSession.cs +++ b/libs/server/Storage/Session/StorageSession.cs @@ -4,17 +4,12 @@ using System; using System.Diagnostics; using System.Threading; +using Garnet.common; using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Storage Session - the internal layer that Garnet uses to perform storage operations /// @@ -26,8 +21,10 @@ sealed partial class StorageSession : IDisposable /// /// Session Contexts for main store /// - public BasicContext basicContext; - public LockableContext lockableContext; + public StringBasicContext stringBasicContext; + public StringTransactionalContext stringTransactionalContext; + public ConsistentReadStringBasicContext consistentReadContext; + public ConsistentReadStringTransactionalContext transactionalConsistentReadContext; SectorAlignedMemory sectorAlignedMemoryHll1; SectorAlignedMemory sectorAlignedMemoryHll2; @@ -39,35 +36,61 @@ sealed partial class StorageSession : IDisposable /// /// Session Contexts for object store /// - public BasicContext objectStoreBasicContext; - public LockableContext objectStoreLockableContext; + public ObjectBasicContext objectBasicContext; + public ObjectTransactionalContext objectTransactionalContext; + public ConsistentReadObjectBasicContext objectStoreConsistentReadContext; + public ConsistentReadObjectTransactionalContext objectStoreTransactionalConsistentReadContext; + + /// + /// Session Contexts for vector store + /// + public VectorBasicContext vectorBasicContext; + public VectorTransactionalContext vectorTransactionalContext; /// - /// Session Contexts for vector ops against the main store + /// Session Contexts for unified store /// - public BasicContext vectorContext; - public LockableContext vectorLockableContext; + public UnifiedBasicContext unifiedBasicContext; + public UnifiedTransactionalContext unifiedTransactionalContext; + public ConsistentReadUnifiedBasicContext unifiedStoreConsistentReadContext; + public ConsistentReadUnifiedTransactionalContext unifiedStoreTransactionalConsistentReadContext; - public readonly ScratchBufferBuilder scratchBufferBuilder; + internal readonly ScratchBufferBuilder scratchBufferBuilder; public readonly FunctionsState functionsState; + internal readonly ScratchBufferAllocator scratchBufferAllocator; public TransactionManager txnManager; public StateMachineDriver stateMachineDriver; readonly ILogger logger; private readonly CollectionItemBroker itemBroker; - public int SessionID => basicContext.Session.ID; - public int ObjectStoreSessionID => objectStoreBasicContext.Session.ID; + public int SessionID => stringBasicContext.Session.ID; + public int ObjectStoreSessionID => objectBasicContext.Session.ID; public readonly int ObjectScanCountLimit; + /// + /// Flag indicating if this is storage session that uses consistent read context + /// + readonly bool IsConsistentReadSession; + + /// + /// Read session state use to enforce prefix consistency with sharded-log + /// + readonly ReadSessionState readSessionState; + + /// + /// Vector manage instance + /// public readonly VectorManager vectorManager; public StorageSession(StoreWrapper storeWrapper, ScratchBufferBuilder scratchBufferBuilder, + ScratchBufferAllocator scratchBufferAllocator, GarnetSessionMetrics sessionMetrics, GarnetLatencyMetricsSession LatencyMetrics, int dbId, + ReadSessionState readSessionState, VectorManager vectorManager, ILogger logger = null, byte respProtocolVersion = ServerOptions.DEFAULT_RESP_VERSION) @@ -75,36 +98,52 @@ public StorageSession(StoreWrapper storeWrapper, this.sessionMetrics = sessionMetrics; this.LatencyMetrics = LatencyMetrics; this.scratchBufferBuilder = scratchBufferBuilder; + this.scratchBufferAllocator = scratchBufferAllocator; this.logger = logger; this.itemBroker = storeWrapper.itemBroker; - this.vectorManager = vectorManager; + this.IsConsistentReadSession = readSessionState != null; + this.readSessionState = readSessionState; parseState.Initialize(); + this.vectorManager = vectorManager; functionsState = storeWrapper.CreateFunctionsState(dbId, respProtocolVersion); - var functions = new MainSessionFunctions(functionsState); + var functions = new MainSessionFunctions(functionsState, readSessionState); var dbFound = storeWrapper.TryGetDatabase(dbId, out var db); Debug.Assert(dbFound); this.stateMachineDriver = db.StateMachineDriver; - var session = db.MainStore.NewSession(functions); - - var objectStoreFunctions = new ObjectSessionFunctions(functionsState); - var objectStoreSession = db.ObjectStore?.NewSession(objectStoreFunctions); - - var vectorFunctions = new VectorSessionFunctions(functionsState); - var vectorSession = db.MainStore.NewSession(vectorFunctions); + var session = db.Store.NewSession(functions, IsConsistentReadSession); - basicContext = session.BasicContext; - lockableContext = session.LockableContext; - if (objectStoreSession != null) + if (!storeWrapper.serverOptions.DisableObjects) { - objectStoreBasicContext = objectStoreSession.BasicContext; - objectStoreLockableContext = objectStoreSession.LockableContext; + var objectStoreFunctions = new ObjectSessionFunctions(functionsState, readSessionState); + var objectStoreSession = db.Store.NewSession(objectStoreFunctions, IsConsistentReadSession); + objectBasicContext = objectStoreSession.BasicContext; + objectTransactionalContext = objectStoreSession.TransactionalContext; + objectStoreConsistentReadContext = objectStoreSession.ConsistentReadContext; + objectStoreTransactionalConsistentReadContext = objectStoreSession.TransactionalConsistentReadContext; } - vectorContext = vectorSession.BasicContext; - vectorLockableContext = vectorSession.LockableContext; + + var unifiedStoreFunctions = new UnifiedSessionFunctions(functionsState, readSessionState); + var unifiedStoreSession = db.Store.NewSession(unifiedStoreFunctions, IsConsistentReadSession); + + var vectorFunctions = new VectorSessionFunctions(functionsState, readSessionState); + var vectorSession = db.Store.NewSession(vectorFunctions); + + stringBasicContext = session.BasicContext; + stringTransactionalContext = session.TransactionalContext; + consistentReadContext = session.ConsistentReadContext; + transactionalConsistentReadContext = session.TransactionalConsistentReadContext; + + unifiedBasicContext = unifiedStoreSession.BasicContext; + unifiedTransactionalContext = unifiedStoreSession.TransactionalContext; + unifiedStoreConsistentReadContext = unifiedStoreSession.ConsistentReadContext; + unifiedStoreTransactionalConsistentReadContext = unifiedStoreSession.TransactionalConsistentReadContext; + + vectorBasicContext = vectorSession.BasicContext; + vectorTransactionalContext = vectorSession.TransactionalContext; ObjectScanCountLimit = storeWrapper.serverOptions.ObjectScanCountLimit; } @@ -123,8 +162,10 @@ public void Dispose() _ = Thread.Yield(); sectorAlignedMemoryBitmap?.Dispose(); - basicContext.Session.Dispose(); - objectStoreBasicContext.Session?.Dispose(); + stringBasicContext.Session.Dispose(); + objectBasicContext.Session?.Dispose(); + unifiedBasicContext.Session?.Dispose(); + vectorBasicContext.Session?.Dispose(); sectorAlignedMemoryHll1?.Dispose(); sectorAlignedMemoryHll2?.Dispose(); } diff --git a/libs/server/Storage/Session/UnifiedStore/AdvancedOps.cs b/libs/server/Storage/Session/UnifiedStore/AdvancedOps.cs new file mode 100644 index 00000000000..cec655061cd --- /dev/null +++ b/libs/server/Storage/Session/UnifiedStore/AdvancedOps.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + sealed partial class StorageSession : IDisposable + { + public GarnetStatus Read_UnifiedStore(ReadOnlySpan key, ref UnifiedInput input, ref UnifiedOutput output, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + { + var status = unifiedContext.Read((FixedSpanByteKey)key, ref input, ref output); + + if (status.IsPending) + CompletePendingForUnifiedStoreSession(ref status, ref output, ref unifiedContext); + + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; + } + + public GarnetStatus RMW_UnifiedStore(ReadOnlySpan key, ref UnifiedInput input, ref UnifiedOutput output, ref TUnifiedContext context) + where TUnifiedContext : ITsavoriteContext + { + var status = context.RMW((FixedSpanByteKey)key, ref input, ref output); + + if (status.IsPending) + CompletePendingForUnifiedStoreSession(ref status, ref output, ref context); + + return status.Found || status.Record.Created || status.Record.InPlaceUpdated + ? GarnetStatus.OK + : GarnetStatus.NOTFOUND; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Session/UnifiedStore/CompletePending.cs b/libs/server/Storage/Session/UnifiedStore/CompletePending.cs new file mode 100644 index 00000000000..1342104e86e --- /dev/null +++ b/libs/server/Storage/Session/UnifiedStore/CompletePending.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + sealed partial class StorageSession + { + /// + /// Handles the complete pending for Unified Store session + /// + /// + /// + /// + internal static void CompletePendingForUnifiedStoreSession(ref Status status, ref UnifiedOutput output, ref TUnifiedContext unified) + where TUnifiedContext : ITsavoriteContext + { + unified.CompletePendingWithOutputs(out var completedOutputs, wait: true); + var more = completedOutputs.Next(); + Debug.Assert(more); + status = completedOutputs.Current.Status; + output = completedOutputs.Current.Output; + Debug.Assert(!completedOutputs.Next()); + completedOutputs.Dispose(); + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/Session/UnifiedStore/UnifiedStoreOps.cs b/libs/server/Storage/Session/UnifiedStore/UnifiedStoreOps.cs new file mode 100644 index 00000000000..68896dc6eed --- /dev/null +++ b/libs/server/Storage/Session/UnifiedStore/UnifiedStoreOps.cs @@ -0,0 +1,321 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using Garnet.common; +using Tsavorite.core; + +namespace Garnet.server +{ + + sealed partial class StorageSession : IDisposable + { + /// + /// GET a value in the unified store context (value is serialized to the 's ). + /// + /// + /// + /// + /// + /// + /// + public GarnetStatus GET(PinnedSpanByte key, ref UnifiedInput input, ref UnifiedOutput output, ref TUnifiedContext context) + where TUnifiedContext : ITsavoriteContext + { + long ctx = default; + var status = context.Read((FixedSpanByteKey)key, ref input, ref output, ctx); + + if (status.IsPending) + { + StartPendingMetrics(); + CompletePendingForUnifiedStoreSession(ref status, ref output, ref context); + StopPendingMetrics(); + } + + if (status.Found) + { + incr_session_found(); + return GarnetStatus.OK; + } + incr_session_notfound(); + return GarnetStatus.NOTFOUND; + } + + /// + /// SET a log record in the unified store context. + /// + /// + /// + /// The log record + /// Basic unifiedContext for the unified store. + /// + public GarnetStatus SET(in TSourceLogRecord srcLogRecord, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + where TSourceLogRecord : ISourceLogRecord + { + _ = unifiedContext.Upsert(in srcLogRecord); + return GarnetStatus.OK; + } + + /// + /// SET a log record in the unified store context. + /// + /// + /// + /// The key to override the one in , e.g. if from RENAME. + /// + /// The log record + /// Basic unifiedContext for the unified store. + /// + public GarnetStatus SET(ReadOnlySpan key, ref UnifiedInput input, in TSourceLogRecord srcLogRecord, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + where TSourceLogRecord : ISourceLogRecord + { + _ = unifiedContext.Upsert((FixedSpanByteKey)key, ref input, in srcLogRecord); + return GarnetStatus.OK; + } + + /// + /// Checks if a key exists in the unified store context. + /// + /// + /// The name of the key to use in the operation + /// Basic unifiedContext for the unified store. + /// + public GarnetStatus EXISTS(PinnedSpanByte key, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + { + // Prepare input + var input = new UnifiedInput(RespCommand.EXISTS); + + // Prepare UnifiedOutput output + var output = new UnifiedOutput(); + + // TODO: The output is unused so optimize ReadMethods to not copy it. + return Read_UnifiedStore(key, ref input, ref output, ref unifiedContext); + } + + /// + /// Deletes a key from the unified store context. + /// + /// The name of the key to use in the operation + /// Basic unifiedContext for the unified store. + /// + public GarnetStatus DELETE(PinnedSpanByte key, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + { + var status = unifiedContext.Delete((FixedSpanByteKey)key); + + Debug.Assert(!status.IsPending); + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; + } + + /// + /// Deletes a key if it is in memory and expired. + /// + /// The name of the key to use in the operation + /// Basic unifiedContext for the unified store. + /// + public GarnetStatus DELIFEXPIM(PinnedSpanByte key, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + { + var input = new UnifiedInput(RespCommand.DELIFEXPIM); + var status = unifiedContext.RMW((FixedSpanByteKey)key, ref input); + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; + } + + /// + /// Set a timeout on key + /// + /// + /// The key to set the timeout on. + /// Milliseconds value for the timeout. + /// True when the timeout was properly set. + /// >Flags to use for the operation. + /// Basic context for the unified store. + /// + public unsafe GarnetStatus EXPIRE(PinnedSpanByte key, PinnedSpanByte expiryMs, out bool timeoutSet, ExpireOption expireOption, ref TUnifiedContext unifiedContext) + where TUnifiedContext : ITsavoriteContext + => EXPIRE(key, TimeSpan.FromMilliseconds(NumUtils.ReadInt64(expiryMs.Length, expiryMs.ToPointer())), out timeoutSet, expireOption, ref unifiedContext); + + /// + /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970). + /// + /// + /// The key to set the timeout on. + /// Absolute Unix timestamp + /// True when the timeout was properly set. + /// Flags to use for the operation. + /// Basic context for the unified store. + /// When true, is treated as milliseconds else seconds + /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND + public unsafe GarnetStatus EXPIREAT(PinnedSpanByte key, long expiryTimestamp, out bool timeoutSet, ExpireOption expireOption, ref TUnifiedContext unifiedContext, bool milliseconds = false) + where TUnifiedContext : ITsavoriteContext + => EXPIRE(key, expiryTimestamp, out timeoutSet, expireOption, ref unifiedContext, milliseconds ? RespCommand.PEXPIREAT : RespCommand.EXPIREAT); + + /// + /// Set a timeout on key. + /// + /// + /// The key to set the timeout on. + /// The timespan value to set the expiration for. + /// True when the timeout was properly set. + /// Flags to use for the operation. + /// Basic context for the unified store. + /// When true the command executed is PEXPIRE, expire by default. + /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND + public unsafe GarnetStatus EXPIRE(PinnedSpanByte key, TimeSpan expiry, out bool timeoutSet, ExpireOption expireOption, ref TUnifiedContext unifiedContext, bool milliseconds = false) + where TUnifiedContext : ITsavoriteContext + => EXPIRE(key, (long)(milliseconds ? expiry.TotalMilliseconds : expiry.TotalSeconds), out timeoutSet, expireOption, + ref unifiedContext, milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE); + + /// + /// Set a timeout on key. + /// + /// + /// The key to set the timeout on. + /// The timespan value to set the expiration for. + /// True when the timeout was properly set. + /// Flags to use for the operation. + /// Basic context for the main store + /// The current RESP command + /// + public unsafe GarnetStatus EXPIRE(PinnedSpanByte key, long expiration, out bool timeoutSet, ExpireOption expireOption, ref TUnifiedContext unifiedContext, RespCommand respCommand) + where TUnifiedContext : ITsavoriteContext + { + Span rmwOutput = stackalloc byte[sizeof(int)]; + var unifiedOutput = new UnifiedOutput(SpanByteAndMemory.FromPinnedSpan(rmwOutput)); + + // Convert to expiration time in ticks + var expirationTimeInTicks = respCommand switch + { + RespCommand.EXPIRE => DateTimeOffset.UtcNow.AddSeconds(expiration).UtcTicks, + RespCommand.PEXPIRE => DateTimeOffset.UtcNow.AddMilliseconds(expiration).UtcTicks, + RespCommand.EXPIREAT => ConvertUtils.UnixTimestampInSecondsToTicks(expiration), + _ => ConvertUtils.UnixTimestampInMillisecondsToTicks(expiration) + }; + + var expirationWithOption = new ExpirationWithOption(expirationTimeInTicks, expireOption); + + var input = new UnifiedInput(RespCommand.EXPIRE, arg1: expirationWithOption.Word); + var status = unifiedContext.RMW((FixedSpanByteKey)key, ref input, ref unifiedOutput); + + if (status.IsPending) + CompletePendingForUnifiedStoreSession(ref status, ref unifiedOutput, ref unifiedContext); + + timeoutSet = status.Found && + unifiedOutput.SpanByteAndMemory.ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.RESP_RETURN_VAL_1); + + return status.Found ? GarnetStatus.OK : GarnetStatus.NOTFOUND; + } + + /// + /// RENAME a key in the unified store context + /// + /// The key to rename + /// The new key name + /// + public unsafe GarnetStatus RENAME(PinnedSpanByte oldKeySlice, PinnedSpanByte newKeySlice) + => RENAME(oldKeySlice, newKeySlice, false, out _); + + /// + /// RENAME a key in the unified store context - if the new key does not exist + /// + /// The key to rename + /// The new key name + /// Number of renamed records + /// + public unsafe GarnetStatus RENAMENX(PinnedSpanByte oldKeySlice, PinnedSpanByte newKeySlice, out int result) + => RENAME(oldKeySlice, newKeySlice, true, out result); + + /// + /// RENAME a key in the unified store context + /// + /// The key to rename + /// The new key name + /// If true, rename only if the new key does not exist + /// Number of renamed records + /// + private unsafe GarnetStatus RENAME(PinnedSpanByte oldKeySlice, PinnedSpanByte newKeySlice, bool isNX, out int result) + { + result = -1; + + // If same name check return early. + if (oldKeySlice.ReadOnlySpan.SequenceEqual(newKeySlice.ReadOnlySpan)) + { + result = 1; + return GarnetStatus.OK; + } + + // Note: RespServerSession.CanServeSlot has already verified the keys are in the same slot + + var createTransaction = false; + if (txnManager.state != TxnState.Running) + { + createTransaction = true; + txnManager.AddTransactionStoreTypes(TransactionStoreTypes.Main | TransactionStoreTypes.Object); + txnManager.SaveKeyEntryToLock(oldKeySlice, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(newKeySlice, LockType.Exclusive); + _ = txnManager.Run(true); + } + + var context = txnManager.UnifiedTransactionalContext; + var oldKey = oldKeySlice; + var newKey = newKeySlice; + + var returnStatus = GarnetStatus.NOTFOUND; + var abortTransaction = false; + + var output = new UnifiedOutput(); + try + { + // Check if new key exists. + UnifiedInput input = new(RespCommand.RENAME); + var status = GET(newKey, ref input, ref output, ref context); + if (isNX && status != GarnetStatus.NOTFOUND) + { + result = 0; // This is the "oldkey was found" return + abortTransaction = true; + return GarnetStatus.OK; + } + + status = GET(oldKey, ref input, ref output, ref context); + if (status != GarnetStatus.OK) + { + abortTransaction = true; + return status; + } + + fixed (byte* recordPtr = output.SpanByteAndMemory.ReadOnlySpan) + { + // We have a record in in-memory, unserialized format, with its objects (if any) resolved to the TransientObjectIdMap. + var logRecord = new LogRecord(recordPtr, functionsState.transientObjectIdMap); + + status = SET(newKey, ref input, in logRecord, ref context); + if (status == GarnetStatus.OK) + { + result = 1; + + // Delete the old key + _ = DELETE(oldKey, ref context); + return GarnetStatus.OK; + } + } + } + finally + { + if (createTransaction) + { + if (abortTransaction) + txnManager.Reset(); + else + txnManager.Commit(true); + } + output.Dispose(); + } + return returnStatus; + } + } +} \ No newline at end of file diff --git a/libs/server/Storage/SizeTracker/CacheSizeTracker.cs b/libs/server/Storage/SizeTracker/CacheSizeTracker.cs index 16a71a072d9..147a5a9ac07 100644 --- a/libs/server/Storage/SizeTracker/CacheSizeTracker.cs +++ b/libs/server/Storage/SizeTracker/CacheSizeTracker.cs @@ -1,18 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; using System.Diagnostics; -using System.Runtime.CompilerServices; using System.Threading; using Microsoft.Extensions.Logging; using Tsavorite.core; namespace Garnet.server { - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Tracks the size of the main log and read cache. /// Based on the current size and the target size, it uses the corresponding LogSizeTracker objects to increase @@ -20,125 +15,121 @@ namespace Garnet.server /// public class CacheSizeTracker { - internal readonly LogSizeTracker mainLogTracker; - internal readonly LogSizeTracker readCacheTracker; - private long targetSize; - public long ReadCacheTargetSize; + internal LogSizeTracker mainLogTracker; + internal LogSizeTracker readCacheTracker; int isStarted = 0; - private const int deltaFraction = 10; // 10% of target size + private const int HighTargetSizeDeltaFraction = 10; // When memory usage grows, trigger trimming at 10% above target size (for both main log and readcache) + private const int LowTargetSizeDeltaFraction = HighTargetSizeDeltaFraction * 5; // When trimming memory, trim down to 1/5 of HighTargetSizeDeltaFraction below target size (for both main log and readcache) - internal bool Stopped => (mainLogTracker == null || mainLogTracker.Stopped) && (readCacheTracker == null || readCacheTracker.Stopped); + internal bool IsStopped => (mainLogTracker == null || mainLogTracker.IsStopped) && (readCacheTracker == null || readCacheTracker.IsStopped); + internal bool IsStarted => isStarted == 1; - /// - /// Total memory size target - /// + /// Whether the tracker has been initialized with a store. + internal bool IsInitialized => mainLogTracker != null || readCacheTracker != null; + + /// Total memory size target for main log public long TargetSize { - get => targetSize; + get => mainLogTracker?.TargetSize ?? 0; set { Debug.Assert(value >= 0); - targetSize = value; - mainLogTracker?.UpdateTargetSize(targetSize, targetSize / deltaFraction); + mainLogTracker?.UpdateTargetSize(value, value / HighTargetSizeDeltaFraction, value / LowTargetSizeDeltaFraction); } } - /// Helps calculate size of a record including heap memory in Object store. - internal struct LogSizeCalculator : ILogSizeCalculator + /// Total memory size target for readcache + public long ReadCacheTargetSize { - /// Calculate the size of a record in the cache - /// Information about the record - /// The record's key - /// The record's value - /// The size of the record - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long CalculateRecordSize(RecordInfo recordInfo, byte[] key, IGarnetObject value) + get => readCacheTracker?.TargetSize ?? 0; + set { - long size = Utility.RoundUp(key.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead; - - if (!recordInfo.Tombstone && value != null) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter) - size += value.Size; - - return size; + Debug.Assert(value >= 0); + readCacheTracker?.UpdateTargetSize(value, value / HighTargetSizeDeltaFraction, value / LowTargetSizeDeltaFraction); } } + /// + /// Creates an uninitialized CacheSizeTracker. Call after + /// the store is created to subscribe to eviction notifications. + /// + public CacheSizeTracker() { } + /// Class to track and update cache size /// Tsavorite store instance - /// Hybrid log settings /// Total memory size target /// Target memory size for read cache /// - public CacheSizeTracker(TsavoriteKV store, KVSettings logSettings, - long targetSize, long readCacheTargetSize, ILoggerFactory loggerFactory = null) + public CacheSizeTracker(TsavoriteKV store, long targetSize, long readCacheTargetSize, ILoggerFactory loggerFactory = null) + => Initialize(store, targetSize, readCacheTargetSize, loggerFactory); + + /// + /// Initialize the tracker with a store. Wires the for + /// heap-size tracking. Tsavorite handles all creation-site and destruction-site + /// accounting internally via logSizeTracker. + /// + public void Initialize(TsavoriteKV store, long targetSize, long readCacheTargetSize, ILoggerFactory loggerFactory = null) { Debug.Assert(store != null); - Debug.Assert(logSettings != null); Debug.Assert(targetSize > 0 || readCacheTargetSize > 0); - this.TargetSize = targetSize; - this.ReadCacheTargetSize = readCacheTargetSize; - var logSizeCalculator = new LogSizeCalculator(); - if (targetSize > 0) { - this.mainLogTracker = new LogSizeTracker(store.Log, logSizeCalculator, - targetSize, targetSize / deltaFraction, loggerFactory?.CreateLogger("ObjSizeTracker")); - store.Log.SubscribeEvictions(mainLogTracker); - store.Log.SubscribeDeserializations(new LogOperationObserver(mainLogTracker, LogOperationType.Deserialize)); - store.Log.IsSizeBeyondLimit = () => mainLogTracker.IsSizeBeyondLimit; + mainLogTracker = new LogSizeTracker(store.Log, targetSize, + targetSize / HighTargetSizeDeltaFraction, targetSize / LowTargetSizeDeltaFraction, loggerFactory?.CreateLogger("MainLogSizeTracker")); + store.Log.SetLogSizeTracker(mainLogTracker); } if (store.ReadCache != null && readCacheTargetSize > 0) { - this.readCacheTracker = new LogSizeTracker(store.ReadCache, logSizeCalculator, - readCacheTargetSize, readCacheTargetSize / deltaFraction, loggerFactory?.CreateLogger("ObjReadCacheSizeTracker")); - store.ReadCache.SubscribeEvictions(readCacheTracker); - store.ReadCache.SubscribeDeserializations(new LogOperationObserver(readCacheTracker, LogOperationType.Deserialize)); - store.ReadCache.IsSizeBeyondLimit = () => readCacheTracker.IsSizeBeyondLimit; + readCacheTracker = new LogSizeTracker(store.ReadCache, readCacheTargetSize, + readCacheTargetSize / HighTargetSizeDeltaFraction, readCacheTargetSize / LowTargetSizeDeltaFraction, loggerFactory?.CreateLogger("ReadCacheSizeTracker")); + store.ReadCache.SetLogSizeTracker(readCacheTracker); } } + /// Start the trackers, ensuring that only one thread does so. We may start it on Checkpoint recovery before starting database operations. public void Start(CancellationToken token) { // Prevent multiple calls to Start var prevIsStarted = Interlocked.CompareExchange(ref isStarted, 1, 0); - if (prevIsStarted == 1) return; - - mainLogTracker?.Start(token); - readCacheTracker?.Start(token); + if (prevIsStarted == 0) + { + mainLogTracker?.Start(token); + readCacheTracker?.Start(token); + } } /// Add to the tracked size of the cache. /// Size to be added - public void AddTrackedSize(long size) + public void AddHeapSize(long size) { - if (size == 0) return; - // mainLogTracker could be null if heap size limit is set just for the read cache - this.mainLogTracker?.IncrementSize(size); + if (size != 0) + mainLogTracker?.IncrementSize(size); } /// Add to the tracked size of read cache. /// Size to be added - public void AddReadCacheTrackedSize(long size) + public void AddReadCacheHeapSize(long size) { - if (size == 0) return; - - // readCacheTracker could be null if read cache is not enabled or heap size limit is set - // just for the main log - this.readCacheTracker?.IncrementSize(size); + // readCacheTracker could be null if read cache is not enabled or heap size limit is set only for the main log + if (size != 0) + readCacheTracker?.IncrementSize(size); } /// - /// If tracker has not started, prevent it from starting + /// If tracker has not started, prevent it from starting, else stop it. /// - /// True if tracker hasn't previously started - public bool TryPreventStart() + public void Stop() { var prevStarted = Interlocked.CompareExchange(ref isStarted, 1, 0); - return prevStarted == 0; + if (prevStarted != 0) + { + mainLogTracker?.Stop(wait: false); + readCacheTracker?.Stop(wait: false); + } } } } \ No newline at end of file diff --git a/libs/server/StoreWrapper.cs b/libs/server/StoreWrapper.cs index cbbd99bf67a..690525b26c8 100644 --- a/libs/server/StoreWrapper.cs +++ b/libs/server/StoreWrapper.cs @@ -7,6 +7,7 @@ using System.Diagnostics; using System.Net; using System.Net.Sockets; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Garnet.common; @@ -19,12 +20,6 @@ namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Wrapper for store and store-specific information /// @@ -43,17 +38,28 @@ public sealed class StoreWrapper /// /// Store (of DB 0) /// - public TsavoriteKV store => this.databaseManager.MainStore; + public TsavoriteKV store => databaseManager.Store; /// - /// Object store (of DB 0) + /// AOF (of DB 0) /// - public TsavoriteKV objectStore => this.databaseManager.ObjectStore; + public GarnetAppendOnlyFile appendOnlyFile => databaseManager.AppendOnlyFile; /// - /// AOF (of DB 0) + /// Get total AOF size (i.e. diff TailAddres - BeginAddress) + /// + /// + public long AofSize() => appendOnlyFile.Log.TailAddress.AggregateDiff(appendOnlyFile.Log.BeginAddress); + + /// + /// AOF BeginAddress + /// + public AofAddress BeginAddress => appendOnlyFile.Log.BeginAddress; + + /// + /// AOF TailAddress /// - public TsavoriteLog appendOnlyFile => databaseManager.AppendOnlyFile; + public AofAddress TailAddress => appendOnlyFile.Log.TailAddress; /// /// Last save time (of DB 0) @@ -61,9 +67,11 @@ public sealed class StoreWrapper public DateTimeOffset lastSaveTime => databaseManager.LastSaveTime; /// - /// Object store size tracker (of DB 0) + /// Store size tracker (of DB 0) /// - public CacheSizeTracker objectStoreSizeTracker => databaseManager.ObjectStoreSizeTracker; + public CacheSizeTracker sizeTracker => databaseManager.SizeTracker; + + public IStoreFunctions storeFunctions => store.StoreFunctions; /// /// Server options @@ -110,6 +118,11 @@ public sealed class StoreWrapper /// public readonly TimeSpan loggingFrequency; + /// + /// RangeIndex (BfTree) manager shared across sessions + /// + internal readonly RangeIndexManager rangeIndexManager; + /// /// Definition for delegate creating a new logical database /// @@ -146,6 +159,8 @@ public sealed class StoreWrapper internal readonly ILogger sessionLogger; internal long safeAofAddress = -1; + private readonly bool enforceConsistentRead; + // Standalone instance node_id internal readonly string runId; @@ -162,14 +177,14 @@ public sealed class StoreWrapper bool disposed; /// - /// Garnet checkpoint manager for main store + /// Garnet checkpoint manager /// public GarnetCheckpointManager StoreCheckpointManager => (GarnetCheckpointManager)store?.CheckpointManager; /// - /// Garnet checkpoint manager for object store + /// Get task manager instance /// - public GarnetCheckpointManager ObjectStoreCheckpointManager => (GarnetCheckpointManager)objectStore?.CheckpointManager; + public TaskManager TaskManager => taskManager; /// /// Constructor @@ -200,12 +215,14 @@ public StoreWrapper( ? new GarnetServerMonitor(this, serverOptions, servers, loggerFactory?.CreateLogger("GarnetServerMonitor")) : null; + this.enforceConsistentRead = serverOptions.EnableCluster && serverOptions.EnableAOF && serverOptions.MultiLogEnabled; this.logger = loggerFactory?.CreateLogger("StoreWrapper"); this.sessionLogger = loggerFactory?.CreateLogger("Session"); this.accessControlList = accessControlList; this.GarnetObjectSerializer = new GarnetObjectSerializer(this.customCommandManager); this.taskManager = new TaskManager(loggerFactory?.CreateLogger("TaskManager")); this.loggingFrequency = TimeSpan.FromSeconds(serverOptions.LoggingFrequency); + this.rangeIndexManager = DefaultDatabase.RangeIndexManager; logger?.LogTrace("StoreWrapper logging frequency: {loggingFrequency} seconds.", this.loggingFrequency); @@ -273,11 +290,6 @@ public StoreWrapper( { StoreCheckpointManager.CurrentHistoryId = runId; } - - if (!serverOptions.DisableObjects && ObjectStoreCheckpointManager != null) - { - ObjectStoreCheckpointManager.CurrentHistoryId = runId; - } } } @@ -353,9 +365,7 @@ internal void Recover() if (serverOptions.EnableCluster) { if (serverOptions.Recover) - { clusterProvider.Recover(); - } } else { @@ -363,7 +373,7 @@ internal void Recover() { RecoverCheckpoint(); RecoverAOF(); - _ = ReplayAOF(); + ReplayAOF(AofAddress.Create(length: serverOptions.AofPhysicalSublogCount, value: -1)); } } @@ -371,34 +381,19 @@ internal void Recover() } /// - /// Take checkpoint of all active databases + /// Take checkpoint of all active databases (or a specified database) /// /// True if method can return before checkpoint is taken - /// Logger + /// ID of database to checkpoint, or -1 (default) to checkpoint all active databases /// Cancellation token - /// False if another checkpointing process is already in progress - public Task TakeCheckpointAsync(bool background, ILogger logger = null, - CancellationToken token = default) => databaseManager.TakeCheckpointAsync(background, logger, token); - - /// - /// Take checkpoint of all active database IDs or a specified database ID - /// - /// True if method can return before checkpoint is taken - /// ID of database to checkpoint (default: -1 - checkpoint all active databases) /// Logger - /// Cancellation token /// False if another checkpointing process is already in progress - public Task TakeCheckpointAsync(bool background, int dbId = -1, ILogger logger = null, CancellationToken token = default) + public Task TakeCheckpointAsync(bool background, int dbId = -1, CancellationToken token = default, ILogger logger = null) { - if (dbId == -1) - { - return databaseManager.TakeCheckpointAsync(background, logger, token); - } - - if (dbId != 0 && !CheckMultiDatabaseCompatibility()) + if (dbId > 0 && !CheckMultiDatabaseCompatibility()) throw new GarnetException($"Unable to call {nameof(databaseManager.TakeCheckpointAsync)} with DB ID: {dbId}"); - return databaseManager.TakeCheckpointAsync(background, dbId, logger, token); + return databaseManager.TakeCheckpointAsync(background, dbId, token, logger); } /// @@ -418,9 +413,11 @@ public async Task TakeOnDemandCheckpointAsync(DateTimeOffset entryTime, int dbId /// /// Recover checkpoint /// - public void RecoverCheckpoint(bool replicaRecover = false, bool recoverMainStoreFromToken = false, - bool recoverObjectStoreFromToken = false, CheckpointMetadata metadata = null) - => databaseManager.RecoverCheckpoint(replicaRecover, recoverMainStoreFromToken, recoverObjectStoreFromToken, metadata); + public void RecoverCheckpoint(bool replicaRecover = false, bool recoverFromToken = false, CheckpointMetadata metadata = null) + { + StartSizeTrackers(); // We need to start this before recovery to have size tracking during the recovery process. + databaseManager.RecoverCheckpoint(replicaRecover, recoverFromToken, metadata); + } /// /// Mark the beginning of a checkpoint by taking and a lock to avoid concurrent checkpointing @@ -455,7 +452,7 @@ public void ResumeCheckpoints(int dbId = 0) /// /// When replaying AOF we do not want to write AOF records again. /// - public long ReplayAOF(long untilAddress = -1) => this.databaseManager.ReplayAOF(untilAddress); + public AofAddress ReplayAOF(AofAddress untilAddress) => this.databaseManager.ReplayAOF(untilAddress); /// /// Append a checkpoint commit to the AOF @@ -480,7 +477,7 @@ internal async ValueTask WaitForCommitAsync(CancellationToken token = defa { if (!serverOptions.EnableAOF) return false; - await databaseManager.WaitForCommitToAofAsync(token); + await databaseManager.WaitForCommitToAofAsync(token).ConfigureAwait(false); return true; } @@ -497,14 +494,14 @@ internal async ValueTask CommitAOFAsync(int dbId = -1, CancellationToken t if (dbId == -1) { - await databaseManager.CommitToAofAsync(token, logger); + await databaseManager.CommitToAofAsync(token, logger).ConfigureAwait(false); return true; } if (dbId != 0 && !CheckMultiDatabaseCompatibility()) throw new GarnetException($"Unable to call {nameof(databaseManager.CommitToAofAsync)} with DB ID: {dbId}"); - await databaseManager.CommitToAofAsync(dbId, token); + await databaseManager.CommitToAofAsync(dbId, token).ConfigureAwait(false); return true; } @@ -512,6 +509,7 @@ internal async ValueTask CommitAOFAsync(int dbId = -1, CancellationToken t /// Create database functions state /// /// Database ID + /// Resp protocol version /// Functions state /// internal FunctionsState CreateFunctionsState(int dbId = 0, byte respProtocolVersion = ServerOptions.DEFAULT_RESP_VERSION) @@ -649,8 +647,8 @@ async Task CommitTaskAsync(int commitFrequencyMs, CancellationToken token = defa { if (token.IsCancellationRequested) break; - // if we are replica and in auto-commit - do not commit as it will clobber the AOF addresses - if (serverOptions.EnableFastCommit && (clusterProvider?.IsReplica() ?? false)) + // Replicas should never run the periodic commit task because it can clobber replicated AOF addresses. + if (clusterProvider?.IsReplica() ?? false) { await Task.Delay(commitFrequencyMs, token).ConfigureAwait(false); } @@ -677,7 +675,7 @@ async Task CompactionTaskAsync(int compactionFrequencySecs, CancellationToken to { if (token.IsCancellationRequested) return; - databaseManager.DoCompaction(token, logger); + await databaseManager.DoCompactionAsync(token, logger).ConfigureAwait(false); if (!serverOptions.CompactionForceDelete) logger?.LogInformation("NOTE: Take a checkpoint (SAVE/BGSAVE) in order to actually delete the older data segments (files) from disk"); @@ -802,56 +800,35 @@ internal void Start() // Start generic node tasks StartGenericNodeTasks(); - // Start object size trackers - databaseManager.StartObjectSizeTrackers(ctsCommit.Token); + StartSizeTrackers(); // We may have already started this for recovery. } + private void StartSizeTrackers() => databaseManager.StartSizeTrackers(ctsCommit.Token); + + /// + /// Cached callback for . Allocated once per StoreWrapper, reused + /// across calls (cluster control-plane operations are serialised by the cluster manager). + /// + private StorageSession.ArrayKeyIterationFunctions.HasKeysInSlotsScan hasKeysInSlotsFuncs; + public bool HasKeysInSlots(List slots) { - if (slots.Count > 0) - { - bool hasKeyInSlots = false; - { - using var iter = store.Iterate>(new SimpleSessionFunctions()); - while (!hasKeyInSlots && iter.GetNext(out RecordInfo record)) - { - ref var key = ref iter.GetKey(); - - // TODO: better way to ignore vector set elements - if (key.MetadataSize == 1) - { - continue; - } - - ushort hashSlotForKey = HashSlotUtils.HashSlot(ref key); - if (slots.Contains(hashSlotForKey)) - { - hasKeyInSlots = true; - } - } - } + if (slots.Count == 0) return false; - if (!hasKeyInSlots && !serverOptions.DisableObjects) - { - var functionsState = databaseManager.CreateFunctionsState(); - var objstorefunctions = new ObjectSessionFunctions(functionsState); - using var objectStoreSession = objectStore?.NewSession(objstorefunctions); - using var iter = objectStoreSession.Iterate(); - while (!hasKeyInSlots && iter.GetNext(out RecordInfo record)) - { - ref var key = ref iter.GetKey(); - ushort hashSlotForKey = HashSlotUtils.HashSlot(key.AsSpan()); - if (slots.Contains(hashSlotForKey)) - { - hasKeyInSlots = true; - } - } - } + // Single lookup-based push scan over the unified context. Since the migration to a + // single store, the unified context surfaces every record (string + object) so one + // scan suffices. No tempKv is allocated. IterateLookupSnapshot pins both untilAddress + // and maxAddress to a single captured TailAddress so the scan is a consistent + // point-in-time view (records RCU'd above the snapshot don't suppress in-range ones). + var functionsState = databaseManager.CreateFunctionsState(); + var unifiedFunctions = new UnifiedSessionFunctions(functionsState); + using var unifiedSession = store.NewSession(unifiedFunctions); - return hasKeyInSlots; - } + hasKeysInSlotsFuncs ??= new StorageSession.ArrayKeyIterationFunctions.HasKeysInSlotsScan(); + hasKeysInSlotsFuncs.Initialize(slots); - return false; + _ = unifiedSession.IterateLookupSnapshot(ref hasKeysInSlotsFuncs); + return hasKeysInSlotsFuncs.Found; } /// @@ -881,20 +858,30 @@ private bool CheckMultiDatabaseCompatibility() } } + /// + /// Check whether to perform consistent read + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool EnforceConsistentRead() + => enforceConsistentRead && clusterProvider.IsReplica(); + /// /// Dispose /// public void Dispose() { - if (disposed) return; + if (disposed) + return; disposed = true; - itemBroker?.Dispose(); clusterProvider?.Dispose(); + itemBroker?.Dispose(); monitor?.Dispose(); luaTimeoutManager?.Dispose(); ctsCommit?.Cancel(); taskManager.Dispose(); + rangeIndexManager?.Dispose(); databaseManager.Dispose(); ctsCommit?.Dispose(); @@ -904,18 +891,18 @@ public void Dispose() /// Suspend background task that may interfere with the replicas AOF /// /// - public async Task SuspendPrimaryOnlyTasksAsync() + public Task SuspendPrimaryOnlyTasksAsync() { - await taskManager.CancelAsync(TaskPlacementCategory.Primary); + return taskManager.CancelAsync(TaskPlacementCategory.Primary); } /// /// Suspend background task that may interfere with the primary store. /// /// - public async Task SuspendReplicaOnlyTasksAsync() + public Task SuspendReplicaOnlyTasksAsync() { - await taskManager.CancelAsync(TaskPlacementCategory.Replica); + return taskManager.CancelAsync(TaskPlacementCategory.Replica); } /// @@ -951,6 +938,7 @@ public void StartPrimaryTasks() } } + /// /// Start background maintenance tasks that should only be run when this node is a replica. /// diff --git a/libs/server/StringOutput.cs b/libs/server/StringOutput.cs new file mode 100644 index 00000000000..e73932976cc --- /dev/null +++ b/libs/server/StringOutput.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Output type used by Garnet main store. + /// Any field / property added to this struct must be set in the back-end (IFunctions) and used in the front-end (GarnetApi caller). + /// That is in order to justify transferring data in this struct through the Tsavorite storage layer. + /// + public struct StringOutput + { + /// + /// Span byte and memory + /// + public SpanByteAndMemory SpanByteAndMemory; + + /// + /// Output flags + /// + public StringOutputFlags OutputFlags; + + public readonly bool HasError => (OutputFlags & StringOutputFlags.Error) != 0; + + public StringOutput() => SpanByteAndMemory = new(null); + + public StringOutput(SpanByteAndMemory span) => SpanByteAndMemory = span; + + public static unsafe StringOutput FromPinnedPointer(byte* pointer, int length) + => new(SpanByteAndMemory.FromPinnedPointer(pointer, length)); + + public static StringOutput FromPinnedSpan(ReadOnlySpan span) + => new(SpanByteAndMemory.FromPinnedSpan(span)); + + /// + /// Reinterprets the output's underlying as a reference to an unmanaged value of type . + /// The span length must exactly match the size of . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref T AsRef() where T : unmanaged + => ref SpanByteAndMemory.SpanByte.AsRef(); + + public void ConvertToHeap() + { + // Does not convert to heap when going pending, because we complete all pending operations before releasing the pinned source bytes. + } + + public void Dispose() + { + SpanByteAndMemory.Dispose(); + } + } + + /// + /// Output flags for ."/> + /// + [Flags] + public enum StringOutputFlags : byte + { + // Non-error flags + None = 0, + + // Error marker + Error = 1 << 7, + + // Error flags (Error bit always set) + InvalidTypeError = Error | (1 << 0), + NaNOrInfinityError = Error | (1 << 1), + } +} \ No newline at end of file diff --git a/libs/server/TaskManager/TaskType.cs b/libs/server/TaskManager/TaskType.cs index ff9beaf137d..a91d14c05ca 100644 --- a/libs/server/TaskManager/TaskType.cs +++ b/libs/server/TaskManager/TaskType.cs @@ -53,6 +53,11 @@ public enum TaskType : byte /// Replays s on replicas in parallel. /// VectorReplicationReplayTask, + + /// + /// Task used to process advance time signals at replica + /// + AdvanceTimeReplicaTask, } /// @@ -77,6 +82,7 @@ static TaskTypeExtensions() TaskPlacementMapping[(int)TaskType.ObjectCollectTask] = TaskPlacementCategory.Primary; TaskPlacementMapping[(int)TaskType.ExpiredKeyDeletionTask] = TaskPlacementCategory.Primary; TaskPlacementMapping[(int)TaskType.IndexAutoGrowTask] = TaskPlacementCategory.All; + TaskPlacementMapping[(int)TaskType.AdvanceTimeReplicaTask] = TaskPlacementCategory.Replica; } /// diff --git a/libs/server/Transaction/TransactionManager.cs b/libs/server/Transaction/TransactionManager.cs index 607190aa1da..692ea3fd641 100644 --- a/libs/server/Transaction/TransactionManager.cs +++ b/libs/server/Transaction/TransactionManager.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Linq; using System.Runtime.CompilerServices; using Garnet.common; using Microsoft.Extensions.Logging; @@ -10,30 +11,14 @@ namespace Garnet.server { - using BasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - using LockableGarnetApi = GarnetApi, - SpanByteAllocator>>, - LockableContext>, - GenericAllocator>>>, - LockableContext, - SpanByteAllocator>>>; - - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; + [Flags] + public enum TransactionStoreTypes : byte + { + None = 0, + Main = 1, + Object = 1 << 1, + Unified = 1 << 2, + } /// /// Transaction manager @@ -45,38 +30,62 @@ public sealed unsafe partial class TransactionManager /// /// Basic context for main store /// - readonly BasicContext basicContext; + readonly StringBasicContext stringBasicContext; /// - /// Lockable context for main store + /// Transactional context for main store /// - readonly LockableContext lockableContext; + readonly StringTransactionalContext stringTransactionalContext; /// /// Basic context for object store /// - readonly BasicContext objectStoreBasicContext; + readonly ObjectBasicContext objectBasicContext; + + /// + /// Transactional context for object store + /// + readonly ObjectTransactionalContext objectTransactionalContext; + + /// + /// Basic context for unified store + /// + readonly UnifiedBasicContext unifiedBasicContext; /// - /// Lockable context for object store + /// Transactional context for unified store /// - readonly LockableContext objectStoreLockableContext; + readonly UnifiedTransactionalContext unifiedTransactionalContext; // Not readonly to avoid defensive copy GarnetWatchApi garnetTxPrepareApi; // Not readonly to avoid defensive copy - LockableGarnetApi garnetTxMainApi; + TransactionalGarnetApi garnetTxMainApi; // Not readonly to avoid defensive copy BasicGarnetApi garnetTxFinalizeApi; + // Not readonly to avoid defensive copy + GarnetWatchApi garnetConsistentTxPrepareApi; + + // Not readonly to avoid defensive copy + TransactionalConsistentReadGarnetApi garnetConsistentTxRunApi; + + // Not readonly to avoid defensive copy + ConsistentReadGarnetApi garnetConsistentTxFinalizeApi; + + readonly bool enableConsistentRead; + private readonly RespServerSession respSession; readonly FunctionsState functionsState; internal readonly ScratchBufferAllocator scratchBufferAllocator; - private readonly TsavoriteLog appendOnlyFile; + internal readonly ScratchBufferAllocator txnScratchBufferAllocator; + internal SessionParseState txnKeysParseState; + private readonly GarnetAppendOnlyFile appendOnlyFile; internal readonly WatchedKeysContainer watchContainer; private readonly StateMachineDriver stateMachineDriver; + readonly GarnetServerOptions serverOptions; internal int txnStartHead; internal int operationCntTxn; @@ -89,16 +98,18 @@ public sealed unsafe partial class TransactionManager public TxnState state; private const int initialSliceBufferSize = 1 << 10; private const int initialKeyBufferSize = 1 << 10; - StoreType transactionStoreType; readonly ILogger logger; long txnVersion; + private TransactionStoreTypes storeTypes; - internal LockableContext LockableContext - => lockableContext; - internal LockableUnsafeContext LockableUnsafeContext - => basicContext.Session.LockableUnsafeContext; - internal LockableContext ObjectStoreLockableContext - => objectStoreLockableContext; + internal StringTransactionalContext StringTransactionalContext + => stringTransactionalContext; + internal StringTransactionalUnsafeContext TransactionalUnsafeContext + => stringBasicContext.Session.TransactionalUnsafeContext; + internal ObjectTransactionalContext ObjectTransactionalContext + => objectTransactionalContext; + internal UnifiedTransactionalContext UnifiedTransactionalContext + => unifiedTransactionalContext; bool IsReplaying { get; set; } = false; @@ -111,49 +122,71 @@ internal TransactionManager( StoreWrapper storeWrapper, RespServerSession respSession, BasicGarnetApi garnetApi, - LockableGarnetApi lockableGarnetApi, + TransactionalGarnetApi transactionalGarnetApi, StorageSession storageSession, ScratchBufferAllocator scratchBufferAllocator, bool clusterEnabled, + bool enableConsistentRead = false, + ConsistentReadGarnetApi garnetConsistentApi = default, + TransactionalConsistentReadGarnetApi transactionalConsistentGarnetApi = default, ILogger logger = null, int dbId = 0) { - var session = storageSession.basicContext.Session; - basicContext = session.BasicContext; - lockableContext = session.LockableContext; + serverOptions = storeWrapper.serverOptions; + var session = storageSession.stringBasicContext.Session; + stringBasicContext = session.BasicContext; + stringTransactionalContext = session.TransactionalContext; - var objectStoreSession = storageSession.objectStoreBasicContext.Session; - if (objectStoreSession != null) + if (!storeWrapper.serverOptions.DisableObjects) { - objectStoreBasicContext = objectStoreSession.BasicContext; - objectStoreLockableContext = objectStoreSession.LockableContext; + var objectSession = storageSession.objectBasicContext.Session; + objectBasicContext = objectSession.BasicContext; + objectTransactionalContext = objectSession.TransactionalContext; } + var unifiedStoreSession = storageSession.unifiedBasicContext.Session; + unifiedBasicContext = unifiedStoreSession.BasicContext; + unifiedTransactionalContext = unifiedStoreSession.TransactionalContext; + this.functionsState = storageSession.functionsState; this.appendOnlyFile = functionsState.appendOnlyFile; this.logger = logger; this.respSession = respSession; - watchContainer = new WatchedKeysContainer(initialSliceBufferSize, functionsState.watchVersionMap); - keyEntries = new TxnKeyEntries(initialSliceBufferSize, lockableContext, objectStoreLockableContext); + txnScratchBufferAllocator = new ScratchBufferAllocator(); + watchContainer = new WatchedKeysContainer(initialSliceBufferSize, functionsState.watchVersionMap, txnScratchBufferAllocator); + keyEntries = new TxnKeyEntries(initialSliceBufferSize, unifiedTransactionalContext); this.scratchBufferAllocator = scratchBufferAllocator; var dbFound = storeWrapper.TryGetDatabase(dbId, out var db); Debug.Assert(dbFound); this.stateMachineDriver = db.StateMachineDriver; - garnetTxMainApi = lockableGarnetApi; + garnetTxMainApi = transactionalGarnetApi; garnetTxPrepareApi = new GarnetWatchApi(garnetApi); garnetTxFinalizeApi = garnetApi; + this.enableConsistentRead = enableConsistentRead; + if (enableConsistentRead) + { + garnetConsistentTxPrepareApi = new GarnetWatchApi(garnetConsistentApi); + garnetConsistentTxRunApi = transactionalConsistentGarnetApi; + garnetConsistentTxFinalizeApi = garnetConsistentApi; + } + this.clusterEnabled = clusterEnabled; if (clusterEnabled) - keys = new ArgSlice[initialKeyBufferSize]; + { + txnKeysParseState.Initialize(initialKeyBufferSize); + txnKeysParseState.Count = 0; + } Reset(false); } + internal void Reset() => Reset(state == TxnState.Running); + internal void Reset(bool isRunning) { if (isRunning) @@ -162,15 +195,12 @@ internal void Reset(bool isRunning) { keyEntries.UnlockAllKeys(); - // Release context - if (transactionStoreType == StoreType.Main || transactionStoreType == StoreType.All) - lockableContext.EndLockable(); - if (transactionStoreType == StoreType.Object || transactionStoreType == StoreType.All) - { - if (objectStoreBasicContext.IsNull) - throw new Exception("Trying to perform object store transaction with object store disabled"); - objectStoreLockableContext.EndLockable(); - } + // Release contexts + if ((storeTypes & TransactionStoreTypes.Main) == TransactionStoreTypes.Main) + stringTransactionalContext.EndTransaction(); + if ((storeTypes & TransactionStoreTypes.Object) == TransactionStoreTypes.Object && !objectBasicContext.IsNull) + objectTransactionalContext.EndTransaction(); + unifiedTransactionalContext.EndTransaction(); } finally { @@ -181,16 +211,59 @@ internal void Reset(bool isRunning) this.txnStartHead = 0; this.operationCntTxn = 0; this.state = TxnState.None; - this.transactionStoreType = 0; + this.storeTypes = TransactionStoreTypes.None; functionsState.StoredProcMode = false; this.PerformWrites = false; - // Reset cluster variables used for slot verification - this.saveKeyRecvBufferPtr = null; - this.keyCount = 0; + // Reset cluster key parse state + if (clusterEnabled) + { + txnKeysParseState.Count = 0; + saveKeyRecvBufferPtr = null; + txnScratchBufferAllocator.Reset(); + } } internal bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, CustomTransactionProcedure proc, ref MemoryResult output, bool isReplaying = false) + { + if (enableConsistentRead) + { + return RunTransactionProcInternal( + ref garnetConsistentTxPrepareApi, + ref garnetConsistentTxRunApi, + ref garnetConsistentTxFinalizeApi, + id, + ref procInput, + proc, + ref output, + isReplaying); + } + else + { + return RunTransactionProcInternal( + ref garnetTxPrepareApi, + ref garnetTxMainApi, + ref garnetTxFinalizeApi, + id, + ref procInput, + proc, + ref output, + isReplaying); + } + } + + private bool RunTransactionProcInternal( + ref TPrepareApi garnetTxPrepareApi, + ref TRunApi garnetTxRunApi, + ref TFinalizeApi garnetTxFinalizeApi, + byte id, + ref CustomProcedureInput procInput, + CustomTransactionProcedure proc, + ref MemoryResult output, + bool isReplaying = false) + where TPrepareApi : IGarnetReadApi + where TRunApi : IGarnetApi + where TFinalizeApi : IGarnetApi { var running = false; scratchBufferAllocator.Reset(); @@ -200,7 +273,20 @@ internal bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, Cu // If cluster is enabled reset slot verification state cache ResetCacheSlotVerificationResult(); + // Reset logAccess for sharded log + if (serverOptions.MultiLogEnabled) + { + proc.physicalSublogAccessVector = 0UL; + proc.virtualSublogParticipantCount = 0; + if (proc.replayTaskAccessVector != null) + { + foreach (var vector in proc.replayTaskAccessVector) + vector.Clear(); + } + } + functionsState.StoredProcMode = true; + // Prepare phase if (!proc.Prepare(garnetTxPrepareApi, ref procInput)) { @@ -225,10 +311,10 @@ internal bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, Cu running = true; // Run main procedure on locked data - proc.Main(garnetTxMainApi, ref procInput, ref output); + proc.Main(garnetTxRunApi, ref procInput, ref output); // Log the transaction to AOF - Log(id, ref procInput); + Log(id, ref procInput, proc); // Transaction Commit Commit(); @@ -258,10 +344,17 @@ internal bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, Cu scratchBufferAllocator.Reset(); } - return true; } + void Log(byte id, ref CustomProcedureInput procInput, CustomTransactionProcedure proc) + { + Debug.Assert(functionsState.StoredProcMode); + + if (PerformWrites && appendOnlyFile != null) + appendOnlyFile.Log.EnqueueStoredProc(AofEntryType.StoredProcedure, id, txnVersion, stringBasicContext.Session.ID, ref procInput, proc); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool IsSkippingOperations() { @@ -273,96 +366,84 @@ internal void Abort() state = TxnState.Aborted; } - internal void Log(byte id, ref CustomProcedureInput procInput) - { - Debug.Assert(functionsState.StoredProcMode); - - if (PerformWrites) - { - appendOnlyFile?.Enqueue( - new AofHeader { opType = AofEntryType.StoredProcedure, procedureId = id, storeVersion = txnVersion, sessionID = basicContext.Session.ID }, - ref procInput, - out _); - } - } - internal void Commit(bool internal_txn = false) { if (PerformWrites && appendOnlyFile != null && !functionsState.StoredProcMode) { - appendOnlyFile.Enqueue(new AofHeader { opType = AofEntryType.TxnCommit, storeVersion = txnVersion, sessionID = basicContext.Session.ID }, out _); + ComputeSublogAccessVector(out var physicalSublogAccessVector, out var virtualSublogAccessVector, out var virtualSublogParticipantCount); + appendOnlyFile.Log.EnqueueTxn(AofEntryType.TxnCommit, txnVersion, stringBasicContext.Session.ID, physicalSublogAccessVector, virtualSublogAccessVector, virtualSublogParticipantCount); } if (!internal_txn) watchContainer.Reset(); Reset(true); } - internal void Watch(ArgSlice key, StoreType type) + internal void Watch(PinnedSpanByte key) { - // Update watch type if object store is disabled - if (type == StoreType.All && objectStoreBasicContext.IsNull) - type = StoreType.Main; - - UpdateTransactionStoreType(type); - watchContainer.AddWatch(key, type); + watchContainer.AddWatch(key); + + // Release context + if ((storeTypes & TransactionStoreTypes.Main) == TransactionStoreTypes.Main) + stringTransactionalContext.ResetModified((FixedSpanByteKey)key); + if ((storeTypes & TransactionStoreTypes.Object) == TransactionStoreTypes.Object && !objectBasicContext.IsNull) + objectTransactionalContext.ResetModified((FixedSpanByteKey)key); + unifiedTransactionalContext.ResetModified((FixedSpanByteKey)key); + } - if (type == StoreType.Main || type == StoreType.All) - basicContext.ResetModified(key.SpanByte); - if ((type == StoreType.Object || type == StoreType.All) && !objectStoreBasicContext.IsNull) - objectStoreBasicContext.ResetModified(key.ToArray()); + internal void AddTransactionStoreTypes(TransactionStoreTypes transactionStoreTypes) + { + this.storeTypes |= transactionStoreTypes; } - void UpdateTransactionStoreType(StoreType type) + internal void AddTransactionStoreType(StoreType storeType) { - if (transactionStoreType != StoreType.All) + var transactionStoreTypes = storeType switch { - if (transactionStoreType == 0) - transactionStoreType = type; - else - { - if (transactionStoreType != type) - transactionStoreType = StoreType.All; - } - } + StoreType.Main => TransactionStoreTypes.Main, + StoreType.Object => TransactionStoreTypes.Object, + StoreType.All => TransactionStoreTypes.Unified, + _ => TransactionStoreTypes.None + }; + + this.storeTypes |= transactionStoreTypes; } internal string GetLockset() => keyEntries.GetLockset(); - internal void GetKeysForValidation(byte* recvBufferPtr, out ArgSlice[] keys, out int keyCount, out bool readOnly) + internal void GetSlotVerificationInput(byte* recvBufferPtr, byte sessionAsking, out ClusterSlotVerificationInput clusterSlotVerificationInput) { - UpdateRecvBufferPtr(recvBufferPtr); + // Copy keys if buffer changed since last queued command + if (recvBufferPtr != saveKeyRecvBufferPtr) + { + CopyExistingKeysToScratchBuffer(); + saveKeyRecvBufferPtr = recvBufferPtr; + } + watchContainer.SaveKeysToKeyList(this); - keys = this.keys; - keyCount = this.keyCount; - readOnly = keyEntries.IsReadOnly; + clusterSlotVerificationInput = new ClusterSlotVerificationInput + { + readOnly = keyEntries.IsReadOnly, + sessionAsking = sessionAsking, + // We don't specify key specs here as slot verification will know to iterate over all keys in this context + }; } - void BeginLockable(StoreType transactionStoreType) + void BeginTransaction() { - if (transactionStoreType is StoreType.All or StoreType.Main) - { - lockableContext.BeginLockable(); - } - if (transactionStoreType is StoreType.All or StoreType.Object) - { - if (objectStoreBasicContext.IsNull) - throw new Exception("Trying to perform object store transaction with object store disabled"); - objectStoreLockableContext.BeginLockable(); - } + if ((storeTypes & TransactionStoreTypes.Main) == TransactionStoreTypes.Main) + stringTransactionalContext.BeginTransaction(); + if ((storeTypes & TransactionStoreTypes.Object) == TransactionStoreTypes.Object && !objectBasicContext.IsNull) + objectTransactionalContext.BeginTransaction(); + unifiedTransactionalContext.BeginTransaction(); } - void LocksAcquired(StoreType transactionStoreType, long txnVersion) + void LocksAcquired(long txnVersion) { - if (transactionStoreType is StoreType.All or StoreType.Main) - { - lockableContext.LocksAcquired(txnVersion); - } - if (transactionStoreType is StoreType.All or StoreType.Object) - { - if (objectStoreBasicContext.IsNull) - throw new Exception("Trying to perform object store transaction with object store disabled"); - objectStoreLockableContext.LocksAcquired(txnVersion); - } + if ((storeTypes & TransactionStoreTypes.Main) == TransactionStoreTypes.Main) + stringTransactionalContext.LocksAcquired(txnVersion); + if ((storeTypes & TransactionStoreTypes.Object) == TransactionStoreTypes.Object && !objectBasicContext.IsNull) + objectTransactionalContext.LocksAcquired(txnVersion); + unifiedTransactionalContext.LocksAcquired(txnVersion); } internal bool Run(bool internal_txn = false, bool fail_fast_on_lock = false, TimeSpan lock_timeout = default) @@ -375,7 +456,7 @@ internal bool Run(bool internal_txn = false, bool fail_fast_on_lock = false, Tim txnVersion = stateMachineDriver.AcquireTransactionVersion(); // Acquire lock sessions - BeginLockable(transactionStoreType); + BeginTransaction(); bool lockSuccess; if (fail_fast_on_lock) @@ -405,16 +486,85 @@ internal bool Run(bool internal_txn = false, bool fail_fast_on_lock = false, Tim txnVersion = stateMachineDriver.VerifyTransactionVersion(txnVersion); // Update sessions with transaction version - LocksAcquired(transactionStoreType, txnVersion); + LocksAcquired(txnVersion); - // Do not write to AOF if no write operations + // Add TxnStart Marker if (PerformWrites && appendOnlyFile != null && !functionsState.StoredProcMode) { - appendOnlyFile.Enqueue(new AofHeader { opType = AofEntryType.TxnStart, storeVersion = txnVersion, sessionID = basicContext.Session.ID }, out _); + ComputeSublogAccessVector(out var physicalSublogAccessVector, out var virtualSublogAccessVector, out var virtualSublogParticipantCount); + appendOnlyFile.Log.EnqueueTxn(AofEntryType.TxnStart, txnVersion, stringBasicContext.Session.ID, physicalSublogAccessVector, virtualSublogAccessVector, virtualSublogParticipantCount); } state = TxnState.Running; return true; } + + /// + /// Compute metadata required for sharded log custom transaction replay + /// + /// + /// + public void ComputeCustomProcShardedLogAccess(PinnedSpanByte key, CustomTransactionProcedure proc) + { + // Skip if AOF is disabled + if (appendOnlyFile == null) + return; + + // Skip if singleLog + if (!serverOptions.MultiLogEnabled) + return; + + var keyHash = GarnetLog.HASH(key); + if (proc.customProcKeyHashCollection == null) + { + // Used with parallel replay, this BitVector will track which replay tasks should participate in the parallel replay of this custom proc. + proc.replayTaskAccessVector ??= [.. Enumerable.Range(0, appendOnlyFile.Log.Size).Select(_ => new BitVector(AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes))]; + var physicalSublogIdx = appendOnlyFile.Log.GetPhysicalSublogIdx(keyHash); + var replayIdx = appendOnlyFile.Log.GetReplayTaskIdx(keyHash); + + // Mark physical sublog participating in custom txn proc to help with replay coordination. + proc.physicalSublogAccessVector |= 1UL << physicalSublogIdx; + // Mark replay task participation and update count replay tasks participating in replay. + proc.virtualSublogParticipantCount += proc.replayTaskAccessVector[physicalSublogIdx].SetBit(replayIdx) ? 1 : 0; + } + else + // Keep track of key hashes to update sequence numbers of keys at end of replay + proc.customProcKeyHashCollection.AddHash(keyHash); + } + + /// + /// Compute metadata required for sharded log transaction replay + /// + /// + /// + /// + void ComputeSublogAccessVector(out ulong physicalSublogAccessVector, out BitVector[] virtualSublogAccessVector, out int participantCount) + { + physicalSublogAccessVector = 0UL; + virtualSublogAccessVector = null; + participantCount = 0; + // Skip if AOF is disabled + if (appendOnlyFile == null) + return; + + // If singleLog no computation is necessary + if (appendOnlyFile.Log.Size == 1 && appendOnlyFile.Log.ReplayTaskCount == 1) + return; + + // Initialize only for multi-log + virtualSublogAccessVector = [.. Enumerable.Range(0, appendOnlyFile.Log.Size).Select(_ => new BitVector(AofShardedLogTransactionHeader.ReplayTaskAccessVectorBytes))]; + + // If sharded log is enabled calculate sublog access bitmap + for (var i = 0; i < txnKeysParseState.Count; i++) + { + ref var key = ref txnKeysParseState.GetArgSliceByRef(i); + var keyHash = GarnetLog.HASH(key.ReadOnlySpan); + var physicalSublogIdx = appendOnlyFile.Log.GetPhysicalSublogIdx(keyHash); + var replayIdx = appendOnlyFile.Log.GetReplayTaskIdx(keyHash); + physicalSublogAccessVector |= 1UL << physicalSublogIdx; + // Calculate sublog access vector for participating replay tasks + participantCount += virtualSublogAccessVector[physicalSublogIdx].SetBit(replayIdx) ? 1 : 0; + } + } } } \ No newline at end of file diff --git a/libs/server/Transaction/TxnClusterSlotCheck.cs b/libs/server/Transaction/TxnClusterSlotCheck.cs index da587fc774f..21d7f2d00f4 100644 --- a/libs/server/Transaction/TxnClusterSlotCheck.cs +++ b/libs/server/Transaction/TxnClusterSlotCheck.cs @@ -1,50 +1,46 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System; +using System.Diagnostics; +using Tsavorite.core; namespace Garnet.server { sealed unsafe partial class TransactionManager { - // Keys involved in the current transaction - ArgSlice[] keys; - int keyCount; - - internal byte* saveKeyRecvBufferPtr; readonly bool clusterEnabled; + internal byte* saveKeyRecvBufferPtr; /// /// Keep track of actual key accessed by command /// - /// - public void SaveKeyArgSlice(ArgSlice argSlice) + /// + public void SaveKeyArgSlice(PinnedSpanByte keySlice) { // Execute method only if clusterEnabled if (!clusterEnabled) return; - // Grow the buffer if needed - if (keyCount >= keys.Length) - { - var oldKeys = keys; - keys = new ArgSlice[keys.Length * 2]; - Array.Copy(oldKeys, keys, oldKeys.Length); - } - keys[keyCount++] = argSlice; + var count = txnKeysParseState.Count; + + // Grow the buffer if needed (EnsureCapacity handles safe resize with proper GC rooting) + txnKeysParseState.EnsureCapacity(count + 1); + + txnKeysParseState.Count = count + 1; + txnKeysParseState.SetArgument(count, keySlice); } /// - /// Update argslice ptr if input buffer has been resized + /// Copy all existing keys into so they are independent of the old receive buffer. + /// Called when the receive buffer has been reallocated since keys were last stored. /// - /// - public unsafe void UpdateRecvBufferPtr(byte* recvBufferPtr) + public void CopyExistingKeysToScratchBuffer() { - // Execute method only if clusterEnabled - if (!clusterEnabled) return; - if (recvBufferPtr != saveKeyRecvBufferPtr) + Debug.Assert(clusterEnabled); + + for (var i = 0; i < txnKeysParseState.Count; i++) { - for (int i = 0; i < keyCount; i++) - keys[i].ptr = recvBufferPtr + (keys[i].ptr - saveKeyRecvBufferPtr); + ref var key = ref txnKeysParseState.GetArgSliceByRef(i); + key = txnScratchBufferAllocator.CreateArgSlice(key.ReadOnlySpan); } } } diff --git a/libs/server/Transaction/TxnKeyEntry.cs b/libs/server/Transaction/TxnKeyEntry.cs index 3b53f44d024..23836172cf0 100644 --- a/libs/server/Transaction/TxnKeyEntry.cs +++ b/libs/server/Transaction/TxnKeyEntry.cs @@ -4,46 +4,38 @@ using System; using System.Runtime.InteropServices; using System.Text; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - /// /// Entry for a key to lock and unlock in transactions /// - [StructLayout(LayoutKind.Explicit, Size = 10)] - struct TxnKeyEntry : ILockableKey + [StructLayout(LayoutKind.Explicit, Size = 9)] + struct TxnKeyEntry : ITransactionalKey { [FieldOffset(0)] internal long keyHash; [FieldOffset(8)] - internal bool isObject; - - [FieldOffset(9)] internal LockType lockType; - #region ILockableKey + #region ITransactionalKey /// - public long KeyHash { get => keyHash; } + public readonly long KeyHash => keyHash; /// - public LockType LockType { get => lockType; } - #endregion ILockableKey + public readonly LockType LockType => lockType; + #endregion ITransactionalKey /// - public override string ToString() + public override readonly string ToString() { // The debugger often can't call the Globalization NegativeSign property so ToString() would just display the class name var keyHashSign = keyHash < 0 ? "-" : string.Empty; - var absKeyHash = this.keyHash >= 0 ? this.keyHash : -this.keyHash; - return $"{keyHashSign}{absKeyHash}:{(isObject ? "obj" : "raw")}:{(lockType == LockType.None ? "-" : (lockType == LockType.Shared ? "s" : "x"))}"; + var absKeyHash = keyHash >= 0 ? keyHash : -keyHash; + return $"{keyHashSign}{absKeyHash}:{(lockType == LockType.None ? "-" : (lockType == LockType.Shared ? "s" : "x"))}"; } } @@ -51,30 +43,28 @@ internal sealed class TxnKeyEntries { // Basic keys int keyCount; - int mainKeyCount; TxnKeyEntry[] keys; - bool mainStoreKeyLocked; - bool objectStoreKeyLocked; + bool unifiedStoreKeyLocked; readonly TxnKeyComparison comparison; public int phase; - internal TxnKeyEntries(int initialCount, LockableContext lockableContext, - LockableContext objectStoreLockableContext) + internal TxnKeyEntries(int initialCount, + TransactionalContext unifiedTransactionalContext) { keys = GC.AllocateArray(initialCount, pinned: true); // We sort a single array for speed, and the sessions use the same sorting logic, - comparison = new(lockableContext, objectStoreLockableContext); + comparison = new(unifiedTransactionalContext); } public bool IsReadOnly { get { - bool readOnly = true; - for (int i = 0; i < keyCount; i++) + var readOnly = true; + for (var i = 0; i < keyCount; i++) { if (keys[i].lockType == LockType.Exclusive) { @@ -86,11 +76,9 @@ public bool IsReadOnly } } - public void AddKey(ArgSlice keyArgSlice, bool isObject, LockType type) + public void AddKey(PinnedSpanByte keyArgSlice, LockType type) { - var keyHash = !isObject - ? comparison.lockableContext.GetKeyHash(keyArgSlice.SpanByte) - : comparison.objectStoreLockableContext.GetKeyHash(keyArgSlice.ToArray()); + var keyHash = comparison.UnifiedTransactionalContext.GetKeyHash((FixedSpanByteKey)keyArgSlice); // Grow the buffer if needed if (keyCount >= keys.Length) @@ -102,11 +90,8 @@ public void AddKey(ArgSlice keyArgSlice, bool isObject, LockType type) // Populate the new key slot. keys[keyCount].keyHash = keyHash; - keys[keyCount].isObject = isObject; keys[keyCount].lockType = type; ++keyCount; - if (!isObject) - ++mainKeyCount; } internal void LockAllKeys() @@ -119,18 +104,11 @@ internal void LockAllKeys() // This does not call Tsavorite's SortKeyHashes because we need to consider isObject as well. MemoryExtensions.Sort(keys.AsSpan().Slice(0, keyCount), comparison.comparisonDelegate); - // Issue main store locks - if (mainKeyCount > 0) + // Issue unified store locks + if (keyCount > 0) { - comparison.lockableContext.Lock(keys.AsSpan()[..mainKeyCount]); - mainStoreKeyLocked = true; - } - - // Issue object store locks - if (mainKeyCount < keyCount) - { - comparison.objectStoreLockableContext.Lock(keys.AsSpan().Slice(mainKeyCount, keyCount - mainKeyCount)); - objectStoreKeyLocked = true; + comparison.UnifiedTransactionalContext.Lock(keys.AsSpan().Slice(0, keyCount)); + unifiedStoreKeyLocked = true; } phase = 0; @@ -146,24 +124,12 @@ internal bool TryLockAllKeys(TimeSpan lock_timeout) // This does not call Tsavorite's SortKeyHashes because we need to consider isObject as well. MemoryExtensions.Sort(keys.AsSpan().Slice(0, keyCount), comparison.comparisonDelegate); - // Issue main store locks - // TryLock will unlock automatically in case of partial failure - if (mainKeyCount > 0) - { - mainStoreKeyLocked = comparison.lockableContext.TryLock(keys.AsSpan()[..mainKeyCount], lock_timeout); - if (!mainStoreKeyLocked) - { - phase = 0; - return false; - } - } - - // Issue object store locks + // Issue unified store locks // TryLock will unlock automatically in case of partial failure - if (mainKeyCount < keyCount) + if (keyCount > 0) { - objectStoreKeyLocked = comparison.objectStoreLockableContext.TryLock(keys.AsSpan().Slice(mainKeyCount, keyCount - mainKeyCount), lock_timeout); - if (!objectStoreKeyLocked) + unifiedStoreKeyLocked = comparison.UnifiedTransactionalContext.TryLock(keys.AsSpan().Slice(0, keyCount), lock_timeout); + if (!unifiedStoreKeyLocked) { phase = 0; return false; @@ -177,14 +143,10 @@ internal bool TryLockAllKeys(TimeSpan lock_timeout) internal void UnlockAllKeys() { phase = 2; - if (mainStoreKeyLocked && mainKeyCount > 0) - comparison.lockableContext.Unlock(keys.AsSpan()[..mainKeyCount]); - if (objectStoreKeyLocked && mainKeyCount < keyCount) - comparison.objectStoreLockableContext.Unlock(keys.AsSpan().Slice(mainKeyCount, keyCount - mainKeyCount)); - mainKeyCount = 0; + if (unifiedStoreKeyLocked && keyCount > 0) + comparison.UnifiedTransactionalContext.Unlock(keys.AsSpan().Slice(0, keyCount)); keyCount = 0; - mainStoreKeyLocked = false; - objectStoreKeyLocked = false; + unifiedStoreKeyLocked = false; phase = 0; } @@ -196,12 +158,12 @@ internal string GetLockset() for (int ii = 0; ii < keyCount; ii++) { ref var entry = ref keys[ii]; - sb.Append(delimiter); - sb.Append(entry.ToString()); + _ = sb.Append(delimiter); + _ = sb.Append(entry.ToString()); } if (sb.Length > 0) - sb.Append($" (phase: {(phase == 0 ? "none" : (phase == 1 ? "lock" : "unlock"))}))"); + _ = sb.Append($" (phase: {(phase == 0 ? "none" : (phase == 1 ? "lock" : "unlock"))}))"); return sb.ToString(); } } diff --git a/libs/server/Transaction/TxnKeyEntryComparison.cs b/libs/server/Transaction/TxnKeyEntryComparison.cs index 8ab74b1f212..95ebd4392a9 100644 --- a/libs/server/Transaction/TxnKeyEntryComparison.cs +++ b/libs/server/Transaction/TxnKeyEntryComparison.cs @@ -2,42 +2,25 @@ // Licensed under the MIT license. using System; +using Garnet.common; using Tsavorite.core; namespace Garnet.server { - using MainStoreAllocator = SpanByteAllocator>; - using MainStoreFunctions = StoreFunctions; - - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - internal sealed class TxnKeyComparison { - public LockableContext lockableContext; - public LockableContext objectStoreLockableContext; + public TransactionalContext UnifiedTransactionalContext; public readonly Comparison comparisonDelegate; - internal TxnKeyComparison(LockableContext lockableContext, - LockableContext objectStoreLockableContext) + internal TxnKeyComparison( + TransactionalContext unifiedTransactionalContext) { - this.lockableContext = lockableContext; - this.objectStoreLockableContext = objectStoreLockableContext; + this.UnifiedTransactionalContext = unifiedTransactionalContext; comparisonDelegate = Compare; } - /// public int Compare(TxnKeyEntry key1, TxnKeyEntry key2) - { - // This sorts by isObject, then calls Tsavorite to sort by lock code and then by lockType. - var cmp = key1.isObject.CompareTo(key2.isObject); - if (cmp != 0) - return cmp; - if (key1.isObject) - return objectStoreLockableContext.CompareKeyHashes(ref key1, ref key2); - else - return lockableContext.CompareKeyHashes(ref key1, ref key2); - } + => UnifiedTransactionalContext.CompareKeyHashes(ref key1, ref key2); } } \ No newline at end of file diff --git a/libs/server/Transaction/TxnKeyManager.cs b/libs/server/Transaction/TxnKeyManager.cs index d06dba34baf..6e8914733b5 100644 --- a/libs/server/Transaction/TxnKeyManager.cs +++ b/libs/server/Transaction/TxnKeyManager.cs @@ -12,14 +12,12 @@ sealed partial class TransactionManager /// Save key entry /// /// - /// /// - public void SaveKeyEntryToLock(ArgSlice key, bool isObject, LockType type) + public void SaveKeyEntryToLock(PinnedSpanByte key, LockType type) { // Indicate whether transaction has to perform a write operation (used to skip writing to AOF otherwise) PerformWrites |= type == LockType.Exclusive; - UpdateTransactionStoreType(isObject ? StoreType.Object : StoreType.Main); - keyEntries.AddKey(key, isObject, type); + keyEntries.AddKey(key, type); } /// @@ -45,7 +43,7 @@ public void WriteCachedSlotVerificationMessage(ref MemoryResult output) /// /// /// - public void VerifyKeyOwnership(ArgSlice key, LockType type) + public void VerifyKeyOwnership(PinnedSpanByte key, LockType type) { if (!clusterEnabled || IsReplaying) return; @@ -65,6 +63,8 @@ internal void LockKeys(SimpleRespCommandInfo cmdInfo) if (cmdInfo.KeySpecs == null || cmdInfo.KeySpecs.Length == 0) return; + AddTransactionStoreType(cmdInfo.StoreType); + foreach (var keySpec in cmdInfo.KeySpecs) { if (!respSession.parseState.TryGetKeySearchArgsFromSimpleKeySpec(keySpec, cmdInfo.IsSubCommand, out var searchArgs)) @@ -76,10 +76,7 @@ internal void LockKeys(SimpleRespCommandInfo cmdInfo) for (var currIdx = searchArgs.firstIdx; currIdx <= searchArgs.lastIdx; currIdx += searchArgs.step) { var key = respSession.parseState.GetArgSliceByRef(currIdx); - if (cmdInfo.StoreType is StoreType.Main or StoreType.All) - SaveKeyEntryToLock(key, false, lockType); - if (cmdInfo.StoreType is StoreType.Object or StoreType.All && !objectStoreBasicContext.IsNull) - SaveKeyEntryToLock(key, true, lockType); + SaveKeyEntryToLock(key, lockType); SaveKeyArgSlice(key); } } diff --git a/libs/server/Transaction/TxnRespCommands.cs b/libs/server/Transaction/TxnRespCommands.cs index 6b31be3c015..1483b4ec303 100644 --- a/libs/server/Transaction/TxnRespCommands.cs +++ b/libs/server/Transaction/TxnRespCommands.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using Garnet.common; using Microsoft.Extensions.Logging; +using Tsavorite.core; namespace Garnet.server { @@ -28,7 +29,7 @@ private bool NetworkMULTI() txnManager.txnStartHead = readHead; txnManager.state = TxnState.Started; txnManager.operationCntTxn = 0; - //Keep track of ptr for key verification when cluster mode is enabled + // Track receive buffer ptr for key pointer adjustment at EXEC time txnManager.saveKeyRecvBufferPtr = recvBufferPtr; while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) @@ -38,35 +39,42 @@ private bool NetworkMULTI() private bool NetworkEXEC() { - // pass over the EXEC in buffer during execution + // Pass over the EXEC in buffer during execution if (txnManager.state == TxnState.Running) { txnManager.Commit(); return true; - } + // Abort and reset the transaction - else if (txnManager.state == TxnState.Aborted) + if (txnManager.state == TxnState.Aborted) { while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_EXEC_ABORT, ref dcurr, dend)) SendAndReset(); txnManager.Reset(false); + txnManager.watchContainer.Reset(); return true; } - // start running transaction and setting readHead to first operation - else if (txnManager.state == TxnState.Started) + + // Start running transaction and setting readHead to first operation + if (txnManager.state == TxnState.Started) { - var _origReadHead = endReadHead; + var origReadHead = endReadHead; endReadHead = txnManager.txnStartHead; - txnManager.GetKeysForValidation(recvBufferPtr, out var keys, out int keyCount, out bool readOnly); - if (NetworkKeyArraySlotVerify(keys, readOnly, waitForStableSlot: false, keyCount)) // TODO: We should actually verify if commands contained are Vector Set writes + if (clusterSession != null) { - logger?.LogWarning("Failed CheckClusterTxnKeys"); - txnManager.Reset(false); - txnManager.watchContainer.Reset(); - endReadHead = _origReadHead; - return true; + txnManager.GetSlotVerificationInput(recvBufferPtr, SessionAsking, out var clusterSlotVerificationInput); + + if (txnManager.txnKeysParseState.Count > 0 && + clusterSession.NetworkMultiKeySlotVerify(ref txnManager.txnKeysParseState, ref clusterSlotVerificationInput, ref dcurr, ref dend, isTxn: true)) + { + logger?.LogWarning("Failed CheckClusterTxnKeys"); + txnManager.Reset(false); + txnManager.watchContainer.Reset(); + endReadHead = origReadHead; + return true; + } } var startTxn = txnManager.Run(); @@ -78,17 +86,17 @@ private bool NetworkEXEC() } else { - endReadHead = _origReadHead; + endReadHead = origReadHead; WriteNullArray(); } return true; } + // EXEC without MULTI command while (!RespWriteUtils.TryWriteError(CmdStrings.RESP_ERR_GENERIC_EXEC_WO_MULTI, ref dcurr, dend)) SendAndReset(); return true; - } /// @@ -176,6 +184,12 @@ private bool NetworkSKIP(RespCommand cmd) return true; } + if (clusterSession != null && recvBufferPtr != txnManager.saveKeyRecvBufferPtr) + { + txnManager.CopyExistingKeysToScratchBuffer(); + txnManager.saveKeyRecvBufferPtr = recvBufferPtr; + } + txnManager.LockKeys(commandInfo); while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_QUEUED, ref dcurr, dend)) @@ -197,6 +211,7 @@ private bool NetworkDISCARD() while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) SendAndReset(); txnManager.Reset(false); + txnManager.watchContainer.Reset(); return true; } @@ -214,7 +229,7 @@ private bool CommonWATCH(StoreType type) return AbortWithErrorMessage(CmdStrings.GenericErrWrongNumArgs); } - List keys = []; + List keys = []; for (var c = 0; c < count; c++) { @@ -222,9 +237,11 @@ private bool CommonWATCH(StoreType type) keys.Add(nextKey); } + txnManager.AddTransactionStoreType(type); + foreach (var toWatch in keys) { - txnManager.Watch(toWatch, type); + txnManager.Watch(toWatch); } while (!RespWriteUtils.TryWriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) diff --git a/libs/server/Transaction/TxnWatchedKeysContainer.cs b/libs/server/Transaction/TxnWatchedKeysContainer.cs index 6907ed32e6d..e32b4871efd 100644 --- a/libs/server/Transaction/TxnWatchedKeysContainer.cs +++ b/libs/server/Transaction/TxnWatchedKeysContainer.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. using System; -using System.Runtime.CompilerServices; using Tsavorite.core; namespace Garnet.server @@ -13,30 +12,26 @@ namespace Garnet.server internal sealed unsafe class WatchedKeysContainer { /// - /// Array to keep slice of keys inside keyBuffer + /// Array to keep watched keys data /// WatchedKeySlice[] keySlices; /// - /// Array to keep slice of keys inside keyBuffer + /// Version map for watch validation /// readonly WatchVersionMap versionMap; - readonly int initialWatchBufferSize = 1 << 16; readonly int initialSliceBufferSize; + readonly ScratchBufferAllocator txnScratchBufferAllocator; int sliceBufferSize; - int watchBufferSize; - byte[] watchBuffer; - byte* watchBufferPtr; - int watchBufferHeadAddress; int sliceCount; - public WatchedKeysContainer(int size, WatchVersionMap versionMap) + public WatchedKeysContainer(int size, WatchVersionMap versionMap, ScratchBufferAllocator txnScratchBufferAllocator) { this.versionMap = versionMap; - watchBufferHeadAddress = 0; sliceCount = 0; initialSliceBufferSize = size; + this.txnScratchBufferAllocator = txnScratchBufferAllocator; } /// @@ -45,63 +40,41 @@ public WatchedKeysContainer(int size, WatchVersionMap versionMap) public void Reset() { sliceCount = 0; - watchBufferPtr -= watchBufferHeadAddress; - watchBufferHeadAddress = 0; + txnScratchBufferAllocator.Reset(); } - public bool RemoveWatch(ArgSlice key) + public bool RemoveWatch(PinnedSpanByte key) { - for (int i = 0; i < sliceCount; i++) + for (var i = 0; i < sliceCount; i++) { if (key.ReadOnlySpan.SequenceEqual(keySlices[i].slice.ReadOnlySpan)) { - keySlices[i].type = 0; + keySlices[i].isWatched = false; return true; } } return false; } - public void AddWatch(ArgSlice key, StoreType type) + public void AddWatch(PinnedSpanByte key) { if (sliceCount >= sliceBufferSize) { // Double the struct buffer sliceBufferSize = sliceBufferSize == 0 ? initialSliceBufferSize : sliceBufferSize * 2; - var _oldBuffer = keySlices; + var oldBuffer = keySlices; keySlices = GC.AllocateUninitializedArray(sliceBufferSize, true); - if (_oldBuffer != null) Array.Copy(_oldBuffer, keySlices, _oldBuffer.Length); - } - if (watchBufferHeadAddress + key.Length > watchBufferSize) - { - // Double the watch buffer - watchBufferSize = watchBufferSize == 0 ? initialWatchBufferSize : watchBufferSize * 2; - var _oldBuffer = watchBuffer; - watchBuffer = GC.AllocateUninitializedArray(watchBufferSize, true); - var watchBufferPtrBase = (byte*)Unsafe.AsPointer(ref watchBuffer[0]); - watchBufferPtr = watchBufferPtrBase + watchBufferHeadAddress; - - if (_oldBuffer != null) - { - Array.Copy(_oldBuffer, watchBuffer, _oldBuffer.Length); - var oldWatchBufferPtrBase = (byte*)Unsafe.AsPointer(ref _oldBuffer[0]); - - // Update pointer for existing watches - for (int i = 0; i < sliceCount; i++) - keySlices[i].slice.ptr = watchBufferPtrBase + (keySlices[i].slice.ptr - oldWatchBufferPtrBase); - } + if (oldBuffer != null) Array.Copy(oldBuffer, keySlices, oldBuffer.Length); } - var slice = new ArgSlice(watchBufferPtr, key.Length); - key.ReadOnlySpan.CopyTo(slice.Span); + // Copy key bytes into scratch buffer (independent of receive buffer lifetime) + var keySlice = txnScratchBufferAllocator.CreateArgSlice(key.ReadOnlySpan); - keySlices[sliceCount].slice = slice; - keySlices[sliceCount].type = type; - keySlices[sliceCount].hash = Utility.HashBytes(slice.ptr, slice.Length); + keySlices[sliceCount].slice = keySlice; + keySlices[sliceCount].isWatched = true; + keySlices[sliceCount].hash = Utility.HashBytes(keySlice.ReadOnlySpan); keySlices[sliceCount].version = versionMap.ReadVersion(keySlices[sliceCount].hash); - watchBufferPtr += key.Length; - watchBufferHeadAddress += key.Length; sliceCount++; } @@ -111,10 +84,10 @@ public void AddWatch(ArgSlice key, StoreType type) /// public bool ValidateWatchVersion() { - for (int i = 0; i < sliceCount; i++) + for (var i = 0; i < sliceCount; i++) { - WatchedKeySlice key = keySlices[i]; - if (key.type == 0) continue; + var key = keySlices[i]; + if (!key.isWatched) continue; if (versionMap.ReadVersion(key.hash) != key.version) return false; } @@ -123,23 +96,20 @@ public bool ValidateWatchVersion() public bool SaveKeysToLock(TransactionManager txnManager) { - for (int i = 0; i < sliceCount; i++) + for (var i = 0; i < sliceCount; i++) { - WatchedKeySlice watchedKeySlice = keySlices[i]; - if (watchedKeySlice.type == 0) continue; + var watchedKeySlice = keySlices[i]; + if (!watchedKeySlice.isWatched) continue; var slice = keySlices[i].slice; - if (watchedKeySlice.type == StoreType.Main || watchedKeySlice.type == StoreType.All) - txnManager.SaveKeyEntryToLock(slice, false, LockType.Shared); - if (watchedKeySlice.type == StoreType.Object || watchedKeySlice.type == StoreType.All) - txnManager.SaveKeyEntryToLock(slice, true, LockType.Shared); + txnManager.SaveKeyEntryToLock(slice, LockType.Shared); } return true; } public bool SaveKeysToKeyList(TransactionManager txnManager) { - for (int i = 0; i < sliceCount; i++) + for (var i = 0; i < sliceCount; i++) { txnManager.SaveKeyArgSlice(keySlices[i].slice); } diff --git a/libs/server/Transaction/WatchedKeySlice.cs b/libs/server/Transaction/WatchedKeySlice.cs index 6af999e6669..640a191c4a1 100644 --- a/libs/server/Transaction/WatchedKeySlice.cs +++ b/libs/server/Transaction/WatchedKeySlice.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using System.Runtime.InteropServices; +using Tsavorite.core; namespace Garnet.server { @@ -12,12 +13,12 @@ struct WatchedKeySlice public long version; [FieldOffset(8)] - public ArgSlice slice; + public PinnedSpanByte slice; [FieldOffset(20)] public long hash; [FieldOffset(28)] - public StoreType type; + public bool isWatched; } } \ No newline at end of file diff --git a/libs/server/UnifiedOutput.cs b/libs/server/UnifiedOutput.cs new file mode 100644 index 00000000000..09c9837ad3e --- /dev/null +++ b/libs/server/UnifiedOutput.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Output type used by Garnet unified store. + /// Any field / property added to this struct must be set in the back-end (IFunctions) and used in the front-end (GarnetApi caller). + /// That is in order to justify transferring data in this struct through the Tsavorite storage layer. + /// + public struct UnifiedOutput + { + /// + /// Span byte and memory + /// + public SpanByteAndMemory SpanByteAndMemory; + + public UnifiedOutput() => SpanByteAndMemory = new(null); + + public UnifiedOutput(SpanByteAndMemory sbam) => SpanByteAndMemory = sbam; + + public static unsafe UnifiedOutput FromPinnedPointer(byte* pointer, int length) + => new(new SpanByteAndMemory() { SpanByte = PinnedSpanByte.FromPinnedPointer(pointer, length) }); + + public void ConvertToHeap() + { + // Does not convert to heap when going pending, because we complete all pending operations before releasing the pinned source bytes. + } + + public void Dispose() + { + SpanByteAndMemory.Dispose(); + } + } +} \ No newline at end of file diff --git a/libs/server/VectorOutput.cs b/libs/server/VectorOutput.cs new file mode 100644 index 00000000000..fd9df75d6fe --- /dev/null +++ b/libs/server/VectorOutput.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Garnet.server +{ + /// + /// Output type used by Garnet vector store. + /// + /// Basically , but Vector Set operations know sizes in advance more often. + /// + public struct VectorOutput + { + /// + /// Span byte and memory + /// + public SpanByteAndMemory SpanByteAndMemory; + + public VectorOutput() => SpanByteAndMemory = new(null); + + public VectorOutput(SpanByteAndMemory span) => SpanByteAndMemory = span; + + public VectorOutput(Span span) => SpanByteAndMemory = new(PinnedSpanByte.FromPinnedSpan(span)); + + public unsafe VectorOutput(byte* ptr, int len) => SpanByteAndMemory = new(PinnedSpanByte.FromPinnedPointer(ptr, len)); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/Tsavorite.slnx b/libs/storage/Tsavorite/cs/Tsavorite.slnx index 98ad76672ce..420e99b86e2 100644 --- a/libs/storage/Tsavorite/cs/Tsavorite.slnx +++ b/libs/storage/Tsavorite/cs/Tsavorite.slnx @@ -1,6 +1,7 @@ + @@ -11,5 +12,11 @@ + + + + + + diff --git a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/BenchmarkDotNetTestsApp.cs b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/BenchmarkDotNetTestsApp.cs index 226eb454421..50a8a215e0c 100644 --- a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/BenchmarkDotNetTestsApp.cs +++ b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/BenchmarkDotNetTestsApp.cs @@ -1,38 +1,204 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -#if DEBUG +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; -#endif +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Diagnostics.Windows; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using Perfolizer.Metrology; namespace BenchmarkDotNetTests { + // Driver class for BDN testing public class BenchmarkDotNetTestsApp { public static string TestDirectory => Path.Combine(Path.GetDirectoryName(typeof(BenchmarkDotNetTestsApp).Assembly.Location), "Tests"); + // Number of records to use; can be configurd by cmdline args. It's here because it's common among more than one test. + public static int NumRecords = 1_000_000; + + const string ExeName = "BDN-Tsavorite.benchmark.exe"; + const string InliningDiag = "inliningDiag"; + const string MemoryDiag = "memoryDiag"; + const string HwCounters = "hwCounters"; + + static void Usage() + { + Console.WriteLine("Usage:"); + Console.WriteLine($"To run BDN, either run 'dotnet run -c Release -f net10.0 --' (that trailing -- is needed) in the project dir, or {ExeName} directly:"); + Console.WriteLine($" {ExeName} [["); + Console.WriteLine(" If BDN or Test Options are present the '--' separator is required (and may be the second one if running with dotnet); otherwise it may be omitted"); + Console.WriteLine("To debug a test directly (without waiting for BDN to generate the exes):"); + Console.WriteLine($" {ExeName} [Test Options (see below)] --debug "); + Console.WriteLine("For this usage message:"); + Console.WriteLine($" {ExeName} -? or /? or --help"); + Console.WriteLine(""); + Console.WriteLine("BDN options (these set options on the BDN config; preceded by --):"); + Console.WriteLine($" --{InliningDiag}: Adds the inlining diagnoser to the run"); + Console.WriteLine($" --{MemoryDiag}: Adds the memory diagnoser to the run"); + Console.WriteLine($" --{HwCounters}: Adds the hardware counters to the run"); + Console.WriteLine("Test Options (these set parameters on the individual tests; preceded by --):"); + Console.WriteLine($" --{nameof(NumRecords)} : Sets the number of records to operate on for {nameof(OperationTests)} or {nameof(IterationTests)} (default: {NumRecords})"); + Console.WriteLine($" --{nameof(IterationTests.NumRecords)} : Sets whether {nameof(IterationTests)}.{nameof(IterationTests.FlushAndEvict)} is set (default: false"); + Console.WriteLine($"Test Name: run {ExeName} --list tree and then pass the unqualified name of the test you want, e.g. {nameof(EpochTests.ResumeSuspend)}:"); + + // Syntax to run BDN: + // <--> + // Syntax to debug a test without waiting for BDN to generate the .exes: + // debug + // For usage: + // -? or /? or --help + } + public static void Main(string[] args) { + var arg = args?[0].ToLower() ?? null; + if (arg is null || arg == "-?" || arg == "/?" || arg == "--help") + { + Usage(); + return; + } + + bool inliningDiag = false, memoryDiag = false, hwCounters = false; + string[] bdnArgs = []; + // Check for debugging a test - if (args[0].ToLower() == "cursor") + for (var ii = 0; ii < args.Length; ii++) { - var test = new IterationTests + arg = args[ii].ToLower(); + + // If we are at the "run BDN" separator, then config options are set; break out and run it. + if (arg == "--") { - FlushAndEvict = true - }; - test.SetupPopulatedStore(); - test.Cursor(); - test.TearDown(); - return; + if (ii + 1 < args.Length) + bdnArgs = [.. args.Skip(ii + 1)]; + break; + } + + // If we are at the "debug the code" separator, then config options are set; debug it and we're done. + if (arg == "--debug") + { + ii++; + if (ii >= args.Length) + throw new ArgumentException($"'{arg}' option must be followed by the name of the test to debug"); + var testName = args[ii].ToLower(); + if (testName == nameof(IterationTests.Cursor).ToLower()) + { + var test = new IterationTests(); + test.SetupPopulatedStore(); + test.Cursor(); + test.TearDown(); + return; + } + if (testName == nameof(OperationTests.Read).ToLower() || testName == nameof(OperationTests.Upsert).ToLower() || testName == nameof(OperationTests.RMW).ToLower()) + { + var tester = new OperationTests(); + tester.SetupPopulatedStore(); + if (testName == nameof(OperationTests.Read).ToLower()) + tester.Read(); + else if (testName == nameof(OperationTests.Upsert).ToLower()) + tester.Upsert(); + else if (testName == nameof(OperationTests.RMW).ToLower()) + tester.RMW(); + else if (testName == "all") + { + tester.Read(); + tester.Upsert(); + tester.RMW(); + } + else + throw new ArgumentException($"Unknown {nameof(OperationTests)} test: {args[1]}"); + return; + } + throw new ArgumentException($"unknown test name '{testName}"); + } + + // BDN options parsing + if (arg == $"--{InliningDiag.ToLower()}") + { + inliningDiag = true; + continue; + } + if (arg == $"--{MemoryDiag.ToLower()}") + { + memoryDiag = true; + continue; + } + if (arg == $"--{HwCounters.ToLower()}") + { + hwCounters = true; + continue; + } + + // Test options parsing + if (arg == $"--{nameof(NumRecords).ToLower()}") + { + ii++; + if (ii >= args.Length) + throw new ArgumentException($"'{arg}' option must be followed by the value"); + NumRecords = int.Parse(args[ii]); + continue; + } + if (arg == $"--{nameof(IterationTests.FlushAndEvict).ToLower()}") + { + ii++; + if (ii >= args.Length) + throw new ArgumentException($"'{arg}' option must be followed by the value"); + IterationTests.FlushAndEvictConfig = bool.Parse(args[ii]); + continue; + } + + // Assume we should pass this through to BDN itself; otherwise if we don't have test or BDN-config options, + // we would need to have two -- -- separators which would be ugly. + if (ii < args.Length) + bdnArgs = [.. args.Skip(ii)]; + break; } - BenchmarkSwitcher.FromAssembly(typeof(BenchmarkDotNetTestsApp).Assembly) #if DEBUG - .Run(args, new DebugInProcessConfig()); + if (inliningDiag || memoryDiag || hwCounters) + Console.WriteLine("Warning: Diagnostics options are ignored in debug runs"); + var config = new DebugInProcessConfig(); #else - .Run(args); + var config = new ReleaseConfig(inliningDiag, memoryDiag, hwCounters); #endif + BenchmarkSwitcher.FromAssembly(typeof(BenchmarkDotNetTestsApp).Assembly).Run(bdnArgs, config); + } + } + + public class ReleaseConfig : ManualConfig + { + public ReleaseConfig(bool inliningDiag, bool memoryDiag, bool hardwareCounters) + { + _ = AddLogger(ConsoleLogger.Default); + _ = AddExporter(DefaultExporters.Markdown); + _ = AddColumnProvider(DefaultColumnProviders.Instance); + _ = WithSummaryStyle(SummaryStyle.Default.WithSizeUnit(SizeUnit.B)); + + var baseJob = Job.Default; + + var net10Job = baseJob + .WithRuntime(CoreRuntime.Core10_0) + .WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredPGO", "0")); + + if (inliningDiag) + { + // If there is no inliningDiagnoser output, it may be necessary to turn off TieredCompilation; see https://github.com/dotnet/BenchmarkDotNet/issues/1791 + //net10Job = net10Job.WithEnvironmentVariables(new EnvironmentVariable("DOTNET_TieredCompilation", "0")); + _ = AddDiagnoser(new InliningDiagnoser(logFailuresOnly: true, allowedNamespaces: ["Tsavorite.core"])); + } + if (memoryDiag) + _ = AddDiagnoser(MemoryDiagnoser.Default); + if (hardwareCounters) + _ = AddHardwareCounters(HardwareCounter.CacheMisses, HardwareCounter.BranchMispredictions); + + _ = AddJob(net10Job.WithId(".NET 10")); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/InliningTests.cs b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/InliningTests.cs deleted file mode 100644 index 9efa88bdb48..00000000000 --- a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/InliningTests.cs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Diagnostics.Windows.Configs; -using Tsavorite.core; - -#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)] -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member -#pragma warning disable IDE0048 // Add parentheses for clarity -#pragma warning disable IDE0130 // Namespace does not match folder structure - -namespace BenchmarkDotNetTests -{ -#pragma warning disable IDE0065 // Misplaced using directive - using SpanByteStoreFunctions = StoreFunctions; - - [InliningDiagnoser(logFailuresOnly: true, allowedNamespaces: ["Tsavorite.core"])] - [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)] - public class InliningTests - { - [Params(1_000_000)] - public int NumRecords; - - TsavoriteKV> store; - IDevice logDevice; - string logDirectory; - - void SetupStore() - { - logDirectory = BenchmarkDotNetTestsApp.TestDirectory; - var logFilename = Path.Combine(logDirectory, $"{nameof(InliningTests)}_{Guid.NewGuid()}.log"); - logDevice = Devices.CreateLogDevice(logFilename, preallocateFile: true, deleteOnClose: true, useIoCompletionPort: true); - - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = logDevice - }, StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - } - - unsafe void PopulateStore() - { - using var session = store.NewSession>(new()); - var bContext = session.BasicContext; - - Span keyVec = stackalloc byte[sizeof(long)]; - var keySpanByte = SpanByte.FromPinnedSpan(keyVec); - - Span valueVec = stackalloc byte[sizeof(long)]; - var valueSpanByte = SpanByte.FromPinnedSpan(valueVec); - - for (long ii = 0; ii < NumRecords; ++ii) - { - *(long*)keySpanByte.ToPointer() = ii; - *(long*)valueSpanByte.ToPointer() = ii + NumRecords; - _ = bContext.Upsert(keySpanByte, valueSpanByte); - } - } - - [GlobalSetup] - public void SetupPopulatedStore() - { - SetupStore(); - PopulateStore(); - } - - [GlobalCleanup] - public void TearDown() - { - store?.Dispose(); - store = null; - logDevice?.Dispose(); - logDevice = null; - try - { - Directory.Delete(logDirectory); - } - catch { } - } - - [BenchmarkCategory("Upsert"), Benchmark] - public unsafe void Upsert() - { - using var session = store.NewSession>(new()); - var bContext = session.BasicContext; - - Span keyVec = stackalloc byte[sizeof(long)]; - var keySpanByte = SpanByte.FromPinnedSpan(keyVec); - - Span valueVec = stackalloc byte[sizeof(long)]; - var valueSpanByte = SpanByte.FromPinnedSpan(valueVec); - - for (long ii = 0; ii < NumRecords; ++ii) - { - *(long*)keySpanByte.ToPointer() = ii; - *(long*)valueSpanByte.ToPointer() = ii + NumRecords * 2; - _ = bContext.Upsert(keySpanByte, valueSpanByte); - } - } - - [BenchmarkCategory("RMW"), Benchmark] - public unsafe void RMW() - { - using var session = store.NewSession>(new()); - var bContext = session.BasicContext; - - Span keyVec = stackalloc byte[sizeof(long)]; - var keySpanByte = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[sizeof(long)]; - var inputSpanByte = SpanByte.FromPinnedSpan(inputVec); - - for (long ii = 0; ii < NumRecords; ++ii) - { - *(long*)keySpanByte.ToPointer() = ii; - *(long*)inputSpanByte.ToPointer() = ii + NumRecords * 3; - _ = bContext.RMW(keySpanByte, inputSpanByte); - } - - _ = bContext.CompletePending(); - } - - [BenchmarkCategory("Read"), Benchmark] - public unsafe void Read() - { - using var session = store.NewSession>(new()); - var bContext = session.BasicContext; - - Span keyVec = stackalloc byte[sizeof(long)]; - var keySpanByte = SpanByte.FromPinnedSpan(keyVec); - - for (long ii = 0; ii < NumRecords; ++ii) - { - *(long*)keySpanByte.ToPointer() = ii; - _ = bContext.Read(keySpanByte); - } - _ = bContext.CompletePending(); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/IterationTests.cs b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/IterationTests.cs index 519c32c55cb..a722fe510bb 100644 --- a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/IterationTests.cs +++ b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/IterationTests.cs @@ -13,17 +13,33 @@ namespace BenchmarkDotNetTests { #pragma warning disable IDE0065 // Misplaced using directive - using SpanByteStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)] public class IterationTests { - const int NumRecords = 1_000_000; + public int NumRecords => BenchmarkDotNetTestsApp.NumRecords; - [Params(true, false)] - public bool FlushAndEvict; + public static bool? FlushAndEvictConfig; - TsavoriteKV> store; + [ParamsSource(nameof(FlushAndEvictProvider))] + public bool FlushAndEvict { get; set; } + + /// + /// Operation parameters provider + /// + public IEnumerable FlushAndEvictProvider() + { + if (FlushAndEvictConfig.HasValue) + { + yield return FlushAndEvictConfig.Value; + yield break; + } + yield return false; + yield return true; + } + + TsavoriteKV> store; IDevice logDevice; string logDirectory; @@ -37,27 +53,25 @@ void SetupStore() { IndexSize = 1L << 26, LogDevice = logDevice - }, StoreFunctions.Create() + }, StoreFunctions.Create(new SpanByteComparer(), new SpanByteRecordTriggers()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } unsafe void PopulateStore() { - using var session = store.NewSession>(new()); + using var session = store.NewSession>(new()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var keySpanByte = SpanByte.FromPinnedSpan(keyVec); - - Span valueVec = stackalloc byte[sizeof(long)]; - var valueSpanByte = SpanByte.FromPinnedSpan(valueVec); + long keyNum = 0, valueNum = 0; + Span key = SpanByte.FromPinnedVariable(ref keyNum); + Span value = SpanByte.FromPinnedVariable(ref valueNum); for (long ii = 0; ii < NumRecords; ++ii) { - *(long*)keySpanByte.ToPointer() = ii; - *(long*)valueSpanByte.ToPointer() = ii + NumRecords; - _ = bContext.Upsert(keySpanByte, valueSpanByte); + keyNum = ii; + valueNum = ii + NumRecords; + _ = bContext.Upsert(new SpanByteKey(key), value); } if (FlushAndEvict) @@ -88,7 +102,7 @@ public void TearDown() [BenchmarkCategory("Cursor"), Benchmark] public void Cursor() { - using var session = store.NewSession>(new()); + using var session = store.NewSession>(new()); var scanFunctions = new ScanFunctions(); var cursor = 0L; @@ -102,7 +116,7 @@ class ScanCounter internal int count; } - internal struct ScanFunctions : IScanIteratorFunctions + internal struct ScanFunctions : IScanIteratorFunctions { private readonly ScanCounter counter; @@ -114,17 +128,14 @@ internal struct ScanFunctions : IScanIteratorFunctions public bool OnStart(long beginAddress, long endAddress) => true; /// - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { ++counter.count; cursorRecordResult = CursorRecordResult.Accept; return true; } - /// - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - /// public void OnStop(bool completed, long numberOfRecords) { } diff --git a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/OperationTests.cs b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/OperationTests.cs new file mode 100644 index 00000000000..a071cb73e21 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/OperationTests.cs @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.InteropServices; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using Tsavorite.core; + +#pragma warning disable 0649 // Field 'field' is never assigned to, and will always have its default value 'value'; happens due to [Params(..)] +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member +#pragma warning disable IDE0048 // Add parentheses for clarity +#pragma warning disable IDE0130 // Namespace does not match folder structure + +namespace BenchmarkDotNetTests +{ +#pragma warning disable IDE0065 // Misplaced using directive + using SpanByteStoreFunctions = StoreFunctions; + + [GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory, BenchmarkLogicalGroupRule.ByParams)] + public class OperationTests + { + public int NumRecords => BenchmarkDotNetTestsApp.NumRecords; + + TsavoriteKV> store; + IDevice logDevice; + string logDirectory; + + ClientSession> session; + BasicContext> bContext; + + void SetupStore() + { + logDirectory = BenchmarkDotNetTestsApp.TestDirectory; + var logFilename = Path.Combine(logDirectory, $"{nameof(OperationTests)}_{Guid.NewGuid()}.log"); + logDevice = Devices.CreateLogDevice(logFilename, preallocateFile: true, deleteOnClose: true, useIoCompletionPort: true); + + store = new(new() + { + IndexSize = 1L << 26, + LogDevice = logDevice + }, StoreFunctions.Create(new SpanByteComparer(), new SpanByteRecordTriggers()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + session = store.NewSession(new()); + bContext = session.BasicContext; + } + + void PopulateStore(int start = 0) + { + Span keySpan = stackalloc byte[sizeof(long)]; + Span valueSpan = stackalloc byte[sizeof(long)]; + + var end = start + NumRecords; + + for (long ii = start; ii < end; ++ii) + { + MemoryMarshal.Cast(keySpan)[0] = ii; + MemoryMarshal.Cast(valueSpan)[0] = ii + NumRecords; + _ = bContext.Upsert(new SpanByteKey(keySpan), valueSpan); + } + } + + [GlobalSetup] + public void SetupPopulatedStore() + { + SetupStore(); + PopulateStore(); + } + + [GlobalCleanup] + public void TearDown() + { + session?.Dispose(); + session = null; + store?.Dispose(); + store = null; + logDevice?.Dispose(); + logDevice = null; + try + { + Directory.Delete(logDirectory); + } + catch { } + } + + [Benchmark] + public void Insert() + { + // Populate with a second batch + PopulateStore(NumRecords); + } + + [Benchmark] + public void Upsert() + { + Span keySpan = stackalloc byte[sizeof(long)]; + var key = new SpanByteKey(keySpan); + ref var keyLongRef = ref MemoryMarshal.Cast(keySpan)[0]; + + Span valueSpan = stackalloc byte[sizeof(long)]; + ref var valueLongRef = ref MemoryMarshal.Cast(valueSpan)[0]; + + for (long ii = 0; ii < NumRecords; ++ii) + { + keyLongRef = ii; + valueLongRef = ii + NumRecords * 2; + _ = bContext.Upsert(key, valueSpan); + } + } + + [Benchmark] + public void RMW() + { + Span keySpan = stackalloc byte[sizeof(long)]; + var key = new SpanByteKey(keySpan); + ref var keyLongRef = ref MemoryMarshal.Cast(keySpan)[0]; + + Span inputSpan = stackalloc byte[sizeof(long)]; + ref var inputLongRef = ref MemoryMarshal.Cast(inputSpan)[0]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(inputSpan); + + for (long ii = 0; ii < NumRecords; ++ii) + { + keyLongRef = ii; + inputLongRef = ii + NumRecords * 3; + _ = bContext.RMW(key, ref pinnedInputSpan); + } + + _ = bContext.CompletePending(); + } + + [Benchmark] + public void Read() + { + Span keySpan = stackalloc byte[sizeof(long)]; + var key = new SpanByteKey(keySpan); + ref var keyLongRef = ref MemoryMarshal.Cast(keySpan)[0]; + + Span outputSpan = stackalloc byte[sizeof(long)]; + var output = SpanByteAndMemory.FromPinnedSpan(outputSpan); + + for (long ii = 0; ii < NumRecords; ++ii) + { + keyLongRef = ii; + _ = bContext.Read(key, ref output); + } + _ = bContext.CompletePending(); + } + } + + public sealed class BDNSpanByteFunctions : SpanByteFunctions + { + /// + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref ReadInfo readInfo) + { + srcLogRecord.ValueSpan.CopyTo(output.SpanByte.Span); + return true; + } + + // Note: Currently, only the ReadOnlySpan form of InPlaceWriter value is used here. + + /// + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration + srcValue.CopyTo(logRecord.ValueSpan); + return true; + } + + /// + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration + srcValue.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + /// + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + => throw new TsavoriteException("InitialUpdater not implemented for BDN"); + + /// + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + => throw new TsavoriteException("CopyUpdater not implemented for BDN"); + + /// + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + // This does not try to set ETag or Expiration + input.CopyTo(logRecord.ValueSpan); + return true; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/SpanByteKey.cs b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/SpanByteKey.cs new file mode 100644 index 00000000000..b9bac02376d --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/BDN-Tsavorite.Benchmark/SpanByteKey.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#if !NET9_0_OR_GREATER +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +#endif +using Tsavorite.core; + +namespace BenchmarkDotNetTests +{ + public readonly +#if NET9_0_OR_GREATER + ref +#endif + struct SpanByteKey : IKey + { +#if !NET9_0_OR_GREATER + private readonly unsafe void* ptr; + private readonly int len; +#endif + + /// + /// In benchmarks, we don't wait for pending operations to complete so the span isn't fixed. + /// + public readonly bool IsPinned => false; + + /// + public readonly bool IsEmpty => false; + +#if NET9_0_OR_GREATER + /// + public readonly ReadOnlySpan KeyBytes { get; } +#else + /// + public readonly unsafe ReadOnlySpan KeyBytes => new(ptr, len); +#endif + + public SpanByteKey(ReadOnlySpan keyBytes) + { +#if NET9_0_OR_GREATER + KeyBytes = keyBytes; +#else + unsafe + { + ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(keyBytes)); + len = keyBytes.Length; + } +#endif + } + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/EntryPoint.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/EntryPoint.cs new file mode 100644 index 00000000000..b48ec41f597 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/EntryPoint.cs @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading; +using CommandLine; + +namespace Tsavorite.kvbench +{ + /// + /// Real entry point. Parses options, runs warmup + load + iterations, emits + /// results, handles SIGINT/SIGTERM idempotently, saves/restores ThreadPool. + /// + internal static class EntryPoint + { + static int _shutdownStarted; // for Interlocked.Exchange + + public static int Run(string[] args) + { + var parser = new Parser(s => { s.HelpWriter = Console.Out; s.CaseSensitive = false; }); + var parsed = parser.ParseArguments(args); + if (parsed.Tag == ParserResultType.NotParsed) + return 64; + + var opts = parsed.Value; + var err = opts.Resolve(); + if (err != null) + { + Console.Error.WriteLine($"ERROR: {err}"); + return 64; + } + + int oldMinW = 0, oldMinIO = 0; + bool tunedThreadPool = false; + if (!opts.NoThreadPoolTune) + { + ThreadPool.GetMinThreads(out oldMinW, out oldMinIO); + // Size to the peak thread count actually used by this invocation (the sweep max + // when --run-threads-sweep is set, otherwise --threads). Previously this used + // opts.Threads, which is the default (often 1) when the sweep is in use, leaving + // the pool undersized for the high-thread sweep cells. Floor at 256 to keep + // small-N runs comfortably above the OS-default min (~CoreCount). + int target = Math.Max(opts.ResolvedMaxThreads * 2, 256); + ThreadPool.SetMinThreads(Math.Max(oldMinW, target), Math.Max(oldMinIO, target)); + tunedThreadPool = true; + } + + KvBenchmark engine = null; + // Per-thread-count run results (key = thread count, value = iter list) + var sweepResults = new Dictionary>(); + + // Idempotent shutdown handlers (only fires once; the finally-path call is a no-op + // unless an interrupt has set the guard first). + void Shutdown(string reason) + { + if (Interlocked.Exchange(ref _shutdownStarted, 1) != 0) return; + Console.Error.WriteLine($"[interrupt] reason={reason}"); + try { engine?.Dispose(); } catch { /* swallow */ } + if (tunedThreadPool) + { + try { ThreadPool.SetMinThreads(oldMinW, oldMinIO); } catch { /* swallow */ } + } + } + + Console.CancelKeyPress += (_, e) => { e.Cancel = true; Shutdown("sigint"); Environment.Exit(130); }; + AppDomain.CurrentDomain.ProcessExit += (_, _) => Shutdown("sigterm"); + + try + { + engine = new KvBenchmark(opts); + var output = new KvOutput(opts, engine.DataPath, args); + output.EmitConfigHuman(engine.Pinning); + + // ---- Load phase ---- + var loadResult = engine.Load(); + output.EmitPhaseHuman(loadResult); + output.EmitResultJson(loadResult, engine.Pinning); + output.EmitResultCsv(loadResult, engine.Pinning); + + // ---- Optional --validate after load ---- + if (opts.Validate) + { + Console.WriteLine("[validate] reading back all keys..."); + var (mismatches, misses) = engine.Validate(); + if (mismatches > 0 || misses > 0) + { + Console.Error.WriteLine($"[validate] FAILED: mismatches={mismatches} misses={misses}"); + return 2; + } + Console.WriteLine("[validate] OK"); + } + + // ---- Run sweep ---- + // Run the full --iterations loop ONCE for each thread count in the sweep + // (single load → multiple run experiments). + PhaseResult lastRun = null; + foreach (var t in opts.ResolvedRunThreadsSweep) + { + var iters = new List(); + sweepResults[t] = iters; + if (opts.ResolvedRunThreadsSweep.Length > 1) + Console.WriteLine($"[sweep] starting run phase with threads={t}"); + for (int it = 1; it <= opts.Iterations; it++) + { + var r = engine.RunIteration(it, t); + iters.Add(r); + lastRun = r; + output.EmitPhaseHuman(r, threadCount: t); + output.EmitResultJson(r, engine.Pinning, threadCount: t); + output.EmitResultCsv(r, engine.Pinning, threadCount: t); + } + if (iters.Count > 0) + { + output.EmitAggregateHuman(iters, threadCount: t); + output.EmitAggregateJson(iters, engine.Pinning, threadCount: t); + output.EmitAggregateCsv(iters, engine.Pinning, threadCount: t); + } + } + + // Final clean summary block. + output.EmitFinalSummary(loadResult, sweepResults, engine.Pinning); + + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"FATAL: {ex.GetType().Name}: {ex.Message}"); + Console.Error.WriteLine(ex.StackTrace); + return 1; + } + finally + { + // Suppress the ProcessExit-driven Shutdown emission for normal exits. + Interlocked.Exchange(ref _shutdownStarted, 1); + try { engine?.Dispose(); } catch { /* swallow */ } + if (tunedThreadPool) + { + try { ThreadPool.SetMinThreads(oldMinW, oldMinIO); } catch { /* swallow */ } + } + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KV.benchmark.csproj b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KV.benchmark.csproj new file mode 100644 index 00000000000..d02dabf6749 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KV.benchmark.csproj @@ -0,0 +1,21 @@ + + + + Exe + ../../../../../../Garnet.snk + false + true + true + + + + + + + + + + + diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs new file mode 100644 index 00000000000..dbdba3295a6 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Setup.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ + // Per-process setup helpers: data-path resolution, per-run directory creation, + // stale-run cleanup (so abandoned runs from killed processes are reclaimed on + // next start), and device construction (native vs managed vs null). + public sealed partial class KvBenchmark + { + const string OwnerSentinelFileName = ".kv-benchmark-owner"; + const string RunDirPrefix = "kv-run-"; + + /// Resolves --data-path or falls back to the OS temp dir. + static string ResolveDataPath(Options opts) + => !string.IsNullOrWhiteSpace(opts.DataPath) + ? opts.DataPath + : Path.Combine(Path.GetTempPath(), "kv-benchmark"); + + /// Creates a unique kv-run-{ts}-{pid} subdir and tags it with an owner sentinel. + static string CreateRunDir(string dataPath) + { + var ts = DateTime.UtcNow.ToString("yyyyMMddTHHmmssZ", CultureInfo.InvariantCulture); + var pid = Environment.ProcessId; + var dir = Path.Combine(dataPath, $"{RunDirPrefix}{ts}-{pid}"); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, OwnerSentinelFileName), $"pid={pid}\nstart_utc={ts}\n"); + return dir; + } + + /// Deletes kv-run-* directories owned by PIDs that are no longer alive. + static void CleanStaleRunDirs(string dataPath) + { + try + { + foreach (var dir in Directory.EnumerateDirectories(dataPath, $"{RunDirPrefix}*")) + { + var sentinel = Path.Combine(dir, OwnerSentinelFileName); + if (!File.Exists(sentinel)) continue; + int? pid = TryReadOwnerPid(sentinel); + if (pid is null || IsPidAlive(pid.Value)) continue; + try { Directory.Delete(dir, recursive: true); } catch { /* skip locked */ } + } + } + catch { /* best-effort */ } + } + + static int? TryReadOwnerPid(string sentinel) + { + try + { + foreach (var line in File.ReadAllLines(sentinel)) + { + if (line.StartsWith("pid=", StringComparison.Ordinal) && + int.TryParse(line.AsSpan(4), out var pid)) return pid; + } + } + catch { /* ignored */ } + return null; + } + + static bool IsPidAlive(int pid) + { + try { _ = Process.GetProcessById(pid); return true; } + catch { return false; } + } + + /// + /// Builds the IDevice for the requested backend. Native (libaio) on Linux, + /// the platform-default managed device elsewhere, or a no-op Null device when + /// the entire dataset fits in the mutable log window. + /// + static IDevice CreateDevice(Options opts, string logPath) + { + var devType = opts.ResolvedDeviceType; + int numCt = opts.DeviceCompletionThreads > 0 ? opts.DeviceCompletionThreads : 1; + IDevice dev; + + if (devType == DeviceType.Native && OperatingSystem.IsLinux()) + { + dev = new NativeStorageDevice(logPath, + deleteOnClose: true, + disableFileBuffering: true, + numCompletionThreads: numCt, + ioBackend: opts.ResolvedIoBackend); + } + else if (devType == DeviceType.Null) + { + dev = Devices.CreateLogDevice(null, deviceType: DeviceType.Null); + } + else + { + dev = Devices.CreateLogDevice(logPath, + deviceType: devType, + preallocateFile: true, + deleteOnClose: true, + useIoCompletionPort: true, + disableFileBuffering: true); + } + + if (opts.DeviceThrottle > 0) + dev.ThrottleLimit = opts.DeviceThrottle; + return dev; + } + } + + internal static class OptionsExtensions + { + /// True when the rumd tuple has a non-zero delete fraction. + public static bool RumdHasDeletes(this Options o) + { + using var en = o.Rumd.GetEnumerator(); + int idx = 0; + while (en.MoveNext()) + { + if (idx == 3) return en.Current > 0; + idx++; + } + return false; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Validate.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Validate.cs new file mode 100644 index 00000000000..5c7d1160e05 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Validate.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ + // Optional post-load read-back: confirms every key the load phase wrote is + // actually retrievable and that its first cache line matches the per-thread + // pattern. Slow (single-threaded), opt-in via --validate; used only as a + // sanity check after experiments, not on the measurement path. + public sealed partial class KvBenchmark + { + /// + /// Reads back every key the load phase wrote and verifies the first + /// bytes match the + /// per-thread baked pattern. Returns (mismatches, readMisses). + /// + /// Records below HeadAddress trigger async disk I/O (Status.IsPending); + /// we issue ops in batches and drain via CompletePendingWithOutputs. + /// + public unsafe (long mismatches, long readMisses) Validate() + { + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + byte* inputPtr = stackalloc byte[Options.ValueSize]; + var inputSpan = new Span(inputPtr, Options.ValueSize); + var pinnedInput = PinnedSpanByte.FromPinnedSpan(inputSpan); + + long mismatches = 0; + long misses = 0; + int nCmp = Math.Min(KvSessionFunctions.kReaderCopyBytes, Options.ValueSize); + int loadThreads = Options.ResolvedLoadThreads; + long keyCount = Options.Keys; + + // Per-op pinned output buffer (one per slot in the batch) so async pending + // reads can write into a stable location while we issue more ops in the + // same chunk. SpanByteAndMemory just records the destination pointer. + const int kBatch = 256; + var outputArr = new byte[kBatch * KvSessionFunctions.kReaderCopyBytes]; + var pinnedHandle = GCHandle.Alloc(outputArr, GCHandleType.Pinned); + try + { + byte* outBase = (byte*)pinnedHandle.AddrOfPinnedObject(); + + KvKey key = default; + long k = 0; + while (k < keyCount) + { + long batchEnd = Math.Min(k + kBatch, keyCount); + int slot = 0; + + for (long kk = k; kk < batchEnd; ++kk, ++slot) + { + int writerThread = (int)((double)kk / keyCount * loadThreads); + if (writerThread >= loadThreads) writerThread = loadThreads - 1; + + key.Value = kk; + var outSpan = new Span(outBase + slot * KvSessionFunctions.kReaderCopyBytes, KvSessionFunctions.kReaderCopyBytes); + var output = SpanByteAndMemory.FromPinnedSpan(outSpan); + var st = bContext.Read(key, ref pinnedInput, ref output, Empty.Default); + if (st.IsPending) + { + // Will be checked in the drain pass via KeyBytes lookup. + continue; + } + if (!st.Found) { ++misses; continue; } + for (int i = 0; i < nCmp; i++) + { + var expected = (byte)((writerThread * 31 + i) & 0xFF); + if (outSpan[i] != expected) { ++mismatches; break; } + } + } + + // Drain any async-completed reads and verify each one. + bContext.CompletePendingWithOutputs(out var completed, wait: true); + while (completed.Next()) + { + ref var cr = ref completed.Current; + if (!cr.Status.Found) { ++misses; cr.Output.Dispose(); continue; } + + // Reconstruct writer thread from the key value. + var kb = cr.Key.KeyBytes; + long completedKey = MemoryMarshal.Read(kb); + int writerThread = (int)((double)completedKey / keyCount * loadThreads); + if (writerThread >= loadThreads) writerThread = loadThreads - 1; + + var outSpan = cr.Output.SpanByte.Span; + for (int i = 0; i < nCmp; i++) + { + var expected = (byte)((writerThread * 31 + i) & 0xFF); + if (outSpan[i] != expected) { ++mismatches; break; } + } + cr.Output.Dispose(); + } + completed.Dispose(); + + k = batchEnd; + } + } + finally + { + pinnedHandle.Free(); + } + + return (mismatches, misses); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Worker.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Worker.cs new file mode 100644 index 00000000000..87a1663f922 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.Worker.cs @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ +#pragma warning disable IDE0065 // Misplaced using directive + using KvAllocator = ObjectAllocator>; + using KvStoreFunctions = StoreFunctions; + + // Per-thread worker entrypoint + RUN-phase hot loop. + // Kept in its own file so the hot path is easy to reason about and review: + // any change here is liable to move ops/sec. + public sealed partial class KvBenchmark + { + /// Global chunk counter for the RUN phase (Interlocked.Add per chunk). + long globalChunkIdx; + + /// + /// Per-thread worker. Owns its Tsavorite session, picks the load or run path, + /// and writes its final counters into . + /// + unsafe void WorkerProc(int threadIdx, bool isLoad, WorkerStats stats, CountdownEvent ready, int threadCount) + { + Pinning.PinWorker(threadIdx); + + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + ref var mySlot = ref scoreboard[threadIdx + 1]; + ref var doneFlag = ref doneBox[0].Value; + + // Load partition (only meaningful when isLoad=true). + long loadFrom = 0, loadTo = 0; + if (isLoad) + { + loadFrom = (long)((double)Options.Keys * threadIdx / threadCount); + loadTo = (long)((double)Options.Keys * (threadIdx + 1) / threadCount); + } + + ready.Signal(); + gate.Wait(); + + long localOps = 0; + long reads = 0, writes = 0, deletes = 0; + var allocBefore = GC.GetAllocatedBytesForCurrentThread(); + + if (isLoad) + { + // Each thread Upserts its partition deterministically. Single small buffer suffices. + byte* valuePtr = stackalloc byte[Options.ValueSize]; + for (int i = 0; i < Options.ValueSize; i++) + valuePtr[i] = (byte)((threadIdx * 31 + i) & 0xFF); + var valueSpan = new Span(valuePtr, Options.ValueSize); + + KvKey key = default; + for (long chunkStart = loadFrom; chunkStart < loadTo; chunkStart += kChunkSize) + { + long chunkEnd = Math.Min(chunkStart + kChunkSize, loadTo); + bContext.CompletePending(false); + for (long k = chunkStart; k < chunkEnd; k++) + { + key.Value = k; + bContext.Upsert(key, valueSpan, Empty.Default); + } + localOps = chunkEnd - loadFrom; + Volatile.Write(ref mySlot.Value, localOps); + } + bContext.CompletePending(true); + writes = localOps; + } + else + { + var deleteReinsert = Options.RumdHasDeletes(); + + // Hot-path buffers: stackalloc'd HERE (caller of RunWorkload) and passed in by ref. + // Two design points: + // 1. Stackalloc here (not inside RunWorkload) keeps the hot-loop method body small + // enough for the JIT to inline BasicContext.Read into it (~3.7% measured). + // 2. Each buffer is overallocated by 31 B and rounded up to a 32-byte boundary + // so Reader's `Slice(0, 32).CopyTo(...)` -> single 32-byte AVX2 vmovdqu has an + // aligned destination. (Source side stays unaligned: Tsavorite records have + // a 21-byte header before the value, which can't be fixed here. Measured + // ~+5% at memory-bound scale.) + byte* valueRaw = stackalloc byte[Options.ValueSize + 31]; + byte* valuePtrD = (byte*)(((nuint)valueRaw + 31u) & ~(nuint)31u); + byte* inputRaw = stackalloc byte[Options.ValueSize + 31]; + byte* inputPtrD = (byte*)(((nuint)inputRaw + 31u) & ~(nuint)31u); + byte* outputRaw = stackalloc byte[KvSessionFunctions.kReaderCopyBytes + 31]; + byte* outputPtrD = (byte*)(((nuint)outputRaw + 31u) & ~(nuint)31u); + for (int i = 0; i < Options.ValueSize; i++) + valuePtrD[i] = (byte)((threadIdx * 31 + i) & 0xFF); + for (int i = 0; i < Options.ValueSize; i++) + inputPtrD[i] = (byte)((threadIdx * 17 + i + 1) & 0xFF); + var valueSpanD = new Span(valuePtrD, Options.ValueSize); + var inputSpanD = new Span(inputPtrD, Options.ValueSize); + var outputSpanD = new Span(outputPtrD, KvSessionFunctions.kReaderCopyBytes); + var pinnedInputD = PinnedSpanByte.FromPinnedSpan(inputSpanD); + var outputD = SpanByteAndMemory.FromPinnedSpan(outputSpanD); + + (localOps, reads, writes, deletes) = RunWorkload( + bContext, ref mySlot, ref doneFlag, + Options.Keys, Options.UseZipf, + valueSpanD, ref pinnedInputD, ref outputD, + Options.ReadPct, Options.UpsertPctCumulative, Options.RmwPctCumulative, + deleteReinsert, + seed: (uint)(Options.Seed) + (uint)(threadIdx + 1)); + } + + // Final scoreboard tick so the post-Join sum captures the in-flight chunk. + Volatile.Write(ref mySlot.Value, localOps); + + var allocAfter = GC.GetAllocatedBytesForCurrentThread(); + stats.LocalOps = localOps; + stats.Reads = reads; + stats.Writes = writes; + stats.Deletes = deletes; + stats.AllocBytesDelta = allocAfter - allocBefore; + stats.FinalExitTicks = Stopwatch.GetTimestamp(); + } + + /// + /// RUN-phase hot loop. Structure (kept identical for all rumd ratios and distributions): + /// + /// Per-chunk: Interlocked.Add on , then iterate kChunkSize ops. + /// Per-op: inline xorshift32 key generation (bitmask if keyCount is a power of two, + /// else Lemire's fast modulo; zipf path delegates to ). + /// Per-op: SECOND independent xorshift32 coin toss for op selection — independence is + /// MANDATORY when distribution=zipf because zipf consumes its source RNG non-uniformly. + /// Per-op: compare coin toss against pre-computed 32-bit cutoffs (no per-op multiply or divide). + /// Per-op: chained if (...) { ...; continue; } branches; delete path optionally re-Upserts. + /// Per-chunk: the running total into the per-thread scoreboard slot. + /// Done flag checked at chunk boundary only (worst-case stop lag: one chunk ≈ 0.3 ms). + /// + /// Returns (localOps, reads, writes, deletes). Reinserts are counted under writes so they appear + /// in both the live scoreboard sum and the final per-phase total. + /// + unsafe (long localOps, long reads, long writes, long deletes) RunWorkload( + Tsavorite.core.BasicContext bContext, + ref PaddedLong slot, ref bool doneFlag, + long keyCount, bool useZipf, + Span value, ref PinnedSpanByte pinnedInputSpan, ref SpanByteAndMemory _output, + int readPercent, int upsertPercent, int rmwPercent, + bool deleteReinsert, + uint seed) + { + // RNG #1 — key generation. RNG #2 — op-select. Seeded distinctly so they're independent. + uint xk = seed == 0 ? 1u : seed; + uint yk = 362436069u, zk = 521288629u, wk = 88675123u; + uint xr = (seed == 0 ? 1u : seed) ^ 0x9E3779B9u; + if (xr == 0) xr = 0x9E3779B9u; + uint yr = 362436069u, zr = 521288629u, wr = 88675123u; + + // Zipf needs a XoshiroRng struct; seeded from the keygen state. + var rngStruct = new XoshiroRng(((ulong)xk << 32) | wk); + var zipf = useZipf ? new ZipfGenerator(ZipfConstants) : default; + + // Hoist all per-thread invariants out of the hot loop. + bool fast32 = keyCount > 0 && keyCount <= uint.MaxValue; + uint keyCount32 = fast32 ? (uint)keyCount : 0u; + bool keysPow2 = fast32 && (keyCount32 & (keyCount32 - 1)) == 0; + uint keyMask32 = fast32 ? keyCount32 - 1 : 0u; + + // Pre-compute 32-bit op-select cutoffs so the coin toss is a single compare per op. + // pct=100 -> cutoff = 2^32 (always > any uint), so the branch is always taken. + ulong readCutoff = ((ulong)(uint)readPercent << 32) / 100; + ulong upsertCutoff = ((ulong)(uint)upsertPercent << 32) / 100; + ulong rmwCutoff = ((ulong)(uint)rmwPercent << 32) / 100; + + long reads_done = 0, writes_done = 0, deletes_done = 0; + KvKey key = default; // hoisted; avoids per-op 16-byte zero-init. + + while (!Volatile.Read(ref doneFlag)) + { + long chunk_idx = Interlocked.Add(ref globalChunkIdx, kChunkSize) - kChunkSize; + long chunk_end = chunk_idx + kChunkSize; + bContext.CompletePending(false); + for (long idx = chunk_idx; idx < chunk_end; ++idx) + { + // ===== Key generation (one of three paths, hoisted bools select the right one) ===== + if (useZipf) + { + long kk = zipf.Next(ref rngStruct); + if (kk >= keyCount) kk = keyCount - 1; + key.Value = kk; + } + else if (fast32) + { + uint tk = xk ^ (xk << 11); + xk = yk; yk = zk; zk = wk; + wk = (wk ^ (wk >> 19)) ^ (tk ^ (tk >> 8)); + key.Value = keysPow2 + ? (long)(wk & keyMask32) + : (long)(uint)(((ulong)wk * keyCount32) >> 32); // Lemire fast-mod + } + else + { + // 64-bit keyCount: combine two xorshift32 advances. + uint t1 = xk ^ (xk << 11); + xk = yk; yk = zk; zk = wk; + ulong rhi = (wk = (wk ^ (wk >> 19)) ^ (t1 ^ (t1 >> 8))); + uint t2 = xk ^ (xk << 11); + xk = yk; yk = zk; zk = wk; + ulong rlo = (wk = (wk ^ (wk >> 19)) ^ (t2 ^ (t2 >> 8))); + key.Value = (long)(((rhi << 32) | rlo) % (ulong)keyCount); + } + + // ===== Op selection (independent RNG; compare against precomputed cutoffs) ===== + uint tr = xr ^ (xr << 11); + xr = yr; yr = zr; zr = wr; + wr = (wr ^ (wr >> 19)) ^ (tr ^ (tr >> 8)); + ulong rcoin = wr; + + if (rcoin < readCutoff) + { + bContext.Read(key, ref pinnedInputSpan, ref _output, Empty.Default); + ++reads_done; + continue; + } + if (rcoin < upsertCutoff) + { + bContext.Upsert(key, value, Empty.Default); + ++writes_done; + continue; + } + if (rcoin < rmwCutoff) + { + bContext.RMW(key, ref pinnedInputSpan, Empty.Default); + ++writes_done; + continue; + } + bContext.Delete(key, Empty.Default); + ++deletes_done; + if (deleteReinsert) + { + bContext.Upsert(key, value, Empty.Default); + ++writes_done; + } + } + + Volatile.Write(ref slot.Value, reads_done + writes_done + deletes_done); + } + bContext.CompletePending(true); + long localOps = reads_done + writes_done + deletes_done; + return (localOps, reads_done, writes_done, deletes_done); + } + } +#pragma warning restore IDE0065 +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.cs new file mode 100644 index 00000000000..b92a271b9e5 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvBenchmark.cs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ +#pragma warning disable IDE0065 // Misplaced using directive + using KvAllocator = ObjectAllocator>; + using KvStoreFunctions = StoreFunctions; + + /// Headline result for one phase (load or one run iteration). + public sealed class PhaseResult + { + public string Phase { get; init; } = ""; + public int Iteration { get; init; } + public double ElapsedSec { get; init; } + public long TotalOpsForThroughput { get; init; } + public long FinalTotalOps { get; init; } + public long OvershootOps { get; init; } + public double MaxWorkerExitLagMs { get; init; } + public long Reads { get; init; } + public long Writes { get; init; } + public long Deletes { get; init; } + public long LogBegin { get; init; } + public long LogHead { get; init; } + public long LogReadOnly { get; init; } + public long LogTail { get; init; } + public long AllocBytesByWorkerMax { get; init; } + public int GcGen0Delta { get; init; } + public int GcGen1Delta { get; init; } + public int GcGen2Delta { get; init; } + public bool Interrupted { get; set; } + public string ErrorMessage { get; set; } + + public double OpsPerSec => ElapsedSec > 0 ? TotalOpsForThroughput / ElapsedSec : 0; + } + + /// Per-worker counters returned to the main thread after Join. + internal sealed class WorkerStats + { + public long LocalOps; + public long Reads; + public long Writes; + public long Deletes; + public long AllocBytesDelta; + public long FinalExitTicks; // Stopwatch.GetTimestamp() right before exit + } + + /// + /// KV.benchmark engine. Owns the device, store, scoreboard, and per-phase + /// worker orchestration. Workers always use BasicContext (safe path). + /// Implementation is split across partial files: + /// + /// KvBenchmark.cs — store construction, public phase API ( / / ), worker orchestration. + /// KvBenchmark.Worker.cs — per-thread worker entrypoint and the RUN hot loop. + /// KvBenchmark.Validate.cs — optional post-load read-back verification. + /// KvBenchmark.Setup.cs — data-path / run-dir / stale-cleanup / device construction helpers. + /// + /// + public sealed partial class KvBenchmark : IDisposable + { + public readonly Options Options; + internal readonly KvNumaPinning Pinning; + public readonly ZipfConstants ZipfConstants; + public readonly string DataPath; + public readonly string RunDir; + + // Hot-path shared state (each on its own cache-line-isolated padded type). + readonly PaddedLong[] scoreboard; // length = Threads + 2; [0] and [N+1] are unused sentinels + readonly PaddedBool[] doneBox = new PaddedBool[1]; + readonly ManualResetEventSlim gate = new(false); + long startTicks; + + readonly KvSessionFunctions functions = new(); + readonly IDevice device; + readonly TsavoriteKV store; + + const long kChunkSize = 1024; // power-of-two so (n & (kChunkSize-1)) is a correct mask + + public KvBenchmark(Options opts) + { + Options = opts; + DataPath = ResolveDataPath(opts); + Directory.CreateDirectory(DataPath); + CleanStaleRunDirs(DataPath); + RunDir = CreateRunDir(DataPath); + + // Pin the SETUP thread BEFORE building the store — first-touch policy + // puts the hash index + log buffer on the requested NUMA node. + // Pinning's worker count is updated per phase; size for the MAX so the + // worker_cpu_mask diagnostic also covers all phases. + Pinning = new KvNumaPinning(opts, opts.ResolvedMaxThreads); + Pinning.PinSetupOrReporter(); + + // Scoreboard sized for the largest phase. Slots 1..MaxThreads are per-worker; + // slot 0 and the last slot are sentinel padding. + scoreboard = GC.AllocateArray(opts.ResolvedMaxThreads + 2, pinned: true); + ZipfConstants = opts.UseZipf ? new ZipfConstants(opts.Keys, opts.ZipfTheta) : null; + + var logPath = opts.ResolvedDeviceType == DeviceType.Null ? null : Path.Combine(RunDir, "hlog"); + device = CreateDevice(opts, logPath); + + // Defaults below match Garnet's defaults.conf for apples-to-apples comparison with the + // Garnet RESP server. Anything not exposed via CLI stays at the Tsavorite KVSettings + // default (MutableFraction=0.9, MaxInlineKeySize=128B — both also match Garnet). + var kvSettings = new KVSettings + { + IndexSize = opts.ResolvedIndexRequestedBytes, + LogDevice = device, + LogMemorySize = opts.ResolvedLogMemoryBytes, + PageSize = opts.ResolvedPageSizeBytes, // Garnet default: 16 MB + SegmentSize = opts.ResolvedSegmentSizeBytes, // Garnet default: 1 GB + MaxInlineValueSize = (int)opts.ResolvedMaxInlineValueSizeBytes, // Garnet default: 16 KB + PreallocateLog = opts.PreallocateLog, // Garnet default: false (CLI override available) + // MutableFraction stays at the KVSettings/Garnet default (0.9). + }; + + store = new TsavoriteKV( + kvSettings, + StoreFunctions.Create(SpanByteComparer.Instance, new SpanByteRecordTriggers()), + (allocSettings, sf) => new KvAllocator(allocSettings, sf)); + } + + // ====== Public phase API ====== + + /// Loads Options.Keys records using Options.ResolvedLoadThreads workers. + public PhaseResult Load() + => RunWorkers(phase: "load", isLoad: true, durationSec: 0, iteration: 0, threadCount: Options.ResolvedLoadThreads); + + /// Runs one warmup window (if configured) followed by a measured run window using workers. + public PhaseResult RunIteration(int iteration, int threadCount) + { + if (Options.WarmupSec > 0) + RunWorkers(phase: "warmup", isLoad: false, durationSec: Options.WarmupSec, iteration: iteration, threadCount: threadCount); + return RunWorkers(phase: "run", isLoad: false, durationSec: Options.RunSec, iteration: iteration, threadCount: threadCount); + } + + // ====== Worker orchestration ====== + + /// + /// Spawns Workers, sleeps for the run window (or waits for load to finish), + /// signals done, joins, and assembles the . + /// + PhaseResult RunWorkers(string phase, bool isLoad, int durationSec, int iteration, int threadCount) + { + // Reset shared state for this phase. + Volatile.Write(ref doneBox[0].Value, false); + Volatile.Write(ref globalChunkIdx, 0); + for (int i = 0; i < scoreboard.Length; i++) + Volatile.Write(ref scoreboard[i].Value, 0); + gate.Reset(); + Pinning.WorkerCount = threadCount; // updates the FirstUnpinnedCpu used by the reporter + + var threads = new Thread[threadCount]; + var stats = new WorkerStats[threadCount]; + var ready = new CountdownEvent(threadCount); + + for (int i = 0; i < threadCount; i++) + { + int idx = i; + stats[idx] = new WorkerStats(); + threads[idx] = new Thread(() => WorkerProc(idx, isLoad, stats[idx], ready, threadCount)) + { + IsBackground = false, + Name = $"kv-bench-{phase}-{idx}", + }; + threads[idx].Start(); + } + + // All workers parked on the gate. Force a full GC + finalizer pass + GC NOW so any + // pending garbage from setup / prior phase is reclaimed BEFORE the timed window opens. + // The double-Collect handles object resurrection during finalization. Hot loop is + // allocation-free (~4 KB per worker for stackallocs), so this purely guards against + // a stray Gen2 firing mid-window and biasing one iteration. + ready.Wait(); + GC.Collect(2, GCCollectionMode.Forced, blocking: true); + GC.WaitForPendingFinalizers(); + GC.Collect(2, GCCollectionMode.Forced, blocking: true); + var gc0 = GC.CollectionCount(0); + var gc1 = GC.CollectionCount(1); + var gc2 = GC.CollectionCount(2); + startTicks = Stopwatch.GetTimestamp(); + gate.Set(); + + long doneTicks; + long totalForThroughput; + if (isLoad) + { + // Load: each partition is bounded; just wait for workers to finish. + foreach (var t in threads) t.Join(); + doneTicks = Stopwatch.GetTimestamp(); + totalForThroughput = SumScoreboard(threadCount); + } + else if (durationSec <= 0) + { + // Degenerate zero-second run: signal done immediately. + Volatile.Write(ref doneBox[0].Value, true); + doneTicks = Stopwatch.GetTimestamp(); + totalForThroughput = SumScoreboard(threadCount); + foreach (var t in threads) t.Join(); + } + else + { + Thread.Sleep(TimeSpan.FromSeconds(durationSec)); + Volatile.Write(ref doneBox[0].Value, true); + doneTicks = Stopwatch.GetTimestamp(); + totalForThroughput = SumScoreboard(threadCount); + foreach (var t in threads) t.Join(); + } + + // Aggregate per-worker stats. + var finalTotal = 0L; + var maxAlloc = 0L; + long reads = 0, writes = 0, deletes = 0; + double maxExitLagMs = 0; + foreach (var s in stats) + { + finalTotal += s.LocalOps; + reads += s.Reads; + writes += s.Writes; + deletes += s.Deletes; + if (s.AllocBytesDelta > maxAlloc) maxAlloc = s.AllocBytesDelta; + var lagMs = (s.FinalExitTicks - doneTicks) * 1000.0 / Stopwatch.Frequency; + if (lagMs > maxExitLagMs) maxExitLagMs = lagMs; + } + + var elapsedSec = (doneTicks - startTicks) / (double)Stopwatch.Frequency; + if (isLoad) totalForThroughput = finalTotal; // load: finalTotal == sum by construction + + return new PhaseResult + { + Phase = phase, + Iteration = iteration, + ElapsedSec = elapsedSec, + TotalOpsForThroughput = totalForThroughput, + FinalTotalOps = finalTotal, + OvershootOps = Math.Max(0, finalTotal - totalForThroughput), + MaxWorkerExitLagMs = maxExitLagMs, + Reads = reads, + Writes = writes, + Deletes = deletes, + LogBegin = store.Log.BeginAddress, + LogHead = store.Log.HeadAddress, + LogReadOnly = store.Log.ReadOnlyAddress, + LogTail = store.Log.TailAddress, + AllocBytesByWorkerMax = maxAlloc, + GcGen0Delta = GC.CollectionCount(0) - gc0, + GcGen1Delta = GC.CollectionCount(1) - gc1, + GcGen2Delta = GC.CollectionCount(2) - gc2, + }; + } + + /// Sums per-thread scoreboard slots [1..threadCount] (sentinel slots stay zero). + long SumScoreboard(int threadCount) + { + long total = 0; + for (int i = 1; i <= threadCount; i++) + total += Volatile.Read(ref scoreboard[i].Value); + return total; + } + + // ====== Cleanup ====== + + public void Dispose() + { + try { store?.Dispose(); } catch { /* swallow */ } + try { device?.Dispose(); } catch { /* swallow */ } + try + { + // Only nuke our own run dir (identified by its owner sentinel file). + if (!string.IsNullOrEmpty(RunDir) && Directory.Exists(RunDir)) + { + var sentinel = Path.Combine(RunDir, ".kv-benchmark-owner"); + if (File.Exists(sentinel)) + Directory.Delete(RunDir, recursive: true); + } + } + catch { /* best-effort */ } + gate?.Dispose(); + } + } +#pragma warning restore IDE0065 +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvKey.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvKey.cs new file mode 100644 index 00000000000..1f92cd9a48e --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvKey.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ + /// + /// 8-byte key holding a single long value. Stored in a stack/array of 16-byte + /// slots (padding for cache-line alignment of subsequent value bytes inside the + /// log record). Only the first 8 bytes are part of the key for hash + equality. + /// + [StructLayout(LayoutKind.Explicit, Size = DataSize)] + public struct KvKey : IKey + { + internal const int DataSize = 16; + + [FieldOffset(0)] + public long Value; + + [FieldOffset(sizeof(long))] + public int padding1, padding2; + + public override readonly string ToString() => "{ " + Value + " }"; + + public readonly bool IsPinned => false; + + public unsafe ReadOnlySpan KeyBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => new(Unsafe.AsPointer(ref this), sizeof(long)); + } + + public readonly bool HasNamespace => false; + + public readonly ReadOnlySpan NamespaceBytes => []; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvNumaPinning.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvNumaPinning.cs new file mode 100644 index 00000000000..d2e87713499 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvNumaPinning.cs @@ -0,0 +1,196 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ + /// + /// NUMA + thread affinity pinning helper. + /// Linux: reads /sys/devices/system/node/* and uses sched_setaffinity via P/Invoke, + /// intersecting with the process's allowed CPU mask (cgroup-friendly). + /// Windows: uses SetThreadGroupAffinity (delegated through ); + /// best-effort on multi-processor-group hosts. + /// + internal sealed class KvNumaPinning + { + public readonly bool Enabled; + public readonly int NumaNode; + public readonly int[] NodeCpus; // physical-first ordering + public readonly bool IsLinux; + public readonly bool IsWindows; + public readonly string DiagnosticMessage; + + public int FirstUnpinnedCpu => NodeCpus.Length > 0 ? NodeCpus[Math.Min(WorkerCount, NodeCpus.Length - 1)] : -1; + public int WorkerCount; + + public KvNumaPinning(Options opts, int workerCount) + { + WorkerCount = workerCount; + NumaNode = opts.NumaNode; + Enabled = !opts.NoNumaPin; + IsLinux = OperatingSystem.IsLinux(); + IsWindows = OperatingSystem.IsWindows(); + NodeCpus = []; + if (!Enabled) + { + DiagnosticMessage = "NUMA pinning disabled (--no-numa-pin)"; + return; + } + if (IsLinux) + { + try { NodeCpus = DiscoverLinuxNodeCpus(NumaNode); } + catch (Exception ex) { DiagnosticMessage = $"Linux NUMA discovery failed: {ex.Message}"; Enabled = false; return; } + if (NodeCpus.Length == 0) + { + DiagnosticMessage = $"NUMA node {NumaNode} has no CPUs after cpuset intersection"; + Enabled = false; + return; + } + DiagnosticMessage = $"Linux NUMA node {NumaNode}: {NodeCpus.Length} CPUs after cpuset intersection"; + } + else if (IsWindows) + { + try + { + var (_, procsPerGroup) = Native32.GetNumGroupsProcsPerGroup(); + NodeCpus = Enumerable.Range(0, (int)procsPerGroup).ToArray(); + DiagnosticMessage = $"Windows: {NodeCpus.Length} CPUs in current processor group (best-effort NUMA)"; + } + catch (Exception ex) { DiagnosticMessage = $"Windows NUMA discovery failed: {ex.Message}"; Enabled = false; } + } + else + { + DiagnosticMessage = "NUMA pinning not supported on this OS"; + Enabled = false; + } + } + + /// Pin the calling thread to the worker CPU for . + public void PinWorker(int workerIndex) + { + if (!Enabled || NodeCpus.Length == 0) return; + var cpu = NodeCpus[workerIndex % NodeCpus.Length]; + PinToCpu(cpu); + } + + /// Pin the calling thread to the first un-pinned node CPU (setup / reporter). + public void PinSetupOrReporter() + { + if (!Enabled || NodeCpus.Length == 0) return; + var cpu = FirstUnpinnedCpu; + if (cpu >= 0) PinToCpu(cpu); + } + + private void PinToCpu(int cpu) + { + try + { + if (IsLinux) LinuxSchedSetAffinity(cpu); + else if (IsWindows) Native32.AffinitizeThreadRoundRobin((uint)cpu, skipHyperthreads: false); + } + catch { /* best-effort; report via diagnostic if needed */ } + } + + // ====== Linux discovery ====== + + private static int[] DiscoverLinuxNodeCpus(int nodeId) + { + var nodeFile = $"/sys/devices/system/node/node{nodeId}/cpulist"; + if (!File.Exists(nodeFile)) + throw new InvalidOperationException($"NUMA node{nodeId}/cpulist not found"); + var nodeCpus = ParseCpuList(File.ReadAllText(nodeFile).Trim()); + + // Intersect with the process's allowed mask (sched_getaffinity). + var allowed = LinuxSchedGetAffinity(); + var intersected = nodeCpus.Where(c => allowed.Contains(c)).ToList(); + if (intersected.Count == 0) return []; + + // Group by physical core (one CPU per core first, then add siblings). + var visited = new HashSet(); + var result = new List(); + foreach (var cpu in intersected) + { + if (visited.Contains(cpu)) continue; + result.Add(cpu); + visited.Add(cpu); + // Add siblings AFTER all physical cores: collect siblings into a tail list. + var sibFile = $"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list"; + if (!File.Exists(sibFile)) continue; + foreach (var sib in ParseCpuList(File.ReadAllText(sibFile).Trim())) + { + if (sib == cpu) continue; + visited.Add(sib); + } + } + // Append siblings (round 2): visited contains physical+all siblings. The set difference + // of (visited - result) is the sibling-only list. + var siblings = visited.Where(c => !result.Contains(c) && intersected.Contains(c)).OrderBy(c => c).ToList(); + result.AddRange(siblings); + return result.ToArray(); + } + + private static IEnumerable ParseCpuList(string s) + { + // Format: "0-5,8-11,16" or "0,2,4" + foreach (var part in s.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + var dash = part.IndexOf('-'); + if (dash >= 0) + { + if (int.TryParse(part.AsSpan(0, dash), out var lo) && + int.TryParse(part.AsSpan(dash + 1), out var hi)) + { + for (int i = lo; i <= hi; i++) yield return i; + } + } + else if (int.TryParse(part, out var v)) + { + yield return v; + } + } + } + + // ====== Linux libc P/Invoke for sched_get/setaffinity ====== + + private const int CPU_SETSIZE_BITS = 1024; + private const int CPU_SETSIZE_LONGS = CPU_SETSIZE_BITS / 64; + + [DllImport("libc", SetLastError = true)] + private static extern int sched_setaffinity(int pid, IntPtr cpusetsize, ulong[] mask); + + [DllImport("libc", SetLastError = true)] + private static extern int sched_getaffinity(int pid, IntPtr cpusetsize, ulong[] mask); + + private static HashSet LinuxSchedGetAffinity() + { + var mask = new ulong[CPU_SETSIZE_LONGS]; + var rc = sched_getaffinity(0, (IntPtr)(CPU_SETSIZE_LONGS * 8), mask); + var result = new HashSet(); + if (rc != 0) return result; + for (int i = 0; i < CPU_SETSIZE_BITS; i++) + if ((mask[i / 64] & (1UL << (i % 64))) != 0) result.Add(i); + return result; + } + + private static void LinuxSchedSetAffinity(int cpu) + { + var mask = new ulong[CPU_SETSIZE_LONGS]; + mask[cpu / 64] = 1UL << (cpu % 64); + _ = sched_setaffinity(0, (IntPtr)(CPU_SETSIZE_LONGS * 8), mask); + } + + /// Format the applied CPU mask for emission in the metadata block. + public string DescribeWorkerCpus() + { + if (!Enabled || NodeCpus.Length == 0) return ""; + var n = Math.Min(WorkerCount, NodeCpus.Length); + return string.Join(",", NodeCpus.Take(n)); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs new file mode 100644 index 00000000000..79c312de246 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Csv.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace Tsavorite.kvbench +{ + // CSV emit. Wide schema, one row per phase per iteration plus an aggregate row. + // Header is written automatically when the file is first created. + internal sealed partial class KvOutput + { + public void EmitResultCsv(PhaseResult r, KvNumaPinning pinning, int threadCount = 0) + { + if (!_csvEnabled) return; + var path = _opts.CsvOutput; + var fresh = !File.Exists(path); + using var w = new StreamWriter(path, append: true); + if (fresh) w.WriteLine(CsvHeader()); + w.WriteLine(CsvRow(r, pinning, threadCount)); + } + + public void EmitAggregateCsv(IList iters, KvNumaPinning pinning, int threadCount = 0) + { + if (!_csvEnabled || iters == null || iters.Count == 0) return; + var ops = iters.Select(p => p.OpsPerSec).ToArray(); + var mean = ops.Average(); + var stddev = ops.Length > 1 ? Math.Sqrt(ops.Select(o => Math.Pow(o - mean, 2)).Sum() / ops.Length) : 0; + var trimmed = ops.Length >= 3 ? TrimmedMean(ops) : mean; + + var path = _opts.CsvOutput; + var fresh = !File.Exists(path); + using var w = new StreamWriter(path, append: true); + if (fresh) w.WriteLine(CsvHeader()); + + var sb = new StringBuilder(); + CsvAppendCommon(sb, "aggregate", iters.Count, pinning, threadCount); + sb.Append(",") // elapsed + .Append(",") // total_ops_for_throughput + .Append(",") // ops_per_sec + .Append(",") // overshoot + .Append(",") // exit_lag + .Append(",") // reads + .Append(",") // writes + .Append(",") // deletes + .Append(",") // gc0 + .Append(",") // gc1 + .Append(",") // gc2 + .Append(",") // alloc_max + .Append(",") // log_begin + .Append(",") // log_head + .Append(",") // log_readonly + .Append(",") // log_tail + .Append(Dbl(mean)).Append(",") + .Append(Dbl(stddev)).Append(",") + .Append(Dbl(trimmed)).Append(",") + .Append(Dbl(ops.Min())).Append(",") + .Append(Dbl(ops.Max())); + w.WriteLine(sb.ToString()); + } + + static string CsvHeader() => + "schema_version,timestamp_utc,git_sha,hostname,phase,iteration,threads,keys,value_size,distribution,rumd,delete_reinsert,reader_copy_bytes,device,device_throttle,device_completion_threads,device_io_backend,session_context,hashpack_configured,hashpack_effective,index_size_requested,index_size_applied,log_memory,page_size,segment_size,mutable_fraction,warmup_sec,runsec,elapsed_sec,total_ops_for_throughput,ops_per_sec,overshoot_ops,max_worker_exit_lag_ms,reads,writes,deletes,gc_gen0,gc_gen1,gc_gen2,alloc_bytes_by_worker_max,log_begin,log_head,log_readonly,log_tail,agg_mean_ops_per_sec,agg_stdev_ops_per_sec,agg_trimmed_mean,agg_min,agg_max"; + + string CsvRow(PhaseResult r, KvNumaPinning pinning, int threadCount = 0) + { + var sb = new StringBuilder(512); + CsvAppendCommon(sb, r.Phase, r.Iteration, pinning, threadCount); + sb.Append(",").Append(Dbl(r.ElapsedSec)) + .Append(",").Append(r.TotalOpsForThroughput) + .Append(",").Append(Dbl(r.OpsPerSec)) + .Append(",").Append(r.OvershootOps) + .Append(",").Append(Dbl(r.MaxWorkerExitLagMs)) + .Append(",").Append(r.Reads) + .Append(",").Append(r.Writes) + .Append(",").Append(r.Deletes) + .Append(",").Append(r.GcGen0Delta) + .Append(",").Append(r.GcGen1Delta) + .Append(",").Append(r.GcGen2Delta) + .Append(",").Append(r.AllocBytesByWorkerMax) + .Append(",").Append(r.LogBegin) + .Append(",").Append(r.LogHead) + .Append(",").Append(r.LogReadOnly) + .Append(",").Append(r.LogTail) + .Append(",,,,"); // empty aggregate columns + return sb.ToString(); + } + + // threadCount > 0 overrides the default _opts.Threads column (used by the run-sweep so + // each row carries the thread count it was actually measured at). + void CsvAppendCommon(StringBuilder sb, string phase, int iteration, KvNumaPinning pinning, int threadCount = 0) + { + sb.Append(SchemaVersion).Append(",") + .Append(DateTime.UtcNow.ToString("O", CultureInfo.InvariantCulture)).Append(",") + .Append(CsvEsc(TryGitSha())).Append(",") + .Append(CsvEsc(Environment.MachineName)).Append(",") + .Append(phase).Append(",") + .Append(iteration).Append(",") + .Append(threadCount > 0 ? threadCount : _opts.Threads).Append(",") + .Append(_opts.Keys).Append(",") + .Append(_opts.ValueSize).Append(",") + .Append(_opts.Distribution).Append(",") + .Append("\"").Append(string.Join("|", _opts.Rumd)).Append("\"").Append(",") + .Append(_opts.RumdHasDeletes() ? "true" : "false").Append(",") + .Append(KvSessionFunctions.kReaderCopyBytes).Append(",") + .Append(_opts.ResolvedDeviceType).Append(",") + .Append(_opts.DeviceThrottle).Append(",") + .Append(_opts.DeviceCompletionThreads).Append(",") + .Append(_opts.ResolvedIoBackend).Append(",") + .Append("basic").Append(",") + .Append(Dbl(_opts.Hashpack)).Append(",") + .Append(Dbl(EffectiveHashpack())).Append(",") + .Append(_opts.ResolvedIndexRequestedBytes).Append(",") + .Append(_opts.ResolvedIndexAppliedBytes).Append(",") + .Append(_opts.ResolvedLogMemoryBytes).Append(",") + .Append(_opts.ResolvedPageSizeBytes).Append(",") + .Append(_opts.ResolvedSegmentSizeBytes).Append(",") + .Append("0.9").Append(",") + .Append(_opts.WarmupSec).Append(",") + .Append(_opts.RunSec); + } + + static string CsvEsc(string s) + { + if (string.IsNullOrEmpty(s)) return string.Empty; + if (s.Contains(',') || s.Contains('"') || s.Contains('\n')) + return "\"" + s.Replace("\"", "\"\"") + "\""; + return s; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs new file mode 100644 index 00000000000..41ba7636083 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.Json.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; + +namespace Tsavorite.kvbench +{ + // JSON emit + schema. Compact JSON is built into a StringBuilder; pretty form is + // produced via System.Text.Json's indented writer. Stdout is one-line (opt-in), + // file output is pretty. + internal sealed partial class KvOutput + { + const string ResultPrefix = "KV-RESULT-JSON: "; + + public void EmitResultJson(PhaseResult r, KvNumaPinning pinning, int threadCount = 0) + { + if (!_jsonEnabled && !_opts.JsonStdout) return; + var json = BuildResultJson(r, pinning, threadCount); + if (_opts.JsonStdout) Console.WriteLine(ResultPrefix + json); + if (_jsonEnabled) AppendLine(_opts.JsonOutput, PrettyJson(json)); + } + + public void EmitAggregateJson(IList iters, KvNumaPinning pinning, int threadCount = 0) + { + if (iters == null || iters.Count == 0) return; + if (!_jsonEnabled && !_opts.JsonStdout) return; + + var ops = iters.Select(p => p.OpsPerSec).ToArray(); + var mean = ops.Average(); + var stddev = ops.Length > 1 ? Math.Sqrt(ops.Select(o => Math.Pow(o - mean, 2)).Sum() / ops.Length) : 0; + var trimmed = ops.Length >= 3 ? TrimmedMean(ops) : mean; + + var sb = new StringBuilder(1024); + sb.Append('{'); + sb.Append($"\"schema_version\":\"{SchemaVersion}\","); + sb.Append("\"phase\":\"aggregate\","); + if (threadCount > 0) sb.Append($"\"threads\":{threadCount},"); + sb.Append($"\"iterations\":{iters.Count},"); + sb.Append($"\"mean_ops_per_sec\":{Dbl(mean)},"); + sb.Append($"\"stdev_ops_per_sec\":{Dbl(stddev)},"); + sb.Append($"\"stdev_pct\":{Dbl(mean > 0 ? stddev / mean * 100 : 0)},"); + sb.Append($"\"trimmed_mean_ops_per_sec\":{Dbl(trimmed)},"); + sb.Append($"\"min_ops_per_sec\":{Dbl(ops.Min())},"); + sb.Append($"\"max_ops_per_sec\":{Dbl(ops.Max())},"); + sb.Append($"\"timestamp_utc\":\"{DateTime.UtcNow:O}\""); + sb.Append('}'); + var compact = sb.ToString(); + if (_opts.JsonStdout) Console.WriteLine(ResultPrefix + compact); + if (_jsonEnabled) AppendLine(_opts.JsonOutput, PrettyJson(compact)); + } + + string BuildResultJson(PhaseResult r, KvNumaPinning pinning, int threadCount = 0) + { + var sb = new StringBuilder(2048); + sb.Append('{'); + sb.Append($"\"schema_version\":\"{SchemaVersion}\","); + sb.Append($"\"phase\":\"{r.Phase}\","); + sb.Append($"\"iteration\":{r.Iteration},"); + if (threadCount > 0) sb.Append($"\"threads\":{threadCount},"); + sb.Append($"\"ops_per_sec\":{Dbl(r.OpsPerSec)},"); + sb.Append($"\"elapsed_sec\":{Dbl(r.ElapsedSec)},"); + sb.Append($"\"total_ops_for_throughput\":{r.TotalOpsForThroughput},"); + sb.Append($"\"final_total_ops\":{r.FinalTotalOps},"); + sb.Append($"\"overshoot_ops\":{r.OvershootOps},"); + sb.Append($"\"max_worker_exit_lag_ms\":{Dbl(r.MaxWorkerExitLagMs)},"); + sb.Append($"\"reads\":{r.Reads},"); + sb.Append($"\"writes\":{r.Writes},"); + sb.Append($"\"deletes\":{r.Deletes},"); + sb.Append($"\"interrupted\":{(r.Interrupted ? "true" : "false")},"); + sb.Append($"\"error\":{(r.ErrorMessage is null ? "null" : JsonString(r.ErrorMessage))},"); + + sb.Append("\"log\":{"); + sb.Append($"\"begin_address\":{r.LogBegin},"); + sb.Append($"\"head_address\":{r.LogHead},"); + sb.Append($"\"readonly_address\":{r.LogReadOnly},"); + sb.Append($"\"tail_address\":{r.LogTail}"); + sb.Append("},"); + + sb.Append("\"gc_delta\":{"); + sb.Append($"\"gen0\":{r.GcGen0Delta},"); + sb.Append($"\"gen1\":{r.GcGen1Delta},"); + sb.Append($"\"gen2\":{r.GcGen2Delta},"); + sb.Append($"\"alloc_bytes_by_worker_max\":{r.AllocBytesByWorkerMax}"); + sb.Append("},"); + + // Config block — every resolved flag. + sb.Append("\"config\":{"); + sb.Append($"\"threads\":{_opts.Threads},"); + sb.Append($"\"load_threads\":{_opts.ResolvedLoadThreads},"); + sb.Append($"\"run_threads_sweep\":\"{string.Join(",", _opts.ResolvedRunThreadsSweep)}\","); + sb.Append($"\"keys\":{_opts.Keys},"); + sb.Append($"\"value_size\":{_opts.ValueSize},"); + sb.Append($"\"reader_copy_bytes\":{KvSessionFunctions.kReaderCopyBytes},"); + sb.Append($"\"rumd\":\"{string.Join(",", _opts.Rumd)}\","); + sb.Append($"\"delete_reinsert\":{(_opts.RumdHasDeletes() ? "true" : "false")},"); + sb.Append($"\"distribution\":\"{_opts.Distribution}\","); + sb.Append($"\"zipf_theta\":{Dbl(_opts.ZipfTheta)},"); + sb.Append($"\"seed\":{_opts.Seed},"); + sb.Append($"\"hashpack_configured\":{Dbl(_opts.Hashpack)},"); + sb.Append($"\"hashpack_effective\":{Dbl(EffectiveHashpack())},"); + sb.Append($"\"index_size_requested\":{_opts.ResolvedIndexRequestedBytes},"); + sb.Append($"\"index_size_applied\":{_opts.ResolvedIndexAppliedBytes},"); + sb.Append($"\"log_memory\":{_opts.ResolvedLogMemoryBytes},"); + sb.Append($"\"page_size\":{_opts.ResolvedPageSizeBytes},"); + sb.Append($"\"segment_size\":{_opts.ResolvedSegmentSizeBytes},"); + sb.Append($"\"record_size_estimated\":{_opts.ResolvedRecordSizeBytes},"); + sb.Append($"\"max_inline_value_size\":{_opts.ResolvedMaxInlineValueSizeBytes},"); + sb.Append("\"mutable_fraction\":0.9,"); + sb.Append($"\"preallocate_log\":{(_opts.PreallocateLog ? "true" : "false")},"); + sb.Append($"\"device\":\"{_opts.ResolvedDeviceType}\","); + sb.Append($"\"device_throttle\":{_opts.DeviceThrottle},"); + sb.Append($"\"device_completion_threads\":{_opts.DeviceCompletionThreads},"); + sb.Append($"\"device_io_backend\":\"{_opts.ResolvedIoBackend}\","); + sb.Append("\"session_context\":\"basic\","); + sb.Append($"\"warmup_sec\":{_opts.WarmupSec},"); + sb.Append($"\"runsec\":{_opts.RunSec},"); + sb.Append($"\"report_interval_sec\":{_opts.ReportIntervalSec}"); + sb.Append("},"); + + // Host block — hardware / OS / runtime forensic info. + sb.Append("\"host\":{"); + sb.Append($"\"hostname\":{JsonString(Environment.MachineName)},"); + sb.Append($"\"os\":{JsonString(RuntimeInformation.OSDescription)},"); + sb.Append($"\"dotnet\":{JsonString(Environment.Version.ToString())},"); + sb.Append($"\"git_sha\":{JsonString(TryGitSha())},"); + sb.Append($"\"cpu_count\":{Environment.ProcessorCount},"); + sb.Append($"\"pinned_numa_node\":{_opts.NumaNode},"); + sb.Append($"\"worker_cpu_mask\":{JsonString(pinning.DescribeWorkerCpus())},"); + sb.Append($"\"server_gc\":{(System.Runtime.GCSettings.IsServerGC ? "true" : "false")},"); + sb.Append($"\"gc_latency_mode\":\"{System.Runtime.GCSettings.LatencyMode}\","); + sb.Append($"\"tiered_compilation\":{JsonString(Environment.GetEnvironmentVariable("DOTNET_TieredCompilation") ?? "default")},"); + sb.Append($"\"tiered_pgo\":{JsonString(Environment.GetEnvironmentVariable("DOTNET_TieredPGO") ?? "default")},"); + sb.Append($"\"thp_enabled\":{JsonString(TryReadFirstLine("/sys/kernel/mm/transparent_hugepage/enabled"))},"); + sb.Append($"\"data_path\":{JsonString(_dataPath)},"); + sb.Append($"\"ram_total_bytes\":{GC.GetGCMemoryInfo().TotalAvailableMemoryBytes}"); + sb.Append("},"); + + sb.Append($"\"argv\":{_argv},"); + sb.Append($"\"timestamp_utc\":\"{DateTime.UtcNow:O}\""); + sb.Append('}'); + return sb.ToString(); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.cs new file mode 100644 index 00000000000..70255e2c63d --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvOutput.cs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; + +namespace Tsavorite.kvbench +{ + /// + /// Output emitter. Three streams share one schema: + /// + /// Human (stdout) — config block, per-phase one-liner, optional aggregate line, final summary block. + /// JSON — pretty-printed file (--json-output) and/or single-line stdout (--json-stdout). + /// CSV — wide schema, one row per phase + aggregate row (--csv-output). + /// + /// Implementation is split across partial files: KvOutput.cs (human + final summary + shared helpers), + /// KvOutput.Json.cs, KvOutput.Csv.cs. + /// + internal sealed partial class KvOutput + { + const string SchemaVersion = "1"; + + readonly Options _opts; + readonly string _dataPath; + readonly string _argv; + readonly bool _csvEnabled; + readonly bool _jsonEnabled; + + public KvOutput(Options opts, string dataPath, string[] args) + { + _opts = opts; + _dataPath = dataPath; + _argv = "[" + string.Join(",", new[] { "dotnet", "KV.benchmark.dll" }.Concat(args).Select(JsonString)) + "]"; + _csvEnabled = !string.IsNullOrWhiteSpace(opts.CsvOutput); + _jsonEnabled = !string.IsNullOrWhiteSpace(opts.JsonOutput); + } + + // ====== Human-readable (stdout) ====== + + public void EmitConfigHuman(KvNumaPinning pinning) + { + if (_opts.Quiet) return; + Console.WriteLine("=== KV.benchmark config ==="); + Console.WriteLine($" threads : {_opts.Threads} (pinned: {pinning.DescribeWorkerCpus()})"); + Console.WriteLine($" keys : {_opts.Keys:N0}"); + Console.WriteLine($" value-size : {_opts.ValueSize} bytes (reader copies first {KvSessionFunctions.kReaderCopyBytes} B)"); + Console.WriteLine($" rumd% : {string.Join(",", _opts.Rumd)} (deletes auto-reinsert: {(_opts.RumdHasDeletes() ? "yes" : "n/a")})"); + Console.WriteLine($" distribution : {_opts.Distribution}{(_opts.UseZipf ? $" (theta={_opts.ZipfTheta})" : "")}"); + Console.WriteLine($" seed : {_opts.Seed}"); + Console.WriteLine($" iterations : {_opts.Iterations}"); + Console.WriteLine($" warmup / runsec : {_opts.WarmupSec} / {_opts.RunSec} s"); + Console.WriteLine($" device : {_opts.ResolvedDeviceType}"); + Console.WriteLine($" hashpack : {_opts.Hashpack} => index requested {KvSize.FormatSize(_opts.ResolvedIndexRequestedBytes)} → applied {KvSize.FormatSize(_opts.ResolvedIndexAppliedBytes)} (effective hashpack ≈ {EffectiveHashpack():F2})"); + Console.WriteLine($" log-memory : {KvSize.FormatSize(_opts.ResolvedLogMemoryBytes)}"); + Console.WriteLine($" page-size : {KvSize.FormatSize(_opts.ResolvedPageSizeBytes)}"); + Console.WriteLine($" segment-size : {KvSize.FormatSize(_opts.ResolvedSegmentSizeBytes)}"); + Console.WriteLine($" max-inline-value : {KvSize.FormatSize(_opts.ResolvedMaxInlineValueSizeBytes)} (values larger overflow to heap)"); + Console.WriteLine($" preallocate-log : {_opts.PreallocateLog}"); + Console.WriteLine($" record-size (est): {_opts.ResolvedRecordSizeBytes} B"); + Console.WriteLine($" data-path : {_dataPath}"); + Console.WriteLine($" report-interval : {(_opts.ReportIntervalSec <= 0 ? "off (reference mode)" : _opts.ReportIntervalSec + "s")}"); + Console.WriteLine($" NUMA : {pinning.DiagnosticMessage}"); + Console.WriteLine("==========================="); + } + + public void EmitPhaseHuman(PhaseResult r, int threadCount = 0) + { + if (_opts.Quiet) return; + var tag = r.Phase switch + { + "load" => "[load] ", + "warmup" => "[warmup] ", + "run" => threadCount > 0 ? $"[run t={threadCount} i={r.Iteration}] " : $"[run {r.Iteration}] ", + _ => $"[{r.Phase}] ", + }; + var rate = r.OpsPerSec.ToString("N0", CultureInfo.InvariantCulture); + Console.WriteLine($"{tag}{r.TotalOpsForThroughput:N0} ops in {r.ElapsedSec:N3} s ({rate} ops/sec) reads={r.Reads:N0} writes={r.Writes:N0} deletes={r.Deletes:N0} overshoot={r.OvershootOps:N0} exit-lag={r.MaxWorkerExitLagMs:N1}ms gc={r.GcGen0Delta}/{r.GcGen1Delta}/{r.GcGen2Delta} alloc/wkr={r.AllocBytesByWorkerMax}B"); + } + + public void EmitAggregateHuman(IList iters, int threadCount = 0) + { + if (_opts.Quiet) return; + var ops = iters.Select(p => p.OpsPerSec).ToArray(); + var mean = ops.Average(); + var stddev = ops.Length > 1 ? Math.Sqrt(ops.Select(o => Math.Pow(o - mean, 2)).Sum() / ops.Length) : 0; + var pct = mean > 0 ? stddev / mean * 100 : 0; + var prefix = threadCount > 0 ? $"[aggregate t={threadCount}]" : "[aggregate]"; + Console.WriteLine($"{prefix} iterations={iters.Count} mean={mean:N0} ops/sec stdev={stddev:N1} ({pct:N1}%) min={ops.Min():N0} max={ops.Max():N0}{(iters.Count >= 3 ? $" trimmed={TrimmedMean(ops):N0}" : "")}"); + } + + /// + /// Prints a single readable block summarising config, load, and run perf at end of run. + /// When the run-thread sweep has multiple entries, the run phase is reported as a table + /// with one row per thread count (with speedup vs the smallest sweep entry). + /// Always prints (regardless of --quiet) since this is the headline output. + /// + public void EmitFinalSummary(PhaseResult loadResult, IDictionary> sweepResults, KvNumaPinning pinning) + { + static string Rate(double opsPerSec) => + opsPerSec >= 1e9 ? $"{opsPerSec / 1e9:F2} G ops/sec" + : opsPerSec >= 1e6 ? $"{opsPerSec / 1e6:F2} M ops/sec" + : opsPerSec >= 1e3 ? $"{opsPerSec / 1e3:F1} K ops/sec" + : $"{opsPerSec:F0} ops/sec"; + + const string sep = "=============================================================================="; + Console.WriteLine(); + Console.WriteLine(sep); + Console.WriteLine(" KV.benchmark — final summary"); + Console.WriteLine(sep); + + var dist = _opts.UseZipf ? $"zipf θ={_opts.ZipfTheta:F2}" : "uniform"; + Console.WriteLine($" workload : {_opts.Keys:N0} keys × {_opts.ValueSize}B value, rumd={string.Join(",", _opts.Rumd)}, {dist}"); + var runThreadsStr = _opts.ResolvedRunThreadsSweep.Length == 1 + ? _opts.ResolvedRunThreadsSweep[0].ToString(CultureInfo.InvariantCulture) + : string.Join(",", _opts.ResolvedRunThreadsSweep); + Console.WriteLine($" parallelism : load-threads={_opts.ResolvedLoadThreads}, run-threads={runThreadsStr}, pinned={pinning.DescribeWorkerCpus()} (NUMA node {_opts.NumaNode})"); + Console.WriteLine($" storage : hashpack={_opts.Hashpack:F2} → index {KvSize.FormatSize(_opts.ResolvedIndexAppliedBytes)} (effective {EffectiveHashpack():F2})"); + Console.WriteLine($" log={KvSize.FormatSize(_opts.ResolvedLogMemoryBytes)} (pages {KvSize.FormatSize(_opts.ResolvedPageSizeBytes)}, segments {KvSize.FormatSize(_opts.ResolvedSegmentSizeBytes)}, record ≈{_opts.ResolvedRecordSizeBytes}B)"); + Console.WriteLine($" device={_opts.ResolvedDeviceType}, session=BasicContext (safe path)"); + Console.WriteLine($" timing : warmup={_opts.WarmupSec}s, run={_opts.RunSec}s × {_opts.Iterations} iter(s)"); + + if (loadResult != null) + { + Console.WriteLine(); + Console.WriteLine($" Load phase ({_opts.ResolvedLoadThreads} thread{(_opts.ResolvedLoadThreads == 1 ? "" : "s")}):"); + Console.WriteLine($" {loadResult.TotalOpsForThroughput:N0} ops in {loadResult.ElapsedSec:F3} s → {Rate(loadResult.OpsPerSec)}"); + Console.WriteLine($" log tail = {loadResult.LogTail:N0} bytes"); + } + + if (sweepResults != null && sweepResults.Count > 0) + { + Console.WriteLine(); + if (sweepResults.Count == 1) + { + // Single thread count: show the same detail as before. + var (tc, iters) = (sweepResults.Keys.First(), sweepResults.Values.First()); + EmitRunPhaseBlock(tc, iters, Rate); + } + else + { + // Sweep: print a compact table with speedup vs the smallest entry. + Console.WriteLine($" Run sweep ({_opts.Iterations} iteration{(_opts.Iterations == 1 ? "" : "s")} per thread count):"); + Console.WriteLine($" {"threads",7} | {"trimmed",14} | {"mean",14} | {"stdev%",7} | speedup"); + Console.WriteLine($" {new string('-', 7)}-+-{new string('-', 14)}-+-{new string('-', 14)}-+-{new string('-', 7)}-+--------"); + var ordered = sweepResults.OrderBy(kv => kv.Key).ToList(); + double basis = 0; + foreach (var (tc, iters) in ordered) + { + var ops = iters.Select(p => p.OpsPerSec).ToArray(); + var mean = ops.Average(); + var sd = ops.Length > 1 ? Math.Sqrt(ops.Select(o => Math.Pow(o - mean, 2)).Sum() / ops.Length) : 0; + var pct = mean > 0 ? sd / mean * 100 : 0; + var trimmed = ops.Length >= 3 ? TrimmedMean(ops) : mean; + if (basis == 0) basis = trimmed; + var speedup = basis > 0 ? trimmed / basis : 0; + Console.WriteLine($" {tc,7} | {Rate(trimmed),14} | {Rate(mean),14} | {pct,6:F1}% | {speedup,6:F2}×"); + } + } + } + Console.WriteLine(sep); + } + + void EmitRunPhaseBlock(int threadCount, IList iters, Func rate) + { + var ops = iters.Select(p => p.OpsPerSec).ToArray(); + var mean = ops.Average(); + var stddev = ops.Length > 1 ? Math.Sqrt(ops.Select(o => Math.Pow(o - mean, 2)).Sum() / ops.Length) : 0; + var pct = mean > 0 ? stddev / mean * 100 : 0; + var trimmed = ops.Length >= 3 ? TrimmedMean(ops) : mean; + + Console.WriteLine($" Run phase ({iters.Count} iteration{(iters.Count == 1 ? "" : "s")}, {threadCount} thread{(threadCount == 1 ? "" : "s")}):"); + Console.WriteLine($" mean : {rate(mean)} ± {pct:F1}% (stdev {stddev:N0} ops/sec)"); + if (ops.Length >= 3) + Console.WriteLine($" trimmed : {rate(trimmed)} (drops hi+lo)"); + Console.WriteLine($" min..max : {rate(ops.Min())} .. {rate(ops.Max())}"); + Console.Write(" per-iter : "); + for (int i = 0; i < ops.Length; i++) + { + if (i > 0) Console.Write(", "); + Console.Write($"{ops[i] / 1e6:F2}M"); + } + Console.WriteLine(); + } + + // ====== Helpers shared across human / JSON / CSV ====== + + /// Live keys / applied hash buckets — how many keys are mapped per hash bucket. + double EffectiveHashpack() + { + var buckets = _opts.ResolvedIndexAppliedBytes / 64; + return buckets > 0 ? (double)_opts.Keys / buckets : 0.0; + } + + /// Trims hi+lo and returns the mean. Caller must pass an array with length >= 3. + static double TrimmedMean(double[] arr) + { + var sorted = arr.OrderBy(x => x).ToArray(); + if (sorted.Length < 3) return sorted.Average(); + return sorted.Skip(1).Take(sorted.Length - 2).Average(); + } + + static string Dbl(double d) => d.ToString("R", CultureInfo.InvariantCulture); + + static string JsonString(string s) + { + if (s is null) return "null"; + var sb = new StringBuilder(s.Length + 2); + sb.Append('"'); + foreach (var c in s) + { + switch (c) + { + case '"': sb.Append("\\\""); break; + case '\\': sb.Append("\\\\"); break; + case '\n': sb.Append("\\n"); break; + case '\r': sb.Append("\\r"); break; + case '\t': sb.Append("\\t"); break; + default: + if (c < 0x20) sb.Append($"\\u{(int)c:X4}"); + else sb.Append(c); + break; + } + } + sb.Append('"'); + return sb.ToString(); + } + + /// Pretty-prints a compact JSON string. On parse failure, returns the input unchanged. + static string PrettyJson(string compactJson) + { + try + { + using var doc = JsonDocument.Parse(compactJson); + using var ms = new MemoryStream(); + using (var writer = new Utf8JsonWriter(ms, new JsonWriterOptions { Indented = true })) + { + doc.WriteTo(writer); + } + return Encoding.UTF8.GetString(ms.ToArray()); + } + catch + { + return compactJson; + } + } + + static string TryReadFirstLine(string path) + { + try { return File.Exists(path) ? File.ReadAllText(path).Trim() : ""; } + catch { return ""; } + } + + static string TryGitSha() + { + try + { + var psi = new System.Diagnostics.ProcessStartInfo("git", "rev-parse --short HEAD") + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + using var p = System.Diagnostics.Process.Start(psi); + if (p == null) return ""; + var output = p.StandardOutput.ReadToEnd().Trim(); + p.WaitForExit(500); + return output; + } + catch { return ""; } + } + + static void AppendLine(string path, string line) + { + try + { + using var w = new StreamWriter(path, append: true); + w.WriteLine(line); + } + catch { /* best-effort */ } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSessionFunctions.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSessionFunctions.cs new file mode 100644 index 00000000000..7bfc83258d5 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSessionFunctions.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Tsavorite.kvbench +{ + /// + /// Session functions for KV.benchmark. + /// - copies the first bytes + /// (one cache line) of the value into the output buffer — isolates engine + /// overhead from memcpy bandwidth for interpretable read throughput. + /// - is implemented (not throwing) so a concurrent + /// RMW that lands on a key during the brief delete-reinsert gap succeeds. + /// + public sealed class KvSessionFunctions : SpanByteFunctions + { + // Hard-coded constant so JIT can const-fold the Slice/CopyTo into a single + // 32-byte SSE memcpy. Reading this from a static-readonly field defeats the + // const-fold and produces measurably slower codegen. + internal const int kReaderCopyBytes = 32; + + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref ReadInfo readInfo) + { + // Constant 32-byte copy — value length must be >= 32 (--value-size validation enforces this). + srcLogRecord.ValueSpan.Slice(0, 32).CopyTo(output.SpanByte.Span); + return true; + } + + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + srcValue.CopyTo(logRecord.ValueSpan); + return true; + } + + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + srcValue.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + input.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + input.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + input.CopyTo(logRecord.ValueSpan); + return true; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSize.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSize.cs new file mode 100644 index 00000000000..21c6e93f848 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KvSize.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Globalization; + +namespace Tsavorite.kvbench +{ + /// + /// Size string parsing/formatting helpers ("4mb", "16GB", "1024" → bytes). + /// + internal static class KvSize + { + /// + /// Parse a size string like "4mb", "64g", "256m", "1024" into bytes. + /// Returns -1 on parse failure. Suffix is case-insensitive. + /// + public static long ParseSize(string s) + { + if (string.IsNullOrWhiteSpace(s)) + return -1; + var span = s.Trim().ToLowerInvariant(); + long mult = 1; + int suffixLen = 0; + if (span.EndsWith("tb")) { mult = 1L << 40; suffixLen = 2; } + else if (span.EndsWith("gb")) { mult = 1L << 30; suffixLen = 2; } + else if (span.EndsWith("mb")) { mult = 1L << 20; suffixLen = 2; } + else if (span.EndsWith("kb")) { mult = 1L << 10; suffixLen = 2; } + else if (span.EndsWith('t')) { mult = 1L << 40; suffixLen = 1; } + else if (span.EndsWith('g')) { mult = 1L << 30; suffixLen = 1; } + else if (span.EndsWith('m')) { mult = 1L << 20; suffixLen = 1; } + else if (span.EndsWith('k')) { mult = 1L << 10; suffixLen = 1; } + var numeric = span.Substring(0, span.Length - suffixLen); + if (!double.TryParse(numeric, NumberStyles.Float, CultureInfo.InvariantCulture, out var raw) || raw < 0) + return -1; + return (long)(raw * mult); + } + + /// Format bytes as "64MB" / "32GB" / etc. + public static string FormatSize(long bytes) + { + if (bytes <= 0) return bytes.ToString(CultureInfo.InvariantCulture); + if ((bytes & ((1L << 40) - 1)) == 0) return (bytes >> 40).ToString(CultureInfo.InvariantCulture) + "TB"; + if ((bytes & ((1L << 30) - 1)) == 0) return (bytes >> 30).ToString(CultureInfo.InvariantCulture) + "GB"; + if ((bytes & ((1L << 20) - 1)) == 0) return (bytes >> 20).ToString(CultureInfo.InvariantCulture) + "MB"; + if ((bytes & ((1L << 10) - 1)) == 0) return (bytes >> 10).ToString(CultureInfo.InvariantCulture) + "KB"; + return bytes.ToString(CultureInfo.InvariantCulture) + "B"; + } + + /// Round n up to the next power of two. Returns n if it is already a power of two. + public static long NextPow2(long n) + { + if (n <= 1) return 1; + long v = n - 1; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + return v + 1; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs new file mode 100644 index 00000000000..2cb2083da38 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Options.cs @@ -0,0 +1,362 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using CommandLine; + +namespace Tsavorite.kvbench +{ + /// + /// CLI options for KV.benchmark. + /// + public class Options + { + // ===== Workload ===== + + [Option('t', "threads", Required = false, Default = 1, + HelpText = "Default run-phase worker thread count (also used for load if --load-threads is unspecified). Pass nodeCpus to saturate the pinned NUMA node.")] + public int Threads { get; set; } + + [Option("load-threads", Required = false, Default = 0, + HelpText = "Threads to use for the load phase. 0 = same as --threads. Useful when you want a fast parallel load followed by a single-thread or sweep run.")] + public int LoadThreads { get; set; } + + [Option("run-threads-sweep", Separator = ',', Required = false, Default = null, + HelpText = "Comma-separated list of run-phase thread counts. When specified, the engine loads ONCE and then runs the full --iterations sweep for each thread count (1,2,4,8,16). Overrides --threads for the run phase.")] + public IEnumerable RunThreadsSweep { get; set; } + + [Option('n', "keys", Required = false, Default = 100_000_000L, + HelpText = "Number of unique keys in the dataset.")] + public long Keys { get; set; } + + [Option('v', "value-size", Required = false, Default = 100, + HelpText = "Value length in bytes. Range: 32..1048576 (must also be <= --max-inline-value-size).")] + public int ValueSize { get; set; } + + [Option("rumd", Separator = ',', Required = false, Default = new[] { 100, 0, 0, 0 }, + HelpText = "#,#,#,#: Percentages of [(r)eads,(u)pserts,r(m)ws,(d)eletes] (summing to 100). When d% > 0, deletes auto-reinsert.")] + public IEnumerable Rumd { get; set; } + + [Option('d', "distribution", Required = false, Default = "uniform", + HelpText = "Key distribution: 'uniform' or 'zipf'.")] + public string Distribution { get; set; } + + [Option("zipf-theta", Required = false, Default = 0.99, + HelpText = "Zipf skew parameter (only used when distribution=zipf).")] + public double ZipfTheta { get; set; } + + [Option("runsec", Required = false, Default = 30, + HelpText = "Run-phase duration in seconds (excludes warmup).")] + public int RunSec { get; set; } + + [Option("warmup-sec", Required = false, Default = 5, + HelpText = "Warmup duration in seconds, discarded from results. 0 disables warmup.")] + public int WarmupSec { get; set; } + + // ===== Reproducibility ===== + + [Option('s', "seed", Required = false, Default = 211UL, + HelpText = "Base RNG seed. Per-thread seeds are derived via SplitMix64(seed, threadIdx).")] + public ulong Seed { get; set; } + + [Option('i', "iterations", Required = false, Default = 1, + HelpText = "Run-phase iterations (load runs once; warmup runs once per iter).")] + public int Iterations { get; set; } + + // ===== Sizing ===== + + [Option("hashpack", Required = false, Default = 2.0, + HelpText = "Hash packing factor (keys per bucket request, before KVSettings round-down).")] + public double Hashpack { get; set; } + + [Option("log-memory", Required = false, Default = null, + HelpText = "Total in-memory log size (e.g. 16GB). Auto-default sizes for whole dataset in mutable region (read-only baseline).")] + public string LogMemory { get; set; } + + [Option("page-size", Required = false, Default = "16MB", + HelpText = "Page size (e.g. 16MB, 32MB). Default matches Garnet (defaults.conf PageSize=16m).")] + public string PageSize { get; set; } + + [Option("segment-size", Required = false, Default = "1GB", + HelpText = "On-disk segment size (e.g. 1GB). Default matches Garnet (defaults.conf SegmentSize=1g).")] + public string SegmentSize { get; set; } + + [Option("max-inline-value-size", Required = false, Default = "16KB", + HelpText = "Max inline value size (KVSettings.MaxInlineValueSize). Values larger than this overflow to a separate heap object. Default matches Garnet (defaults.conf ValueOverflowThreshold=16k).")] + public string MaxInlineValueSize { get; set; } + + [Option("preallocate-log", Required = false, Default = false, + HelpText = "Pre-touch every log page at startup to commit physical pages. Default matches Garnet (false). Enable for stable single-thread benchmarks where first-touch page faults would bias the timed window.")] + public bool PreallocateLog { get; set; } + + // ===== Device ===== + + [Option("device", Required = false, Default = "default", + HelpText = "Device backend: native, randomaccess, filestream, null, default.")] + public string Device { get; set; } + + [Option("device-throttle", Required = false, Default = 0, + HelpText = "Max in-flight IOs. 0 = device default (120 for every Tsavorite device).")] + public int DeviceThrottle { get; set; } + + [Option("device-io-backend", Required = false, Default = "default", + HelpText = "Linux native backend: libaio, default (=libaio).")] + public string DeviceIoBackend { get; set; } + + [Option("device-completion-threads", Required = false, Default = 0, + HelpText = "Native completion thread count. 0 = Garnet default (1).")] + public int DeviceCompletionThreads { get; set; } + + [Option("data-path", Required = false, Default = null, + HelpText = "Directory where hlog files live. Default OS temp.")] + public string DataPath { get; set; } + + // ===== Host tuning ===== + + [Option("no-numa-pin", Required = false, Default = false, + HelpText = "Disable in-process NUMA pinning.")] + public bool NoNumaPin { get; set; } + + [Option("numa-node", Required = false, Default = 0, + HelpText = "Which NUMA node to pin to.")] + public int NumaNode { get; set; } + + [Option("no-threadpool-tune", Required = false, Default = false, + HelpText = "Disable auto ThreadPool.SetMinThreads(max(t*2, 256)).")] + public bool NoThreadPoolTune { get; set; } + + // ===== Validation ===== + + [Option("validate", Required = false, Default = false, + HelpText = "After load: single-threaded readback of every key. Aborts on mismatch.")] + public bool Validate { get; set; } + + // ===== Output ===== + + [Option("report-interval-sec", Required = false, Default = 1, + HelpText = "Live throughput reporter tick (seconds). 0 disables — recommended for canonical numbers.")] + public int ReportIntervalSec { get; set; } + + [Option("json-output", Required = false, Default = null, + HelpText = "Append pretty-printed JSON summary rows to this file (one row per phase).")] + public string JsonOutput { get; set; } + + [Option("json-stdout", Required = false, Default = false, + HelpText = "Also emit single-line `KV-RESULT-JSON: {...}` blobs to stdout for log scraping. Off by default.")] + public bool JsonStdout { get; set; } + + [Option("csv-output", Required = false, Default = null, + HelpText = "Append CSV summary rows to this file.")] + public string CsvOutput { get; set; } + + [Option("quiet", Required = false, Default = false, + HelpText = "Suppress human-readable progress/config (final results still print).")] + public bool Quiet { get; set; } + + // ===== Resolved values (filled in after parsing) ===== + + internal long ResolvedPageSizeBytes; + internal long ResolvedSegmentSizeBytes; + internal long ResolvedLogMemoryBytes; + internal long ResolvedIndexRequestedBytes; + internal long ResolvedIndexAppliedBytes; + internal long ResolvedRecordSizeBytes; + internal long ResolvedMaxInlineValueSizeBytes; + internal int ReadPct, UpsertPctCumulative, RmwPctCumulative; + internal bool UseZipf; + internal Tsavorite.core.DeviceType ResolvedDeviceType; + internal Tsavorite.core.NativeStorageDevice.IoBackend ResolvedIoBackend; + + /// Thread count used for the load phase (load-threads if specified, else threads). + internal int ResolvedLoadThreads; + /// Thread counts for the run phase: either the sweep list (if --run-threads-sweep was set) or [Threads]. + internal int[] ResolvedRunThreadsSweep; + /// Maximum worker count across all phases — used to size the scoreboard. + internal int ResolvedMaxThreads; + + /// + /// Validate inputs and resolve all auto-defaults. Returns null on success or an error message. + /// + internal string Resolve() + { + if (Threads < 1) return "--threads must be >= 1"; + if (LoadThreads < 0) return "--load-threads must be >= 0 (0 = same as --threads)"; + ResolvedLoadThreads = LoadThreads > 0 ? LoadThreads : Threads; + + var sweep = RunThreadsSweep?.ToArray() ?? []; + if (sweep.Length > 0) + { + if (sweep.Any(t => t < 1)) return "--run-threads-sweep entries must be >= 1"; + ResolvedRunThreadsSweep = sweep; + } + else + { + ResolvedRunThreadsSweep = [Threads]; + } + ResolvedMaxThreads = Math.Max(ResolvedLoadThreads, ResolvedRunThreadsSweep.Max()); + + if (Keys <= 0) return "--keys must be > 0"; + // Validate --value-size: lower bound 32 (Reader copies 32 bytes), upper bound 1MB + // (validated against --max-inline-value-size below for the per-record cap). + if (ValueSize < 32 || ValueSize > 1024 * 1024) return "--value-size must be in [32, 1048576]"; + if (Hashpack <= 0) return "--hashpack must be > 0"; + if (RunSec < 0) return "--runsec must be >= 0"; + if (WarmupSec < 0) return "--warmup-sec must be >= 0"; + if (Iterations < 1) return "--iterations must be >= 1"; + if (ReportIntervalSec < 0) return "--report-interval-sec must be >= 0"; + + var dist = (Distribution ?? "uniform").ToLowerInvariant(); + if (dist != "uniform" && dist != "zipf") return "--distribution must be 'uniform' or 'zipf'"; + Distribution = dist; + UseZipf = dist == "zipf"; + if (UseZipf) + { + // ZipfConstants computes Alpha = 1/(1-theta); theta in [0,1) gives valid alpha. + // theta == 1 divides by zero; theta < 0 or > 1 produces NaN / negative samples. + if (!(ZipfTheta >= 0 && ZipfTheta < 1)) + return $"--zipf-theta must be in [0, 1); got {ZipfTheta}"; + } + + var rumd = Rumd?.ToArray() ?? [100, 0, 0, 0]; + if (rumd.Length != 4) return "--rumd must be 4 numbers"; + if (rumd.Any(x => x < 0)) return "--rumd entries must be >= 0"; + if (rumd.Sum() != 100) return $"--rumd must sum to 100 (got {rumd.Sum()})"; + Rumd = rumd; + ReadPct = rumd[0]; + UpsertPctCumulative = ReadPct + rumd[1]; + RmwPctCumulative = UpsertPctCumulative + rumd[2]; + + ResolvedDeviceType = ParseDeviceType(Device); + if (ResolvedDeviceType == Tsavorite.core.DeviceType.Default && !IsKnownDeviceName(Device)) + return $"--device must be one of: native, randomaccess, filestream, null, default (got: {Device})"; + ResolvedIoBackend = ParseIoBackend(DeviceIoBackend); + if (ResolvedIoBackend == Tsavorite.core.NativeStorageDevice.IoBackend.Default && !IsKnownIoBackendName(DeviceIoBackend)) + return $"--device-io-backend must be one of: libaio, default (got: {DeviceIoBackend})"; + + ResolvedPageSizeBytes = KvSize.ParseSize(PageSize); + if (ResolvedPageSizeBytes <= 0) return $"--page-size invalid: {PageSize}"; + ResolvedSegmentSizeBytes = KvSize.ParseSize(SegmentSize); + if (ResolvedSegmentSizeBytes <= 0) return $"--segment-size invalid: {SegmentSize}"; + ResolvedMaxInlineValueSizeBytes = KvSize.ParseSize(MaxInlineValueSize); + if (ResolvedMaxInlineValueSizeBytes <= 0) return $"--max-inline-value-size invalid: {MaxInlineValueSize}"; + if (ValueSize > ResolvedMaxInlineValueSizeBytes) + return $"--value-size ({ValueSize}) exceeds --max-inline-value-size ({ResolvedMaxInlineValueSizeBytes}); values larger than the inline threshold overflow to heap and skew the benchmark."; + + // Estimated record size: 8 RecordInfo + 5 length-byte hdr + 8 key + value, aligned to 8. + var rec = 21L + ValueSize; + ResolvedRecordSizeBytes = (rec + 7) & ~7L; + + // --log-memory auto-default: NextPow2(ceil(keys * record / 0.9)), floored at 2 * page-size. + if (!string.IsNullOrWhiteSpace(LogMemory)) + { + ResolvedLogMemoryBytes = KvSize.ParseSize(LogMemory); + if (ResolvedLogMemoryBytes <= 0) return $"--log-memory invalid: {LogMemory}"; + } + else + { + var dbBytes = Keys * ResolvedRecordSizeBytes; + var target = (long)Math.Ceiling(dbBytes / 0.9); + var auto = KvSize.NextPow2(target); + var floor = 2 * ResolvedPageSizeBytes; + if (auto < floor) auto = floor; + ResolvedLogMemoryBytes = ClampToRam(auto); + } + + // --hashpack -> index_size_requested: (long)(keys / hashpack) << 6. KVSettings rounds DOWN + // to power of 2 — we track both requested and applied here. + ResolvedIndexRequestedBytes = (long)(Keys / Hashpack) << 6; + if (ResolvedIndexRequestedBytes < 64) ResolvedIndexRequestedBytes = 64; + ResolvedIndexAppliedBytes = PreviousPow2(ResolvedIndexRequestedBytes); + + return null; + } + + internal long ClampToRam(long autoLogMemory) + { + // Only auto-derived log-memory is clamped; explicit user values pass through. + try + { + var available = TryGetAvailableRamBytes(); + if (available <= 0) return autoLogMemory; + var cap = (long)(available * 0.7) - ResolvedIndexAppliedBytes; // leave index room + if (cap <= 0) return autoLogMemory; // give up; user will see OOM + var result = autoLogMemory; + while (result > cap && result > (2 * ResolvedPageSizeBytes)) + result /= 2; + return result; + } + catch + { + return autoLogMemory; + } + } + + private static long TryGetAvailableRamBytes() + { + if (OperatingSystem.IsLinux()) + { + try + { + foreach (var line in System.IO.File.ReadAllLines("/proc/meminfo")) + { + if (line.StartsWith("MemAvailable:", StringComparison.Ordinal)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && long.TryParse(parts[1], out var kb)) + return kb * 1024L; + } + } + } + catch { /* fall through */ } + } + return GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + } + + private static long PreviousPow2(long n) + { + if (n <= 1) return 1; + long p = 1; + while ((p << 1) > 0 && (p << 1) <= n) p <<= 1; + return p; + } + + internal static Tsavorite.core.DeviceType ParseDeviceType(string s) + { + if (string.IsNullOrWhiteSpace(s)) return Tsavorite.core.DeviceType.Default; + return s.ToLowerInvariant() switch + { + "native" => Tsavorite.core.DeviceType.Native, + "randomaccess" => Tsavorite.core.DeviceType.RandomAccess, + "filestream" => Tsavorite.core.DeviceType.FileStream, + "null" => Tsavorite.core.DeviceType.Null, + "default" => Tsavorite.core.DeviceType.Default, + _ => Tsavorite.core.DeviceType.Default, + }; + } + + internal static Tsavorite.core.NativeStorageDevice.IoBackend ParseIoBackend(string s) + { + if (string.IsNullOrWhiteSpace(s)) return Tsavorite.core.NativeStorageDevice.IoBackend.Default; + return s.ToLowerInvariant() switch + { + "default" => Tsavorite.core.NativeStorageDevice.IoBackend.Default, + "libaio" => Tsavorite.core.NativeStorageDevice.IoBackend.Libaio, + _ => Tsavorite.core.NativeStorageDevice.IoBackend.Default, + }; + } + + static bool IsKnownDeviceName(string s) + { + if (string.IsNullOrWhiteSpace(s)) return true; + return s.ToLowerInvariant() is "native" or "randomaccess" or "filestream" or "null" or "default"; + } + + static bool IsKnownIoBackendName(string s) + { + if (string.IsNullOrWhiteSpace(s)) return true; + return s.ToLowerInvariant() is "default" or "libaio"; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/PaddedTypes.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/PaddedTypes.cs new file mode 100644 index 00000000000..d2254d03674 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/PaddedTypes.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.InteropServices; + +namespace Tsavorite.kvbench +{ + /// + /// 128-byte padded long. Two cache-line stride per slot fully isolates each + /// counter from its neighbours, including against L2 spatial prefetch. + /// + [StructLayout(LayoutKind.Explicit, Size = 128)] + public struct PaddedLong + { + [FieldOffset(0)] public long Value; + } + + /// + /// 128-byte padded bool. Lives on its own field to prevent cross-line + /// prefetch aliasing with worker scoreboard counters. + /// + [StructLayout(LayoutKind.Explicit, Size = 128)] + public struct PaddedBool + { + [FieldOffset(0)] public bool Value; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Program.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Program.cs new file mode 100644 index 00000000000..66d084e493a --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/Program.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.kvbench +{ + public static class Program + { + public static int Main(string[] args) + { + return EntryPoint.Run(args); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md new file mode 100644 index 00000000000..e1d168fe1d3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/README.md @@ -0,0 +1,491 @@ +# KV.benchmark + +A lean throughput benchmark for the Tsavorite key-value store. Measures load +(insert) and run (RUMD = reads / upserts / RMWs / deletes) throughput on a +synthetic 8-byte-key + fixed-length-value dataset, using ObjectAllocator and +the safe `BasicContext` session path. + +Designed to **reflect underlying Tsavorite engine performance** without +benchmark-side noise: zero per-op allocations on the hot path, no shared +counters on the inner loop, NUMA-pinned worker threads, scoreboard layout that +prevents false sharing across worker cores, central tick timing that excludes +worker-join lag from the measured duration, and rich per-iteration metadata in +human-readable, JSON, and CSV streams. + +## Quick start + +```bash +# Build +dotnet build libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KV.benchmark.csproj \ + -c Release -f net10.0 + +# Minimal in-memory smoke test (1 thread, null device, 5 s read run) +dotnet libs/storage/Tsavorite/cs/benchmark/KV.benchmark/bin/Release/net10.0/KV.benchmark.dll \ + -t 1 -n 1000000 -v 100 --device null --rumd 100,0,0,0 --runsec 5 +``` + +```powershell +# Windows (PowerShell) +dotnet libs\storage\Tsavorite\cs\benchmark\KV.benchmark\bin\Release\net10.0\KV.benchmark.dll ` + -t 1 -n 1000000 -v 100 --device null --rumd 100,0,0,0 --runsec 5 +``` + +Expected output (excerpt): +``` +=== KV.benchmark config === + threads : 1 (pinned: 0) + keys : 1,000,000 + value-size : 100 bytes (reader copies first 32 B) + rumd% : 100,0,0,0 (deletes auto-reinsert: n/a) + ... +=========================== +[load] 1,000,000 ops in 0.5 s (~2,000,000 ops/sec) ... +[run 1] ~5,000,000 ops in 5.000 s (~1,000,000 ops/sec) ... +[aggregate] iterations=1 mean=... stdev=0.0 (0.0%) ... +``` + +## Building + +```bash +dotnet build libs/storage/Tsavorite/cs/benchmark/KV.benchmark/KV.benchmark.csproj -c Release -f net10.0 +``` + +Performance numbers should always be taken from a **Release** build on +**net10.0**. The project uses Workstation GC by default (the .NET runtime +default). Set `DOTNET_gcServer=1` in the environment when running with high +thread counts to switch to Server GC, which scales better past ~8 threads. +Debug builds are fine for correctness checks but not for measurement. + +## Worked examples + +### Cookbook (copy-paste matrix: distribution × dataset size × log size) + +All examples assume the binary path is in `$KV` (set with +`KV=libs/storage/Tsavorite/cs/benchmark/KV.benchmark/bin/Release/net10.0/KV.benchmark.dll`) +and use 32-byte aligned NUMA pinning. Pick the row matching your scenario: + +| Dataset | Log fit | Distribution | Command | +| --- | --- | --- | --- | +| small (4.6 M × 100 B ≈ 580 MB) | **fits in mutable log** (auto-sized) | uniform | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 16 -n 4600000 -v 100 --device null --rumd 100,0,0,0 --runsec 15 --warmup-sec 5 -i 3` | +| small (4.6 M × 100 B) | fits | **zipf θ=0.99** | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 16 -n 4600000 -v 100 --device null --rumd 100,0,0,0 -d zipf --zipf-theta 0.99 --runsec 15 --warmup-sec 5 -i 3` | +| small (4.6 M × 100 B) | **smaller than dataset** (256 MB → ~45 % in mutable, rest on disk) | uniform | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 16 -n 4600000 -v 100 --device randomaccess --log-memory 256m --rumd 100,0,0,0 --runsec 15 --warmup-sec 5 -i 3 --data-path /mnt/nvme/kv` | +| small (4.6 M × 100 B) | smaller than dataset (256 MB) | zipf θ=0.99 | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 16 -n 4600000 -v 100 --device randomaccess --log-memory 256m --rumd 100,0,0,0 -d zipf --zipf-theta 0.99 --runsec 15 --warmup-sec 5 -i 3 --data-path /mnt/nvme/kv` | +| **large (100 M × 100 B ≈ 12 GB)** | fits (auto → 16 GB) | uniform | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 32 -n 100000000 -v 100 --device null --rumd 100,0,0,0 --runsec 15 --warmup-sec 5 -i 3` | +| large (100 M × 100 B) | fits (16 GB) | zipf θ=0.99 | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 32 -n 100000000 -v 100 --device null --rumd 100,0,0,0 -d zipf --zipf-theta 0.99 --runsec 15 --warmup-sec 5 -i 3` | +| large (100 M × 100 B) | **constrained** (2 GB → ~13 % in mutable, rest on NVMe) | uniform | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 32 -n 100000000 -v 100 --device randomaccess --log-memory 2g --rumd 100,0,0,0 --runsec 15 --warmup-sec 5 -i 3 --data-path /mnt/nvme/kv` | +| large (100 M × 100 B) | constrained (2 GB) | zipf θ=0.99 | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 32 -n 100000000 -v 100 --device randomaccess --log-memory 2g --rumd 100,0,0,0 -d zipf --zipf-theta 0.99 --runsec 15 --warmup-sec 5 -i 3 --data-path /mnt/nvme/kv` | +| 4.6 M | fits | 50/40/5/5 RUMD | `numactl --cpunodebind=0 --membind=0 dotnet $KV -t 16 -n 4600000 -v 100 --device null --rumd 50,40,5,5 --runsec 15 --warmup-sec 5 -i 3` | +| 100 M | fits | thread sweep (1→32) | `numactl --cpunodebind=0 --membind=0 dotnet $KV -n 100000000 -v 100 --device null --rumd 100,0,0,0 --load-threads 32 --run-threads-sweep 1,2,4,8,16,32 --runsec 15 --warmup-sec 5 -i 3` | + +**Knobs that affect the in-memory vs out-of-memory mix**: +- `--log-memory` (default: auto-sized to fit dataset at 90 % mutable). Set + smaller to force records to spill below `ReadOnlyAddress` and be re-read + from disk on lookup. Use units: `512m`, `1g`, `16g`. +- `--device`: `null` skips all I/O (pure engine ceiling), `randomaccess` + uses `RandomAccessLocalStorageDevice` (Linux default), `native` uses + `NativeStorageDevice` (libaio on Linux), `filestream` is the slowest + managed device. +- `--max-inline-value-size` (default 16 KB): values larger than this overflow + to a heap-allocated buffer (slower path). + +### Detailed examples + +#### 1. In-memory throughput ceiling (100 % read) + +`--device null` + log auto-sized to fit the dataset in the mutable region. + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device null --rumd 100,0,0,0 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 3 +``` + +Sample result on a 2 × 80-core / 2-NUMA host: +``` +[run 1] 566,958,848 ops in 5.000 s (113,386,218 ops/sec) reads=566,974,720 writes=0 deletes=0 gc=0/0/0 alloc/wkr=4360B +[run 2] 567,287,296 ops in 5.000 s (113,451,754 ops/sec) reads=567,303,424 writes=0 deletes=0 gc=0/0/0 alloc/wkr=4328B +[run 3] 567,573,760 ops in 5.000 s (113,509,656 ops/sec) reads=567,589,888 writes=0 deletes=0 gc=0/0/0 alloc/wkr=4328B +[aggregate] iterations=3 mean=113,449,209 ops/sec stdev=50,425.7 (0.0%) min=113,386,218 max=113,509,656 trimmed=113,451,754 +``` + +#### 2. In-memory throughput with skewed reads (zipf) + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device null --rumd 100,0,0,0 \ + --distribution zipf --zipf-theta 0.99 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 3 +``` + +Zipf reads concentrate on the same hot keys, so cache-line reuse goes up; +typically 1.5–2× higher than uniform. + +#### 3. Mixed RUMD throughput + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device null --rumd 50,40,5,5 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 3 +``` + +When `d % > 0`, every Delete is **immediately followed by a re-Upsert of the +same key** (counted as a separate op). This keeps the dataset stable across +the run; otherwise a delete-heavy workload would silently degrade into a +read-miss workload. + +#### 4. NVMe write throughput (load only is write-bound) + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device randomaccess \ + --rumd 100,0,0,0 --runsec 1 --warmup-sec 0 \ + --data-path /mnt/nvme/kv +``` + +Load throughput is reported by the `[load]` line. The run phase here is +short — we just want the load number. + +#### 5. NVMe read-only after warm load (in-memory reads, large log) + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device randomaccess --rumd 100,0,0,0 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 3 \ + --data-path /mnt/nvme/kv +``` + +With `--log-memory` auto-sized to fit the dataset, reads stay in memory and +the NVMe device is mostly idle. Sample result: `115.83 M ops/sec trimmed +mean, 0.1 % stdev`. + +#### 6. Multi-iteration stability run + +```bash +dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device null --rumd 100,0,0,0 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 7 +``` + +The final `[aggregate]` line shows `mean`, `stdev`, `stdev%`, `min`, `max`, +and (when `-i ≥ 3`) a `trimmed` mean that drops the single highest and +lowest samples. Use `trimmed` for stability-focused reports. + +#### 7. Data integrity check after load + +```bash +dotnet KV.benchmark.dll -t 4 -n 1000000 -v 100 \ + --device null --validate \ + --runsec 0 --warmup-sec 0 +``` + +`--validate` runs a single-threaded scan after the load phase, reading back +every key and asserting the value bytes match what was written. Exits with +code `2` on mismatch. + +#### 8. Device backend sweep + +```bash +for dev in null randomaccess filestream; do + rm -rf /mnt/nvme/kv-sweep/* + numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 -v 100 \ + --device $dev --rumd 100,0,0,0 \ + --runsec 5 --warmup-sec 2 --report-interval-sec 0 -i 3 \ + --data-path /mnt/nvme/kv-sweep \ + --csv-output /tmp/kv-sweep.csv --quiet +done +column -t -s , /tmp/kv-sweep.csv | head +``` + +#### 9. Run-thread scalability sweep (single load → multiple thread counts) + +```bash +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -n 100000000 -v 96 \ + --load-threads 32 --run-threads-sweep 1,2,4,8,16,32 \ + --device null --rumd 100,0,0,0 \ + --runsec 15 --warmup-sec 5 -i 3 +``` + +Loads the 100 M-key dataset ONCE using 32 threads, then runs the full +`--iterations` loop at each thread count in the sweep. The final summary +prints a compact table with trimmed mean, stdev%, and speedup vs the +smallest thread count: + +``` + Run sweep (3 iterations per thread count): + threads | trimmed | mean | stdev% | speedup + --------+----------------+----------------+---------+-------- + 1 | 2.08 M ops/sec | 2.05 M ops/sec | 2.8% | 1.00× + 2 | 4.22 M ops/sec | 4.22 M ops/sec | 0.9% | 2.03× + 4 | 8.29 M ops/sec | 8.32 M ops/sec | 0.8% | 3.99× + 8 |16.53 M ops/sec |16.75 M ops/sec | 2.7% | 7.95× + 16 |33.94 M ops/sec |34.02 M ops/sec | 1.9% | 16.32× + 32 |69.94 M ops/sec |69.99 M ops/sec | 0.2% | 33.64× +``` + +## All flags + +### Workload (10) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-t / --threads` | `1` | Default run-phase worker count (also used for load if `--load-threads` is unspecified). | +| `--load-threads` | `0` | Threads to use for the LOAD phase. `0` = same as `--threads`. Useful when you want a fast parallel load followed by single-thread or sweep runs on the same dataset. | +| `--run-threads-sweep` | none | Comma-separated list of run-phase thread counts (e.g. `1,2,4,8,16,32`). When set, the engine loads ONCE and then runs the full `--iterations` loop for each thread count. Overrides `--threads` for the run phase. | +| `-n / --keys` | `100_000_000` | Number of unique keys. | +| `-v / --value-size` | `100` | Value length in bytes. Range: **32 ≤ value-size ≤ `--max-inline-value-size`** (inline-value path only). | +| `--rumd` | `100,0,0,0` | Percent of [reads, upserts, RMWs, deletes] (sum=100). When `d% > 0`, deletes auto-reinsert. | +| `-d / --distribution` | `uniform` | `uniform` or `zipf`. | +| `--zipf-theta` | `0.99` | Zipf skew parameter. | +| `--runsec` | `30` | Run-phase duration in seconds (excludes warmup). | +| `--warmup-sec` | `5` | Warmup duration in seconds, discarded from results. `0` disables. | + +### Reproducibility (2) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `-s / --seed` | `211` | Base RNG seed. Per-thread seeds = `SplitMix64(seed, threadIdx)`. Same seeds every iteration → workload is bit-deterministic across iterations (for read-only RUMD). | +| `-i / --iterations` | `1` | Run-phase iterations. Load runs once; warmup runs once per iteration. | + +### Sizing (4) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--hashpack` | `2.0` | Hash packing factor. `index_size_requested = (long)(keys / hashpack) << 6`. Note the engine rounds **down** to the nearest power of 2, so the effective hashpack is typically higher than configured — both `requested` and `applied` are emitted in metadata. | +| `--log-memory` | auto | Total in-memory log window (e.g. `16GB`). Auto-default = `NextPow2(keys × recordSize / 0.9)`, capped at 70 % of host MemAvailable. Explicit values bypass the RAM cap. | +| `--page-size` | `16MB` | Page size (e.g. `16MB`, `4MB`). **Matches Garnet `defaults.conf` PageSize=16m**. | +| `--segment-size` | `1GB` | On-disk segment size. **Matches Garnet SegmentSize=1g**. | +| `--max-inline-value-size` | `16KB` | `KVSettings.MaxInlineValueSize` — values larger than this overflow to a separate heap object. **Matches Garnet `ValueOverflowThreshold=16k`**. | +| `--preallocate-log` | `false` | When `true`, pre-touches every log page at startup so first-touch faults don't bias the timed window. **Default `false` matches Garnet**; enable for the most stable single-thread benchmark numbers (cost: 6 s per 16 GB of log at setup). | + +### Device (5) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--device` | `default` | `native`, `randomaccess`, `filestream`, `null`, `default`. | +| `--device-throttle` | `0` | Max in-flight IOs. `0` = device default (`120` for every Tsavorite device). | +| `--device-io-backend` | `default` | Linux native backend: `libaio`, `default` (→ libaio). | +| `--device-completion-threads` | `0` | Native completion thread count. `0` → 1. | +| `--data-path` | OS temp | Where hlog files live. A unique `/kv-run--/` child directory is created per run and removed on exit. | + +### Host tuning (3) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--no-numa-pin` | off | Disable in-process NUMA pinning. | +| `--numa-node` | `0` | Which NUMA node to pin to. | +| `--no-threadpool-tune` | off | Disable auto `ThreadPool.SetMinThreads(max(t*2, 256))` (and the matching restore-on-exit). | + +### Output / hygiene (5) + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--report-interval-sec` | `1` | Live throughput tick (seconds). `0` disables — **recommended for canonical numbers**. | +| `--validate` | off | After load: single-threaded readback of every key. Aborts (exit 2) on mismatch. | +| `--json-output FILE` | none | Append **pretty-printed** JSON summary rows to FILE (one row per phase). | +| `--json-stdout` | off | Also emit single-line `KV-RESULT-JSON: {…}` blobs to stdout for log scraping. | +| `--csv-output FILE` | none | Append CSV rows to FILE. | +| `--quiet` | off | Suppress human-readable progress/config; final summary still prints. | + +### Hard-coded (no flag) + +- `MutableFraction = 0.9` (KVSettings default; matches Garnet `MutablePercent=90`). +- `MaxInlineKeySize = 128 B` (KVSettings default; matches Garnet). +- `O_DIRECT` / `FILE_FLAG_NO_BUFFERING` on managed devices. +- Run-dir scoped cleanup (no recursive delete of `--data-path`). +- Session context: **`BasicContext` only** (safe path — per-op epoch resume/suspend is included in every measurement). +- Synthetic data only. +- `ObjectAllocator` only. + +### Hot-loop architecture + +`RunWorkload` is a single per-thread method: + +- **Per-op inline xorshift32 key gen** — uniform: `wk & (N-1)` if `N` is a power of two (1 cycle), otherwise Lemire's fast modulo `((ulong)wk * N) >> 32` (~5 cycles). 64-bit keyCount paths use plain `% N`. +- **Per-op independent xorshift32 coin toss** for op selection (read/upsert/RMW/delete). Two independent RNG states are MANDATORY when distribution=zipf: zipf consumes its source RNG non-uniformly, so reusing it for op-select would bias the rumd ratio. +- **Pre-computed op cutoffs** in the 32-bit RNG domain so the coin toss is `wr < cutoff` with no per-op multiply. +- **Chunk-boundary done check** — saves a per-op `Volatile.Read(ref doneFlag)`. Stop latency is bounded at one chunk (~0.3 ms at 2 M ops/sec). +- **`CompletePending(false)` every 512 ops** — JIT folds the literal `% 512` to a bitmask. +- **`Interlocked.Add(ref globalChunkIdx, kChunkSize)`** for chunk scheduling — at single thread it's no slower than a thread-local counter and at multi-thread it gives even chunk distribution across workers. +- **Hot-path buffers (`value`, `input`, `output`) are stackalloc'd in `WorkerProc`** and passed into `RunWorkload` by `ref`. This is measurably (~3.7 %) faster than declaring them inside `RunWorkload`: a smaller method body lets the JIT inline `BasicContext.Read` into the hot loop rather than emitting an out-of-line `call ContextRead`. +- **All three buffers are 32-byte aligned** via overallocate-then-round-up. Reader's `Slice(0, 32).CopyTo(...)` JITs to a single `vmovdqu ymm0, [src]` + `vmovdqu [dst], ymm0` (32-byte AVX2). The SOURCE (value bytes in the Tsavorite log) is always 5-byte unaligned within an 8-byte-aligned record (header is `RecordInfo(8) + NumIndicatorBytes(3) + KeyLen(1) + RecLen(1) + Key(8) = 21`), and with 120 B records the value start cycles `21,13,5,61,53,45,37,29` mod 64 — half of reads already cross a cache line on the source side, which can't be fixed without changing the log layout. Aligning the DESTINATION (output) and INPUT buffers to 32 B removes the dest-side cross-line penalty; measured ~+5 % at memory-bound scale (100 M keys, BasicContext, in-memory log). +- **`KvSessionFunctions.Reader` copies a constant 32 bytes** (one cache line). The `32` is a `const int`, not a `static readonly int`, so the JIT const-folds the `Slice(0, 32).CopyTo(...)` into a single SSE memcpy (≈15 % single-thread improvement over a non-const length). + +At single thread on a 100M-key SpanByte workload (96-byte values, BasicContext, in-memory log), the difference between an inlined `BasicContext.Read` and an out-of-line call is roughly **30–40 % throughput** — much more than the per-op work in the call itself, because the inlined version lets the JIT keep arguments in registers and skip a full call-prologue/epilogue. + + +## Output schema + +### Human-readable + +The startup **config block** echoes every resolved flag plus NUMA placement. + +Per-phase line shape: +``` +[][optionally ] ops in s () reads=N writes=N deletes=N overshoot=N exit-lag=Xms gc=g0/g1/g2 alloc/wkr=Nbytes +``` + +After the last iteration, an `[aggregate]` line with `mean`, `stdev`, `stdev%`, +`min`, `max`, and (when `-i ≥ 3`) `trimmed`, followed by a multi-line +`KV.benchmark — final summary` block that recaps config, load, and run perf +in a readable form. + +### JSON + +By default, NO JSON is written to stdout — the per-phase blob is huge and +clutters the terminal. Two opt-ins: + +- `--json-output FILE` — appends a **pretty-printed** JSON object per phase + (load / each run iter / final aggregate) to FILE. Good for archiving or + diffing across runs. +- `--json-stdout` — also emits a single-line `KV-RESULT-JSON: {…}` blob per + phase to stdout. Useful for log-scraping pipelines. + +Schema is `schema_version: "1"`. + +Top-level fields include: `phase`, `iteration`, `ops_per_sec`, `elapsed_sec`, +`total_ops_for_throughput`, `final_total_ops`, `overshoot_ops`, +`max_worker_exit_lag_ms`, `reads`, `writes`, `deletes`, `interrupted`, +`error`, `log.{begin,head,readonly,tail}_address`, `gc_delta.{gen0,gen1,gen2,alloc_bytes_by_worker_max}`, +`config.*` (every flag's resolved value), `host.*` (hostname, OS, dotnet version, +git sha, NUMA node, worker CPU mask, Server GC, GC latency mode, tiered compilation, +THP mode, data path, RAM), and `argv` (the exact command-line that produced the row). + +The `reads` / `writes` / `deletes` counters are per-RESP-op (not per-record): +when `--rumd ...,d=N` is non-zero, every delete is immediately followed by a +re-insert (Upsert) which is counted under `writes`. + +### CSV (`--csv-output FILE`) + +Same fields, wide schema, one row per phase per iteration plus an aggregate row. +Header is written automatically the first time the file is created. + +### Headline throughput calculation + +`ops/sec` is computed from a **scoreboard snapshot taken immediately after +`done = true` is published**, divided by `doneTicks - startTicks`. The +post-join sum (`final_total_ops`) may be slightly larger because workers +flush one last `localOps` after observing `done` at their next chunk boundary — +that delta is reported as `overshoot_ops` for diagnostics, never as part of +the headline number. This excludes worker-join lag from the measured duration. + +## Determinism + +With `--rumd 100,0,0,0` (pure reads), `--device null`, the same `--seed`, +the same `--threads`, and the same `--keys`, every iteration touches exactly +the same set of keys in the same order against an unchanging store. Any +delta in `ops_per_sec` between iterations is OS scheduling noise on top of +an otherwise identical workload — this is the "stable mean ± stdev" target +and typically produces stdev% well under 1 %. + +For mixed RUMD, the same key/op stream is replayed each iteration, but +writes from prior iterations remain visible. + +**GC quiescence**: between iterations the benchmark forces a full +Gen2 collection + finalizer pass + Gen2 collection again *after* all +workers have parked on the start gate but *before* the timed window +opens. This guarantees the per-iteration `gc_delta` reflects only +collections that actually happened during the measured window (almost +always `0/0/0` since the hot loop is allocation-free). + +## NUMA + ThreadPool + +On Linux, `KvNumaPinning` reads `/sys/devices/system/node/node/cpulist`, +intersects with `sched_getaffinity` (so cgroup CPU quotas are honored), +deduplicates hyperthread siblings, and pins each worker thread to one +physical core on the chosen node via `sched_setaffinity`. The setup/reporter +thread pins to the first **un-pinned** CPU on the same node, so it never +competes with a worker. + +On Windows the helper uses `SetThreadGroupAffinity` with the same exclusion +semantics; best-effort on multi-processor-group hosts. + +`--threads` is the only knob that controls reservation: when +`--threads < nodeCpus`, the remaining `nodeCpus - threads` CPUs are +naturally un-pinned and absorb Tsavorite IO completion threads, the +reporter, and runtime GC threads. The default `--threads 1` reserves +essentially the entire node — a single-thread baseline with massive +headroom. + +**Always run with `numactl --membind=$N --cpunodebind=$N`** on multi-NUMA +Linux hosts for full memory locality. Without it, even with our CPU +pinning, the .NET allocator may place pages on the wrong node and you can +see ~2× variance between runs. Worked example: + +```bash +# Two runs of the same config — first without numactl, second with. +dotnet KV.benchmark.dll -t 64 -n 50000000 --device null --runsec 5 -i 5 +numactl --membind=0 --cpunodebind=0 \ + dotnet KV.benchmark.dll -t 64 -n 50000000 --device null --runsec 5 -i 5 +``` + +`ThreadPool.SetMinThreads(max(threads * 2, 256))` is applied at startup and +**restored to the previous values on exit** (so the benchmark plays nicely +with larger test harnesses that keep the process alive afterward). +`--no-threadpool-tune` skips both. + +## Device backends + +| `--device` | Linux | Windows | Notes | +| --- | --- | --- | --- | +| `default` | `RandomAccessLocalStorageDevice` | `LocalStorageDevice` (IOCP) | Platform default. | +| `native` | `NativeStorageDevice` (libaio) | `LocalStorageDevice` | Linux uses libaio via the shipped `libnative_device.so`. | +| `randomaccess` | `RandomAccessLocalStorageDevice` | same | Pure .NET `RandomAccess` API; no native deps. | +| `filestream` | `ManagedLocalStorageDevice` | same | Pure .NET `FileStream`; lowest performance, no native deps. | +| `null` | `NullDevice` | `NullDevice` | No I/O; for measuring engine-only throughput. | + +`--device-throttle 0` resolves to the device default of 120 in-flight IOs. +`--device-completion-threads 0` resolves to 1. + +## Sizing cheatsheet + +| `--keys` × `--value-size` | Auto `--log-memory` | `--hashpack 2.0` → applied index | +| --- | --- | --- | +| 10 M × 100 B | 2 GB | 256 MB (4 M buckets, effective hashpack ≈ 2.50) | +| 100 M × 100 B | 16 GB | 2 GB (32 M buckets, effective hashpack ≈ 2.98) | +| 250 M × 100 B | 32 GB | 8 GB (128 M buckets, effective hashpack ≈ 1.95) | +| 1 G × 100 B | 128 GB (or clamped) | 16 GB (256 M buckets, effective hashpack ≈ 3.91) | +| 1 M × 16 B | 64 MB (floored at 2 × page) | 32 MB (512 K buckets) | + +The auto-default `--log-memory` is capped at 70 % of `MemAvailable` on +Linux. Explicit `--log-memory` values are never clamped — let the OOM +killer be the final authority for "did you really mean that?". + +## Troubleshooting + +- **"Throughput much lower than expected on Linux"** → check `numa` line in + the config block to verify pinning; check `worker_cpu_mask` in metadata; + make sure you're running under `numactl --membind=$N` on multi-NUMA hosts; + check `--device` is right for your goal (use `null` for engine-only). +- **"Process killed by OOM"** → reduce `--keys` or `--log-memory`. The auto- + clamp only fires for auto-derived `--log-memory`, not for explicit values. +- **`--validate` reports mismatches** → likely a session-functions + regression. The exit code is `2` and the message reports the first + mismatch. +- **"Numbers vary wildly between runs"** → check `stdev%` in the + `[aggregate]` line. Increase `-i`. Check whether the host is under other + load (`mpstat`, `top`). For canonical numbers use + `--report-interval-sec 0`. +- **"Can't access `/proc/sys/vm/drop_caches`"** → this benchmark doesn't + drop caches; it relies on `--warmup-sec` to stabilize hot state. Use + `numactl --membind=0` + an external `echo 3 > /proc/sys/vm/drop_caches` + between invocations if cold-start measurement is needed. + +## Reproducing the numbers in this README + +Numbers above were collected on `git rev-parse --short HEAD = +f0652b638` on Linux 6.8 / Ubuntu 24.04 / .NET 10, dual-socket +80-physical-core / 2-NUMA-node host with 540 GB RAM and a /DATA2 +NVMe SSD (`ext4`). Each row used the exact command shown next to it, +preceded by `numactl --membind=0 --cpunodebind=0`. + +## Related + +- Tsavorite onboarding: +- Garnet benchmark scripts: `benchmark/` at the repo root. diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/XoshiroRng.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/XoshiroRng.cs new file mode 100644 index 00000000000..12b14a38f89 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/XoshiroRng.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +namespace Tsavorite.kvbench +{ + /// + /// xoshiro256** PRNG with a 256-bit state. The four state words are seeded from + /// a single 64-bit seed via SplitMix64 (per the xoshiro paper recommendation) so + /// that adjacent seeds produce well-decorrelated streams. + /// + /// + /// Hot-path: each Next/NextUInt64 call is 4 register operations + /// plus a rotl. Constructor is ~30 ns (4 SplitMix64 calls + zero-state check). + /// + public struct XoshiroRng + { + ulong s0, s1, s2, s3; + + public XoshiroRng(ulong seed) + { + ulong basis = seed; + s0 = SplitMix64(ref basis); + s1 = SplitMix64(ref basis); + s2 = SplitMix64(ref basis); + s3 = SplitMix64(ref basis); + // Reject the (vanishingly unlikely) all-zero state - it's a fixed point. + if ((s0 | s1 | s2 | s3) == 0UL) + { + s0 = 1UL; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong SplitMix64(ref ulong x) + { + x += 0x9E3779B97F4A7C15UL; + ulong z = x; + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9UL; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBUL; + return z ^ (z >> 31); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong Rotl(ulong x, int k) => (x << k) | (x >> (64 - k)); + + /// + /// Produce the next 64-bit pseudorandom value. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ulong NextUInt64() + { + ulong result = Rotl(s1 * 5UL, 7) * 9UL; + ulong t = s1 << 17; + s2 ^= s0; + s3 ^= s1; + s1 ^= s2; + s0 ^= s3; + s2 ^= t; + s3 = Rotl(s3, 45); + return result; + } + + /// + /// Produce the next 32-bit pseudorandom value. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint NextUInt32() => (uint)(NextUInt64() >> 32); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/ZipfGenerator.cs b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/ZipfGenerator.cs new file mode 100644 index 00000000000..1aaaa430c0f --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/KV.benchmark/ZipfGenerator.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace Tsavorite.kvbench +{ + /// + /// Pre-computed Zipf distribution constants, shared across all per-thread + /// generators. Computing these is O(N) in the number of keys due to + /// — doing it once on the main thread saves + /// O(N * threads) Math.Pow calls. + /// + public sealed class ZipfConstants + { + public readonly long Size; + public readonly double Theta; + public readonly double ZetaN; + public readonly double Alpha; + public readonly double Cutoff2; + public readonly double Eta; + + public ZipfConstants(long size, double theta) + { + Size = size; + Theta = theta; + ZetaN = Zeta(size, theta); + Alpha = 1.0 / (1.0 - theta); + Cutoff2 = Math.Pow(0.5, theta); + var zeta2 = Zeta(2, theta); + Eta = (1.0 - Math.Pow(2.0 / size, 1.0 - theta)) / (1.0 - zeta2 / ZetaN); + } + + private static double Zeta(long count, double theta) + { + double zetaN = 0.0; + for (long i = 1; i <= count; ++i) + zetaN += 1.0 / Math.Pow(i, theta); + return zetaN; + } + } + + /// + /// Per-thread Zipf key sampler. Stores 5 doubles + a reference to a shared + /// ; no per-instance setup cost. + /// + public struct ZipfGenerator + { + readonly long size; + readonly double zetaN; + readonly double alpha; + readonly double cutoff2; + readonly double eta; + + public ZipfGenerator(ZipfConstants constants) + { + size = constants.Size; + zetaN = constants.ZetaN; + alpha = constants.Alpha; + cutoff2 = constants.Cutoff2; + eta = constants.Eta; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long Next(ref XoshiroRng rng) + { + // Use top 53 bits of a 64-bit draw for a uniform [0,1) double. + double u = (rng.NextUInt64() >> 11) * (1.0 / (1UL << 53)); + double uz = u * zetaN; + if (uz < 1.0) return 0; + if (uz < 1.0 + cutoff2) return 1; + return (long)(size * Math.Pow(eta * u - eta + 1.0, alpha)); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ConcurrentDictionaryBenchmark.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ConcurrentDictionaryBenchmark.cs index 4037a5c4753..cd757ec8465 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ConcurrentDictionaryBenchmark.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ConcurrentDictionaryBenchmark.cs @@ -14,13 +14,13 @@ namespace Tsavorite.benchmark { - internal class KeyComparer : IEqualityComparer + internal class KeyComparer : IEqualityComparer { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Equals(Key x, Key y) => x.value == y.value; + public bool Equals(FixedLengthKey x, FixedLengthKey y) => x.value == y.value; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetHashCode(Key obj) => (int)Utility.GetHashCode(obj.value); + public int GetHashCode(FixedLengthKey obj) => (int)Utility.GetHashCode(obj.value); } internal unsafe class ConcurrentDictionary_YcsbBenchmark @@ -31,17 +31,17 @@ internal unsafe class ConcurrentDictionary_YcsbBenchmark readonly int readPercent, upsertPercent, rmwPercent; readonly Input[] input_; - readonly Key[] init_keys_; - readonly Key[] txn_keys_; + readonly FixedLengthKey[] init_keys_; + readonly FixedLengthKey[] txn_keys_; - readonly ConcurrentDictionary store; + readonly ConcurrentDictionary store; long idx_ = 0; long total_ops_done = 0; volatile bool done = false; Input* input_ptr; - internal ConcurrentDictionary_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoader) + internal ConcurrentDictionary_YcsbBenchmark(FixedLengthKey[] i_keys_, FixedLengthKey[] t_keys_, TestLoader testLoader) { this.testLoader = testLoader; init_keys_ = i_keys_; @@ -90,7 +90,7 @@ private void RunYcsb(int thread_idx) } var sw = Stopwatch.StartNew(); - Value value = default; + FixedLengthValue value = default; long reads_done = 0; long writes_done = 0; long deletes_done = 0; @@ -129,7 +129,7 @@ private void RunYcsb(int thread_idx) } if (r < rmwPercent) { - store.AddOrUpdate(txn_keys_[idx], *(Value*)(input_ptr + (idx & 0x7)), (k, v) => new Value { value = v.value + (input_ptr + (idx & 0x7))->value }); + store.AddOrUpdate(txn_keys_[idx], *(FixedLengthValue*)(input_ptr + (idx & 0x7)), (k, v) => new FixedLengthValue { value = v.value + (input_ptr + (idx & 0x7))->value }); ++writes_done; continue; } @@ -267,7 +267,7 @@ private void SetupYcsb(int thread_idx) int count = 0; #endif - Value value = default; + FixedLengthValue value = default; for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; chunk_idx < testLoader.InitCount; @@ -276,7 +276,7 @@ private void SetupYcsb(int thread_idx) for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) { - Key key = init_keys_[idx]; + FixedLengthKey key = init_keys_[idx]; store[key] = value; } #if DASHBOARD @@ -371,14 +371,14 @@ void DoContinuousMeasurements() #region Load Data - internal static void CreateKeyVectors(TestLoader testLoader, out Key[] i_keys, out Key[] t_keys) + internal static void CreateKeyVectors(TestLoader testLoader, out FixedLengthKey[] i_keys, out FixedLengthKey[] t_keys) { - i_keys = new Key[testLoader.InitCount]; - t_keys = new Key[testLoader.TxnCount]; + i_keys = new FixedLengthKey[testLoader.InitCount]; + t_keys = new FixedLengthKey[testLoader.TxnCount]; } - internal class KeySetter : IKeySetter + internal class KeySetter : IKeySetter { - public void Set(Key[] vector, long idx, long value) => vector[idx].value = value; + public void Set(FixedLengthKey[] vector, long idx, long value) => vector[idx].value = value; } #endregion diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLenYcsbBenchmark.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLenYcsbBenchmark.cs index c74ad0335f9..e3b4b1548bd 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLenYcsbBenchmark.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLenYcsbBenchmark.cs @@ -12,10 +12,23 @@ namespace Tsavorite.benchmark { #pragma warning disable IDE0065 // Misplaced using directive - using StructStoreFunctions = StoreFunctions>; + using FixedLenStoreFunctions = StoreFunctions; - internal class Tsavorite_YcsbBenchmark + internal class FixedLenYcsbBenchmark + where TAllocator : IAllocator { + RevivificationSettings FixedLengthBins = new() + { + FreeRecordBins = + [ + new RevivificationBin() + { + RecordSize = RecordInfo.Size + 2 * (sizeof(int) + sizeof(long)), // We have "fixed length" for these integer bins, with long Key and Value + BestFitScanLimit = RevivificationBin.UseFirstFit + } + ] + }; + // Ensure sizes are aligned to chunk sizes static long InitCount; static long TxnCount; @@ -24,20 +37,20 @@ internal class Tsavorite_YcsbBenchmark readonly ManualResetEventSlim waiter = new(); readonly int numaStyle; readonly int readPercent, upsertPercent, rmwPercent; - readonly SessionFunctions functions; + readonly SessionFixedLenFunctions functions; readonly Input[] input_; - readonly Key[] init_keys_; - readonly Key[] txn_keys_; + readonly FixedLengthKey[] init_keys_; + readonly FixedLengthKey[] txn_keys_; readonly IDevice device; - readonly TsavoriteKV> store; + readonly TsavoriteKV> store; long idx_ = 0; long total_ops_done = 0; volatile bool done = false; - internal Tsavorite_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLoader) + internal FixedLenYcsbBenchmark(FixedLengthKey[] i_keys_, FixedLengthKey[] t_keys_, TestLoader testLoader) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -54,7 +67,7 @@ internal Tsavorite_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLo readPercent = testLoader.ReadPercent; upsertPercent = testLoader.UpsertPercent; rmwPercent = testLoader.RmwPercent; - functions = new SessionFunctions(); + functions = new SessionFixedLenFunctions(); input_ = new Input[8]; for (int i = 0; i < 8; i++) @@ -64,7 +77,7 @@ internal Tsavorite_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLo { RevivificationLevel.None => default, RevivificationLevel.Chain => new RevivificationSettings(), - RevivificationLevel.Full => RevivificationSettings.DefaultFixedLength.Clone(), + RevivificationLevel.Full => FixedLengthBins, _ => throw new ApplicationException("Invalid RevivificationLevel") }; @@ -81,12 +94,12 @@ internal Tsavorite_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLo if (testLoader.Options.ThreadCount >= 16) device.ThrottleLimit = testLoader.Options.ThreadCount * 12; - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = testLoader.GetHashTableSize(), LogDevice = device, PreallocateLog = true, - MemorySize = 1L << 34, + LogMemorySize = 1L << 34, RevivificationSettings = revivificationSettings, CheckpointDir = testLoader.BackupPath }; @@ -95,11 +108,11 @@ internal Tsavorite_YcsbBenchmark(Key[] i_keys_, Key[] t_keys_, TestLoader testLo { kvSettings.PageSize = 1L << 25; kvSettings.SegmentSize = 1L << 30; - kvSettings.MemorySize = 1L << 28; + kvSettings.LogMemorySize = 1L << 28; } store = new(kvSettings - , StoreFunctions.Create(new Key.Comparer()) + , StoreFunctions.Create(new FixedLengthKey.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -125,7 +138,9 @@ private void RunYcsbUnsafeContext(int thread_idx) var sw = Stopwatch.StartNew(); - Value value = default; + FixedLengthKey key = default; + FixedLengthValue valueStruct = default; + Span value = valueStruct.AsSpan(); Input input = default; Output output = default; @@ -133,7 +148,8 @@ private void RunYcsbUnsafeContext(int thread_idx) long writes_done = 0; long deletes_done = 0; - using var session = store.NewSession(functions); + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); var uContext = session.UnsafeContext; uContext.BeginUnsafe(); @@ -157,26 +173,29 @@ private void RunYcsbUnsafeContext(int thread_idx) _ = uContext.CompletePending(false); } + key = txn_keys_[idx]; // Copy locally for SpanByte backing int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 if (r < readPercent) { - _ = uContext.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default); + _ = uContext.Read(key, ref input, ref output, Empty.Default); ++reads_done; continue; } if (r < upsertPercent) { - _ = uContext.Upsert(ref txn_keys_[idx], ref value, Empty.Default); + _ = uContext.Upsert(key, value, Empty.Default); ++writes_done; continue; } if (r < rmwPercent) { - _ = uContext.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default); + _ = uContext.RMW(key, ref input_[idx & 0x7], Empty.Default); ++writes_done; continue; } - _ = uContext.Delete(ref txn_keys_[idx], Empty.Default); + _ = uContext.Delete(key, Empty.Default); + if (di) + uContext.Upsert(key, value, Empty.Default); ++deletes_done; } } @@ -209,7 +228,9 @@ private void RunYcsbSafeContext(int thread_idx) var sw = Stopwatch.StartNew(); - Value value = default; + FixedLengthKey key = default; + FixedLengthValue valueStruct = default; + Span value = valueStruct.AsSpan(); Input input = default; Output output = default; @@ -217,7 +238,8 @@ private void RunYcsbSafeContext(int thread_idx) long writes_done = 0; long deletes_done = 0; - using var session = store.NewSession(functions); + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); var bContext = session.BasicContext; while (!done) @@ -235,26 +257,29 @@ private void RunYcsbSafeContext(int thread_idx) if (idx % 512 == 0) _ = bContext.CompletePending(false); + key = txn_keys_[idx]; // Copy locally for SpanByte backing int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 if (r < readPercent) { - _ = bContext.Read(ref txn_keys_[idx], ref input, ref output, Empty.Default); + _ = bContext.Read(key, ref input, ref output, Empty.Default); ++reads_done; continue; } if (r < upsertPercent) { - _ = bContext.Upsert(ref txn_keys_[idx], ref value, Empty.Default); + _ = bContext.Upsert(key, value, Empty.Default); ++writes_done; continue; } if (r < rmwPercent) { - _ = bContext.RMW(ref txn_keys_[idx], ref input_[idx & 0x7], Empty.Default); + _ = bContext.RMW(key, ref input_[idx & 0x7], Empty.Default); ++writes_done; continue; } - _ = bContext.Delete(ref txn_keys_[idx], Empty.Default); + _ = bContext.Delete(key, Empty.Default); + if (di) + bContext.Upsert(key, value, Empty.Default); ++deletes_done; } } @@ -349,7 +374,7 @@ internal unsafe (double insPerSec, double opsPerSec, long tailAddress) Run(TestL if (checkpointTaken < swatch.ElapsedMilliseconds / testLoader.Options.PeriodicCheckpointMilliseconds) { long start = swatch.ElapsedTicks; - if (store.TryInitiateHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType, testLoader.Options.PeriodicCheckpointTryIncremental)) + if (store.TryInitiateHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType)) { store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); var timeTaken = (swatch.ElapsedTicks - start) / TimeSpan.TicksPerMillisecond; @@ -389,11 +414,13 @@ private void SetupYcsbUnsafeContext(int thread_idx) } waiter.Wait(); - var session = store.NewSession(functions); + var session = store.NewSession(functions); var uContext = session.UnsafeContext; uContext.BeginUnsafe(); - Value value = default; + FixedLengthKey key = default; + FixedLengthValue valueStruct = default; + Span value = valueStruct.AsSpan(); try { @@ -410,7 +437,8 @@ private void SetupYcsbUnsafeContext(int thread_idx) _ = uContext.CompletePending(false); } - _ = uContext.Upsert(ref init_keys_[idx], ref value, Empty.Default); + key = txn_keys_[idx]; // Copy locally for SpanByte backing + _ = uContext.Upsert(key, value, Empty.Default); } } _ = uContext.CompletePending(true); @@ -433,10 +461,12 @@ private void SetupYcsbSafeContext(int thread_idx) } waiter.Wait(); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var bContext = session.BasicContext; - Value value = default; + FixedLengthKey key = default; + FixedLengthValue valueStruct = default; + Span value = valueStruct.AsSpan(); for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; chunk_idx < InitCount; @@ -451,29 +481,25 @@ private void SetupYcsbSafeContext(int thread_idx) _ = bContext.CompletePending(false); } - _ = bContext.Upsert(ref init_keys_[idx], ref value, Empty.Default); + key = txn_keys_[idx]; // Copy locally for SpanByte backing + _ = bContext.Upsert(key, value, Empty.Default); } } _ = bContext.CompletePending(true); } - - #region Load Data - - internal static void CreateKeyVectors(TestLoader testLoader, out Key[] i_keys, out Key[] t_keys) + internal static void CreateKeyVectors(TestLoader testLoader, out FixedLengthKey[] i_keys, out FixedLengthKey[] t_keys) { InitCount = YcsbConstants.kChunkSize * (testLoader.InitCount / YcsbConstants.kChunkSize); TxnCount = YcsbConstants.kChunkSize * (testLoader.TxnCount / YcsbConstants.kChunkSize); - i_keys = new Key[InitCount]; - t_keys = new Key[TxnCount]; - } - - internal class KeySetter : IKeySetter - { - public void Set(Key[] vector, long idx, long value) => vector[idx].value = value; + i_keys = new FixedLengthKey[InitCount]; + t_keys = new FixedLengthKey[TxnCount]; } + } - #endregion + internal class FixedLenYcsbKeySetter : IKeySetter + { + public void Set(FixedLengthKey[] vector, long idx, long value) => vector[idx].value = value; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthKey.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthKey.cs new file mode 100644 index 00000000000..ccf8b39fc0e --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthKey.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Tsavorite.benchmark +{ + [StructLayout(LayoutKind.Explicit, Size = sizeof(long))] + public struct FixedLengthKey : IKey + { + [FieldOffset(0)] + public long value; + + // Not always pinned, so don't assume it is + public readonly bool IsPinned => false; + + [UnscopedRef] + public readonly ReadOnlySpan KeyBytes => MemoryMarshal.Cast(new ReadOnlySpan(in value)); + + public override readonly string ToString() => "{ " + value + " }"; + + // Only call this for stack-based structs, not the ones in the *_keys vectors + public unsafe ReadOnlySpan AsReadOnlySpan() => new(Unsafe.AsPointer(ref this), sizeof(long)); + + public struct Comparer : IKeyComparer + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(key.KeyBytes.AsRef().value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool Equals(TFirstKey key1, TSecondKey key2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => key1.KeyBytes.AsRef().value == key2.KeyBytes.AsRef().value; + } + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Value.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthValue.cs similarity index 59% rename from libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Value.cs rename to libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthValue.cs index 0e93c207c7c..c4a170bc37b 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Value.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/FixedLengthValue.cs @@ -5,15 +5,20 @@ //#define FIXED_SIZE_VALUE //#define FIXED_SIZE_VALUE_WITH_LOCK +using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace Tsavorite.benchmark { [StructLayout(LayoutKind.Explicit, Size = 8)] - public struct Value + public struct FixedLengthValue { public const int Size = 8; + // Only call this for stack-based structs, not the ones in the *_keys vectors + public unsafe Span AsSpan() => new(Unsafe.AsPointer(ref this), sizeof(long)); + [FieldOffset(0)] public long value; } diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Key.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Key.cs deleted file mode 100644 index 708026612c6..00000000000 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Key.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using Tsavorite.core; - -namespace Tsavorite.benchmark -{ - [StructLayout(LayoutKind.Explicit, Size = 8)] - public struct Key - { - [FieldOffset(0)] - public long value; - - public override string ToString() => "{ " + value + " }"; - - public struct Comparer : IKeyComparer - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetHashCode64(ref Key key) => Utility.GetHashCode(key.value); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool Equals(ref Key key1, ref Key key2) => key1.value == key2.value; - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/KeySpanByte.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/KeySpanByte.cs index 433a30fa57a..164194a0589 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/KeySpanByte.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/KeySpanByte.cs @@ -1,16 +1,59 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using Tsavorite.core; namespace Tsavorite.benchmark { - [StructLayout(LayoutKind.Explicit, Size = SpanByteYcsbBenchmark.kKeySize)] - public struct KeySpanByte + /// + /// A key in . + /// + [StructLayout(LayoutKind.Explicit, Size = DataSize)] + public struct KeySpanByte : IKey { + internal const int DataSize = 16; + + /// The data of the key [FieldOffset(0)] - public int length; - [FieldOffset(4)] public long value; + + /// + /// These fields are for kRecordAlignment of the key since Tsavorite no longer aligns key size (i.e. Value start) to . + /// The size remains the same as the previous key size for comparison purposes, making sure the large init_key and txn_key arrays use the same amount of memory. + /// + /// + /// Combined with the header length total of bytes, we get: + /// [RecordInfo header no_extended_namespace keydata valuedata] + /// = [8 + 5 (NumIndicatorBytes + 2 1-byte lengths) + 12 + 100 (see )] = 125 + /// which is rounded up to (8) so the final record size is exactly aligned to two cache lines. + /// To illustrate why this is important: during the conversion to , the change in key alignment was not correctly accounted for; + /// the record was 8 bytes shorter, and the next record's RecordInfo was in the final bytes of the previous record's cache line. This resulted in about a 10% slowdown. + /// + [FieldOffset(sizeof(long))] + public int padding1, padding2; + + /// + /// Convert to string; Only call this for stack-based structs, not the ones in the *_keys vectors. + /// + public override readonly string ToString() => "{ " + value + " }"; + + /// + public readonly bool IsPinned => false; + + /// + public unsafe ReadOnlySpan KeyBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => new(Unsafe.AsPointer(ref this), sizeof(long)); // Not including the padding in the key bytes since it's not actually part of the key; it's just for alignment purposes. + } + + /// + public readonly bool HasNamespace => false; + + /// + public readonly ReadOnlySpan NamespaceBytes => []; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectValue.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectValue.cs new file mode 100644 index 00000000000..78625959ea6 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectValue.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#define EIGHT_BYTE_VALUE +//#define FIXED_SIZE_VALUE +//#define FIXED_SIZE_VALUE_WITH_LOCK + +using System; +using System.IO; +using Tsavorite.core; + +namespace Tsavorite.benchmark +{ + public class ObjectValue : HeapObjectBase + { + public long value; + + public override string ToString() => value.ToString(); + + public override void Dispose() { } + + public override HeapObjectBase Clone() => throw new NotImplementedException(); + public override void DoSerialize(BinaryWriter writer) => throw new NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new NotImplementedException(); + + public ObjectValue() + { + HeapMemorySize = sizeof(long); + } + + public class Serializer : BinaryObjectSerializer + { + public override void Deserialize(out IHeapObject obj) => obj = new ObjectValue { value = reader.ReadInt32() }; + + public override void Serialize(IHeapObject obj) => writer.Write(((ObjectValue)obj).value); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectYcsbBenchmark.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectYcsbBenchmark.cs new file mode 100644 index 00000000000..c7244130daf --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/ObjectYcsbBenchmark.cs @@ -0,0 +1,530 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; +using Tsavorite.core; + +#pragma warning disable IDE0007 // Use implicit type + +namespace Tsavorite.benchmark +{ +#pragma warning disable IDE0065 // Misplaced using directive + using ObjectStoreFunctions = StoreFunctions; + + internal class ObjectYcsbBenchmark + { + // Ensure sizes are aligned to chunk sizes + static long InitCount; + static long TxnCount; + + readonly TestLoader testLoader; + readonly ManualResetEventSlim waiter = new(); + readonly int numaStyle; + readonly int readPercent, upsertPercent, rmwPercent; + readonly SessionObjectFunctions functions; + readonly Input[] input_; + + readonly FixedLengthKey[] init_keys_; + readonly FixedLengthKey[] txn_keys_; + readonly ObjectValue[] object_values; // In parallel with init_keys_ + + readonly IDevice device; + readonly TsavoriteKV> store; + + long idx_ = 0; + long total_ops_done = 0; + volatile bool done = false; + + internal const int kValueDataSize = SpanByteYcsbConstants.kValueDataSize; + + internal ObjectYcsbBenchmark(FixedLengthKey[] i_keys_, FixedLengthKey[] t_keys_, TestLoader testLoader) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // Affinize main thread to last core on first socket if not used by experiment + var (numGrps, numProcs) = Native32.GetNumGroupsProcsPerGroup(); + if ((testLoader.Options.NumaStyle == 0 && testLoader.Options.ThreadCount <= (numProcs - 1)) || + (testLoader.Options.NumaStyle == 1 && testLoader.Options.ThreadCount <= numGrps * (numProcs - 1))) + Native32.AffinitizeThreadRoundRobin(numProcs - 1); + } + this.testLoader = testLoader; + init_keys_ = i_keys_; + txn_keys_ = t_keys_; + if (testLoader.Options.UseObjectValues) + object_values = new ObjectValue[InitCount]; + numaStyle = testLoader.Options.NumaStyle; + readPercent = testLoader.ReadPercent; + upsertPercent = testLoader.UpsertPercent; + rmwPercent = testLoader.RmwPercent; + functions = new SessionObjectFunctions(); + + input_ = new Input[8]; + for (int i = 0; i < 8; i++) + input_[i].value = i; + + var revivificationSettings = testLoader.Options.RevivificationLevel switch + { + RevivificationLevel.None => default, + RevivificationLevel.Chain => new RevivificationSettings(), + RevivificationLevel.Full => new RevivificationSettings() + { + FreeRecordBins = + [ + new RevivificationBin() + { + RecordSize = RecordInfo.Size + KeySpanByte.DataSize + kValueDataSize + 8, // extra to ensure rounding up of value + NumberOfRecords = testLoader.Options.RevivBinRecordCount, + BestFitScanLimit = RevivificationBin.UseFirstFit + } + ], + }, + _ => throw new ApplicationException("Invalid RevivificationLevel") + }; + + if (revivificationSettings is not null) + { + revivificationSettings.RevivifiableFraction = testLoader.Options.RevivifiableFraction; + revivificationSettings.RestoreDeletedRecordsIfBinIsFull = true; + } + + device = Devices.CreateLogDevice(TestLoader.DevicePath, preallocateFile: true, deleteOnClose: !testLoader.RecoverMode, useIoCompletionPort: true); + + var kvSettings = new KVSettings() + { + IndexSize = testLoader.GetHashTableSize(), + LogDevice = device, + PreallocateLog = true, + LogMemorySize = 1L << 35, + RevivificationSettings = revivificationSettings, + CheckpointDir = testLoader.BackupPath, + MaxInlineValueSize = testLoader.Options.UseOverflowValues ? 64 : 128 + }; + + if (testLoader.Options.UseSmallMemoryLog) + { + kvSettings.PageSize = 1L << 22; + kvSettings.SegmentSize = 1L << 26; + kvSettings.LogMemorySize = 1L << 26; + } + + store = new(kvSettings + , StoreFunctions.Create(SpanByteComparer.Instance, DefaultRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + } + + internal void Dispose() + { + store.Dispose(); + device.Dispose(); + } + + private void RunYcsbUnsafeContext(int thread_idx) + { + RandomGenerator rng = new((uint)(1 + thread_idx)); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (numaStyle == 0) + Native32.AffinitizeThreadRoundRobin((uint)thread_idx); + else + Native32.AffinitizeThreadShardedNuma((uint)thread_idx, 2); // assuming two NUMA sockets + } + waiter.Wait(); + + var sw = Stopwatch.StartNew(); + + Span value = stackalloc byte[kValueDataSize]; + Span input = stackalloc byte[kValueDataSize]; + Span output = stackalloc byte[kValueDataSize]; + + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + SpanByteAndMemory _output = SpanByteAndMemory.FromPinnedSpan(output); + + long reads_done = 0; + long writes_done = 0; + long deletes_done = 0; + + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); + var uContext = session.UnsafeContext; + uContext.BeginUnsafe(); + + try + { + while (!done) + { + long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + while (chunk_idx >= TxnCount) + { + if (chunk_idx == TxnCount) + idx_ = 0; + chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + } + + for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) + { + if (idx % 512 == 0) + { + uContext.Refresh(); + uContext.CompletePending(false); + } + + unsafe + { + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = txn_keys_[idx]; + + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) + { + uContext.Read(key, ref pinnedInputSpan, ref _output, Empty.Default); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + uContext.Upsert(key, value, Empty.Default); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + uContext.RMW(key, ref pinnedInputSpan, Empty.Default); + ++writes_done; + continue; + } + uContext.Delete(key, Empty.Default); + if (di) + uContext.Upsert(key, value, Empty.Default); + ++deletes_done; + } + } + } + + uContext.CompletePending(true); + } + finally + { + uContext.EndUnsafe(); + } + + sw.Stop(); + + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); + } + + private void RunYcsbSafeContext(int thread_idx) + { + RandomGenerator rng = new((uint)(1 + thread_idx)); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (numaStyle == 0) + Native32.AffinitizeThreadRoundRobin((uint)thread_idx); + else + Native32.AffinitizeThreadShardedNuma((uint)thread_idx, 2); // assuming two NUMA sockets + } + waiter.Wait(); + + var sw = Stopwatch.StartNew(); + + Span value = stackalloc byte[kValueDataSize]; + Span input = stackalloc byte[kValueDataSize]; + Span output = stackalloc byte[kValueDataSize]; + + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + SpanByteAndMemory _output = SpanByteAndMemory.FromPinnedSpan(output); + + long reads_done = 0; + long writes_done = 0; + long deletes_done = 0; + + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + while (!done) + { + long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + while (chunk_idx >= TxnCount) + { + if (chunk_idx == TxnCount) + idx_ = 0; + chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + } + + for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize && !done; ++idx) + { + if (idx % 512 == 0) + { + if (!testLoader.Options.UseSafeContext) + bContext.Refresh(); + bContext.CompletePending(false); + } + + unsafe + { + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = txn_keys_[idx]; + + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) + { + bContext.Read(key, ref pinnedInputSpan, ref _output, Empty.Default); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + bContext.Upsert(key, value, Empty.Default); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + bContext.RMW(key, ref pinnedInputSpan, Empty.Default); + ++writes_done; + continue; + } + bContext.Delete(key, Empty.Default); + if (di) + bContext.Upsert(key, value, Empty.Default); + ++deletes_done; + } + } + } + + bContext.CompletePending(true); + + sw.Stop(); + + Console.WriteLine($"Thread {thread_idx} done; {reads_done} reads, {writes_done} writes, {deletes_done} deletes in {sw.ElapsedMilliseconds} ms."); + Interlocked.Add(ref total_ops_done, reads_done + writes_done + deletes_done); + } + + internal unsafe (double insPerSec, double opsPerSec, long tailAddress) Run(TestLoader testLoader) + { + Thread[] workers = new Thread[testLoader.Options.ThreadCount]; + + Console.WriteLine("Executing setup."); + + var storeWasRecovered = testLoader.MaybeRecoverStore(store); + long elapsedMs = 0; + if (!storeWasRecovered) + { + // Setup the store for the YCSB benchmark. + Console.WriteLine("Loading TsavoriteKV from data"); + for (int idx = 0; idx < testLoader.Options.ThreadCount; ++idx) + { + int x = idx; + if (testLoader.Options.UseSafeContext) + workers[idx] = new Thread(() => SetupYcsbSafeContext(x)); + else + workers[idx] = new Thread(() => SetupYcsbUnsafeContext(x)); + } + + foreach (Thread worker in workers) + worker.Start(); + + waiter.Set(); + var sw = Stopwatch.StartNew(); + foreach (Thread worker in workers) + worker.Join(); + + sw.Stop(); + waiter.Reset(); + + elapsedMs = sw.ElapsedMilliseconds; + } + double insertsPerSecond = elapsedMs == 0 ? 0 : ((double)InitCount / elapsedMs) * 1000; + Console.WriteLine(TestStats.GetLoadingTimeLine(insertsPerSecond, elapsedMs)); + Console.WriteLine(TestStats.GetAddressesLine(AddressLineNum.Before, store.Log.BeginAddress, store.Log.HeadAddress, store.Log.ReadOnlyAddress, store.Log.TailAddress)); + + if (!storeWasRecovered) + testLoader.MaybeCheckpointStore(store); + + // Uncomment below to dispose log from memory, use for 100% read workloads only + // store.Log.DisposeFromMemory(); + + idx_ = 0; + + if (testLoader.Options.DumpDistribution) + Console.WriteLine(store.DumpDistribution()); + + // Ensure first fold-over checkpoint is fast + if (testLoader.Options.PeriodicCheckpointMilliseconds > 0 && testLoader.Options.PeriodicCheckpointType == CheckpointType.FoldOver) + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, true); + + Console.WriteLine("Executing experiment."); + + // Run the experiment. + for (int idx = 0; idx < testLoader.Options.ThreadCount; ++idx) + { + int x = idx; + if (testLoader.Options.UseSafeContext) + workers[idx] = new Thread(() => RunYcsbSafeContext(x)); + else + workers[idx] = new Thread(() => RunYcsbUnsafeContext(x)); + } + + // Start threads. + foreach (Thread worker in workers) + worker.Start(); + + waiter.Set(); + var swatch = Stopwatch.StartNew(); + + if (testLoader.Options.PeriodicCheckpointMilliseconds <= 0) + { + Thread.Sleep(TimeSpan.FromSeconds(testLoader.Options.RunSeconds)); + } + else + { + var checkpointTaken = 0; + while (swatch.ElapsedMilliseconds < 1000 * testLoader.Options.RunSeconds) + { + if (checkpointTaken < swatch.ElapsedMilliseconds / testLoader.Options.PeriodicCheckpointMilliseconds) + { + long start = swatch.ElapsedTicks; + if (store.TryInitiateHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType)) + { + store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); + var timeTaken = (swatch.ElapsedTicks - start) / TimeSpan.TicksPerMillisecond; + Console.WriteLine("Checkpoint time: {0}ms", timeTaken); + checkpointTaken++; + } + } + } + Console.WriteLine($"Checkpoint taken {checkpointTaken}"); + } + + swatch.Stop(); + + done = true; + foreach (Thread worker in workers) + worker.Join(); + + waiter.Reset(); + + double seconds = swatch.ElapsedMilliseconds / 1000.0; + Console.WriteLine(TestStats.GetAddressesLine(AddressLineNum.After, store.Log.BeginAddress, store.Log.HeadAddress, store.Log.ReadOnlyAddress, store.Log.TailAddress)); + + double opsPerSecond = total_ops_done / seconds; + Console.WriteLine(TestStats.GetTotalOpsString(total_ops_done, seconds)); + Console.WriteLine(TestStats.GetStatsLine(StatsLineNum.Iteration, YcsbConstants.OpsPerSec, opsPerSecond)); + return (insertsPerSecond, opsPerSecond, store.Log.TailAddress); + } + + private void SetupYcsbUnsafeContext(int thread_idx) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (numaStyle == 0) + Native32.AffinitizeThreadRoundRobin((uint)thread_idx); + else + Native32.AffinitizeThreadShardedNuma((uint)thread_idx, 2); // assuming two NUMA sockets + } + waiter.Wait(); + + using var session = store.NewSession(functions); + var uContext = session.UnsafeContext; + uContext.BeginUnsafe(); + + Span value = stackalloc byte[kValueDataSize]; + + try + { + for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + chunk_idx < InitCount; + chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize) + { + for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) + { + if (idx % 256 == 0) + { + uContext.Refresh(); + if (idx % 65536 == 0) + uContext.CompletePending(false); + } + + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = init_keys_[idx]; + if (object_values is null) + uContext.Upsert(key, value, Empty.Default); + else + uContext.Upsert(key, object_values[idx] = new ObjectValue() { value = init_keys_[idx].value }, Empty.Default); + } + } + uContext.CompletePending(true); + } + finally + { + uContext.EndUnsafe(); + } + } + + private void SetupYcsbSafeContext(int thread_idx) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (numaStyle == 0) + Native32.AffinitizeThreadRoundRobin((uint)thread_idx); + else + Native32.AffinitizeThreadShardedNuma((uint)thread_idx, 2); // assuming two NUMA sockets + } + waiter.Wait(); + + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + Span value = stackalloc byte[kValueDataSize]; + + for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; + chunk_idx < InitCount; + chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize) + { + for (long idx = chunk_idx; idx < chunk_idx + YcsbConstants.kChunkSize; ++idx) + { + if (idx % 256 == 0) + { + bContext.Refresh(); + if (idx % 65536 == 0) + bContext.CompletePending(false); + } + + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = init_keys_[idx]; + if (object_values is null) + bContext.Upsert(key, value, Empty.Default); + else + bContext.Upsert(key, object_values[idx] = new ObjectValue() { value = init_keys_[idx].value }, Empty.Default); + } + } + + bContext.CompletePending(true); + } + + #region Load Data + + internal static void CreateKeyVectors(TestLoader testLoader, out FixedLengthKey[] i_keys, out FixedLengthKey[] t_keys) + { + InitCount = YcsbConstants.kChunkSize * (testLoader.InitCount / YcsbConstants.kChunkSize); + TxnCount = YcsbConstants.kChunkSize * (testLoader.TxnCount / YcsbConstants.kChunkSize); + + i_keys = new FixedLengthKey[InitCount]; + t_keys = new FixedLengthKey[TxnCount]; + } + + internal class KeySetter : IKeySetter + { + public void Set(FixedLengthKey[] vector, long idx, long value) => vector[idx].value = value; + } + + #endregion + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Options.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Options.cs index 375e2b6a549..e9e9cd726a5 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Options.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Options.cs @@ -11,9 +11,10 @@ class Options { [Option('b', "benchmark", Required = false, Default = 0, HelpText = "Benchmark to run:" + - "\n 0 = YCSB" + - "\n 1 = YCSB with SpanByte" + - "\n 2 = ConcurrentDictionary")] + "\n 0 = YCSB with Fixed-length (long- and int-sized) SpanByte values" + + "\n 1 = YCSB with longer SpanByte keys and values" + + "\n 2 = YCSB with longer values that may also be represented as overflow byte[] or as Object" + + "\n 3 = ConcurrentDictionary")] public int Benchmark { get; set; } [Option('t', "threads", Required = false, Default = 8, @@ -49,6 +50,10 @@ class Options HelpText = "#,#,#,#: Percentages of [(r)eads,(u)pserts,r(m)ws,(d)eletes] (summing to 100) operations in this run")] public IEnumerable RumdPercents { get; set; } + [Option("sba", Required = false, Default = false, + HelpText = "Use SpanByteAllocator (default is to use ObjectAllocator)")] + public bool UseSBA { get; set; } + [Option("reviv", Required = false, Default = RevivificationLevel.None, HelpText = "Revivification of tombstoned records:" + $"\n {nameof(RevivificationLevel.None)} = No revivification" + @@ -62,6 +67,10 @@ class Options " # (one value): All bins have this number of records, else error")] public int RevivBinRecordCount { get; set; } + [Option("di", Required = false, Default = false, + HelpText = "Delete+insert; immediately reinsert the key after deleting it")] + public bool DeleteAndReinsert { get; set; } + [Option("reviv-mutable%", Separator = ',', Required = false, Default = RevivificationSettings.DefaultRevivifiableFraction, HelpText = "Percentage of in-memory region that is eligible for revivification")] public double RevivifiableFraction { get; set; } @@ -82,6 +91,14 @@ class Options HelpText = "Use Small Memory log in experiment")] public bool UseSmallMemoryLog { get; set; } + [Option("ovf", Required = false, Default = false, + HelpText = "Use Small MaxInlineValueSize in SpanByte benchmark to test (ov)er(f)low value allocations")] + public bool UseOverflowValues { get; set; } + + [Option("obj", Required = false, Default = false, + HelpText = "Use (obj)ect values")] + public bool UseObjectValues { get; set; } + [Option("hashpack", Required = false, Default = 2.0, HelpText = "The hash table packing; divide the number of keys by this to cause hash collisions")] public double HashPacking { get; set; } @@ -91,17 +108,13 @@ class Options public bool UseSafeContext { get; set; } [Option("chkptms", Required = false, Default = 0, - HelpText = "If > 0, the number of milliseconds between checkpoints in experiment (else checkpointing is not done")] + HelpText = "If > 0, the number of milliseconds between checkpoints in experiment (else checkpointing is not done)")] public int PeriodicCheckpointMilliseconds { get; set; } [Option("chkptsnap", Required = false, Default = false, HelpText = "Use Snapshot checkpoint if doing periodic checkpoints (default is FoldOver)")] public bool PeriodicCheckpointUseSnapshot { get; set; } - [Option("chkptincr", Required = false, Default = false, - HelpText = "Try incremental checkpoint if doing periodic checkpoints")] - public bool PeriodicCheckpointTryIncremental { get; set; } - [Option("dumpdist", Required = false, Default = false, HelpText = "Dump the distribution of each non-empty bucket in the hash table")] public bool DumpDistribution { get; set; } @@ -111,11 +124,11 @@ class Options public string GetOptionsString() { static string boolStr(bool value) => value ? "y" : "n"; - return $"b: {Benchmark}; d: {DistributionName.ToLower()}; n: {NumaStyle}; rumd: {string.Join(',', RumdPercents)}; reviv: {RevivificationLevel}; revivbinrecs: {RevivBinRecordCount};" - + $" revivfrac {RevivifiableFraction}; t: {ThreadCount}; i: {IterationCount}; hp: {HashPacking};" - + $" sd: {boolStr(UseSmallData)}; sm: {boolStr(UseSmallMemoryLog)}; sy: {boolStr(UseSyntheticData)}; safectx: {boolStr(UseSafeContext)};" - + $" chkptms: {PeriodicCheckpointMilliseconds}; chkpttype: {(PeriodicCheckpointMilliseconds > 0 ? PeriodicCheckpointType.ToString() : "None")};" - + $" chkptincr: {boolStr(PeriodicCheckpointTryIncremental)}"; + var allocator = UseSBA ? "sba" : "oa"; + return $"b: {Benchmark}; a: {allocator}; d: {DistributionName.ToLower()}; n: {NumaStyle}; rumd: {string.Join(',', RumdPercents)}; reviv: {RevivificationLevel}; revivbinrecs: {RevivBinRecordCount};" + + $" revivfrac {RevivifiableFraction}; t: {ThreadCount}; i: {IterationCount}; ov: {boolStr(UseOverflowValues)}; obj: {boolStr(UseObjectValues)}; hp: {HashPacking};" + + $" sd: {boolStr(UseSmallData)}; sm: {boolStr(UseSmallMemoryLog)}; synth: {boolStr(UseSyntheticData)}; safectx: {boolStr(UseSafeContext)};" + + $" chkptms: {PeriodicCheckpointMilliseconds}; chkpttype: {(PeriodicCheckpointMilliseconds > 0 ? PeriodicCheckpointType.ToString() : "None")}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Output.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Output.cs index 45c562f6949..b2beeb8b577 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Output.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Output.cs @@ -11,6 +11,6 @@ namespace Tsavorite.benchmark public struct Output { [FieldOffset(0)] - public Value value; + public FixedLengthValue value; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Program.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Program.cs index ce329fcdb7b..ef49624ac0b 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Program.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/Program.cs @@ -3,9 +3,14 @@ using System; using System.Threading; +using Tsavorite.core; namespace Tsavorite.benchmark { +#pragma warning disable IDE0065 // Misplaced using directive + using FixedLenStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; + public class Program { const int kTrimResultCount = 3; // Use some high value like int.MaxValue to disable @@ -16,6 +21,9 @@ public static void Main(string[] args) if (testLoader.error) return; + // Output the options at the start, for easy verification (and to stop immediately if we forgot something...). + Console.WriteLine(testLoader.Options.GetOptionsString()); + TestStats testStats = new(testLoader.Options); testLoader.LoadData(); var options = testLoader.Options; // shortcut @@ -28,21 +36,42 @@ public static void Main(string[] args) switch (testLoader.BenchmarkType) { - case BenchmarkType.Ycsb: + case BenchmarkType.FixedLen: + if (options.UseSBA) + { + var tester = new FixedLenYcsbBenchmark>(testLoader.init_keys, testLoader.txn_keys, testLoader); + testStats.AddResult(tester.Run(testLoader)); + tester.Dispose(); + } + else { - var tester = new Tsavorite_YcsbBenchmark(testLoader.init_keys, testLoader.txn_keys, testLoader); + var tester = new FixedLenYcsbBenchmark>(testLoader.init_keys, testLoader.txn_keys, testLoader); testStats.AddResult(tester.Run(testLoader)); tester.Dispose(); } break; case BenchmarkType.SpanByte: + if (options.UseSBA) + { + var tester = new SpanByteYcsbBenchmark>(testLoader.init_span_keys, testLoader.txn_span_keys, testLoader, (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + testStats.AddResult(tester.Run(testLoader)); + tester.Dispose(); + } + else + { + var tester = new SpanByteYcsbBenchmark>(testLoader.init_span_keys, testLoader.txn_span_keys, testLoader, (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + testStats.AddResult(tester.Run(testLoader)); + tester.Dispose(); + } + break; + case BenchmarkType.Object: { - var tester = new SpanByteYcsbBenchmark(testLoader.init_span_keys, testLoader.txn_span_keys, testLoader); + var tester = new ObjectYcsbBenchmark(testLoader.init_keys, testLoader.txn_keys, testLoader); testStats.AddResult(tester.Run(testLoader)); tester.Dispose(); } break; - case BenchmarkType.ConcurrentDictionaryYcsb: + case BenchmarkType.ConcurrentDictionary: { var tester = new ConcurrentDictionary_YcsbBenchmark(testLoader.init_keys, testLoader.txn_keys, testLoader); testStats.AddResult(tester.Run(testLoader)); diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFixedLenFunctions.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFixedLenFunctions.cs new file mode 100644 index 00000000000..e4d904bae63 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFixedLenFunctions.cs @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Tsavorite.core; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member +#pragma warning disable IDE0130 // Namespace does not match folder structure + +namespace Tsavorite.benchmark +{ + public struct SessionFixedLenFunctions : ISessionFunctions + { + public readonly void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref Input input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) + { + } + + public readonly void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref Input input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) + { + } + + // Read functions + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool Reader(in TSourceLogRecord srcLogRecord, ref Input input, ref Output output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord + { + output.value = srcLogRecord.ValueSpan.AsRef(); + return true; + } + + public readonly bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) => true; + + public readonly bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) => true; + + // Upsert functions + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, ReadOnlySpan srcValue, ref Output output, ref UpsertInfo upsertInfo) + { + srcValue.CopyTo(logRecord.ValueSpan); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, IHeapObject srcValue, ref Output output, ref UpsertInfo upsertInfo) + => logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + + public readonly bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Input input, in TSourceLogRecord inputLogRecord, ref Output output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => true; // not used + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref Input input, ReadOnlySpan srcValue, ref Output output, ref UpsertInfo upsertInfo) + { + srcValue.CopyTo(logRecord.ValueSpan); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref Input input, IHeapObject srcValue, ref Output output, ref UpsertInfo upsertInfo) + { + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + return logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InPlaceWriter(ref LogRecord dstLogRecord, ref Input input, in TSourceLogRecord inputLogRecord, ref Output output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(dstLogRecord, inputLogRecord, ref input) }; + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + + // RMW functions + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) + { + logRecord.ValueSpan.AsRef().value = input.value; + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool InPlaceUpdater(ref LogRecord logRecord, ref Input input, ref Output output, ref RMWInfo rmwInfoo) + { + logRecord.ValueSpan.AsRef().value = input.value; + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + { + dstLogRecord.ValueSpan.AsRef().value = input.value; + return true; + } + + public readonly bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + public readonly bool NeedInitialUpdate(TKey key, ref Input input, ref Output output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => true; + + public readonly void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) { } + + public readonly bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref Input input, ref Output output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + public readonly RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref Input input) + where TSourceLogRecord : ISourceLogRecord + => GetFieldInfo(); + + /// Initial expected length of value object when populated by RMW using given input + public readonly RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref Input input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => GetFieldInfo(); + + /// Length of value object, when populated by Upsert using given value and input + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref Input input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => GetFieldInfo(); + + /// Length of value object, when populated by Upsert using given value and input + public readonly unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref Input input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => new() { KeySize = sizeof(FixedLengthKey), ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + + /// Length of value object, when populated by Upsert using given log record and input + public readonly unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref Input input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + => throw new NotImplementedException("GetUpsertFieldInfo(TSourceLogRecord)"); + + static unsafe RecordFieldInfo GetFieldInfo() => new() { KeySize = sizeof(FixedLengthKey), ValueSize = sizeof(FixedLengthValue) }; + + public readonly void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { } + + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, ReadOnlySpan srcValue, ref Output output, ref UpsertInfo upsertInfo) { } + + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, IHeapObject srcValue, ref Output output, ref UpsertInfo upsertInfo) { } + + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Input input, in TSourceLogRecord inputLogRecord, ref Output output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { } + + public readonly void PostUpsertOperation(TKey key, ref Input input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostUpsertOperation(TKey key, ref Input input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostRMWOperation(TKey key, ref Input input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + + public readonly void ConvertOutputToHeap(ref Input input, ref Output output) { } + + public void BeforeConsistentReadCallback(long hash) { } + + public void AfterConsistentReadKeyCallback() { } + + public void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) { } + + public bool AfterConsistentReadKeyBatchCallback(int keyCount) => true; + } + + static class StaticUtilities + { + public static unsafe ref T AsRef(this Span spanByte) where T : unmanaged + { + Debug.Assert(spanByte.Length == Unsafe.SizeOf()); + return ref Unsafe.As(ref spanByte[0]); + } + + public static ref readonly T AsRef(this ReadOnlySpan spanByte) where T : unmanaged + { + Debug.Assert(spanByte.Length == Unsafe.SizeOf()); + return ref MemoryMarshal.Cast(spanByte)[0]; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFunctions.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFunctions.cs deleted file mode 100644 index 19d755ea07a..00000000000 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionFunctions.cs +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.CompilerServices; -using Tsavorite.core; - -namespace Tsavorite.benchmark -{ - public struct SessionFunctions : ISessionFunctions - { - public void RMWCompletionCallback(ref Key key, ref Input input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - } - - public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - } - - // Read functions - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool SingleReader(ref Key key, ref Input input, ref Value value, ref Output dst, ref ReadInfo readInfo) - { - dst.value = value; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool ConcurrentReader(ref Key key, ref Input input, ref Value value, ref Output dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst.value = value; - return true; - } - - public bool SingleDeleter(ref Key key, ref Value value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) { value = default; return true; } - - public bool ConcurrentDeleter(ref Key key, ref Value value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostDeleteOperation(ref Key key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - // Upsert functions - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool SingleWriter(ref Key key, ref Input input, ref Value src, ref Value dst, ref Output output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - dst = src; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool ConcurrentWriter(ref Key key, ref Input input, ref Value src, ref Value dst, ref Output output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - dst = src; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostUpsertOperation(ref Key key, ref Input input, ref Value src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - // RMW functions - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool InitialUpdater(ref Key key, ref Input input, ref Value value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value.value = input.value; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool InPlaceUpdater(ref Key key, ref Input input, ref Value value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value.value += input.value; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CopyUpdater(ref Key key, ref Input input, ref Value oldValue, ref Value newValue, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue.value = input.value + oldValue.value; - return true; - } - - public bool PostCopyUpdater(ref Key key, ref Input input, ref Value oldValue, ref Value newValue, ref Output output, ref RMWInfo rmwInfo) => true; - - public bool NeedInitialUpdate(ref Key key, ref Input input, ref Output output, ref RMWInfo rmwInfo) => true; - - public void PostInitialUpdater(ref Key key, ref Input input, ref Value value, ref Output output, ref RMWInfo rmwInfo) { } - - public bool NeedCopyUpdate(ref Key key, ref Input input, ref Value oldValue, ref Output output, ref RMWInfo rmwInfo) => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostRMWOperation(ref Key key, ref Input input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - public int GetRMWModifiedValueLength(ref Value value, ref Input input) => 0; - public int GetRMWInitialValueLength(ref Input input) => 0; - public int GetUpsertValueLength(ref Value value, ref Input input) => Value.Size; - - public void PostSingleDeleter(ref Key key, ref DeleteInfo deleteInfo) { } - - public void PostSingleWriter(ref Key key, ref Input input, ref Value src, ref Value dst, ref Output output, ref UpsertInfo upsertInfo, WriteReason reason) { } - - public void ConvertOutputToHeap(ref Input input, ref Output output) { } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionObjectFunctions.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionObjectFunctions.cs new file mode 100644 index 00000000000..1ca117e5b41 --- /dev/null +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionObjectFunctions.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Tsavorite.benchmark +{ + public sealed class SessionObjectFunctions : SpanByteFunctions + { + /// + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref ReadInfo readInfo) + { + if (!srcLogRecord.Info.ValueIsObject) + { + // Copy only the first cache line for more interpretable results + srcLogRecord.ValueSpan.Slice(0, 32).CopyTo(output.SpanByte.Span); + } + else // Slice the output because it is a larger buffer + output.SpanByte.AsSpan(0, sizeof(long)).AsRef() = ((ObjectValue)srcLogRecord.ValueObject).value; + return true; + } + + // Note: Currently, only the ReadOnlySpan form of InPlaceWriter value is used here. + + /// + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration + if (!logRecord.Info.ValueIsObject) // If !ValueIsObject, the destination data length, either inline or out-of-line, should already be sufficient + srcValue.CopyTo(logRecord.ValueSpan); + else // Slice the input because it comes from a larger buffer + ((ObjectValue)logRecord.ValueObject).value = srcValue.Slice(0, FixedLengthValue.Size).AsRef().value; + return true; + } + + /// + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration + if (dstLogRecord.Info.ValueIsInline && srcValue.Length <= dstLogRecord.ValueSpan.Length) + srcValue.CopyTo(dstLogRecord.ValueSpan); + else if (!dstLogRecord.Info.ValueIsObject) // process overflow + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); + else // Slice the input because it comes from a larger buffer + ((ObjectValue)dstLogRecord.ValueObject).value = srcValue.Slice(0, FixedLengthValue.Size).AsRef().value; + return true; + } + + /// + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, IHeapObject srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration. It is called only during Setup. + return dstLogRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + } + + /// + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) => throw new TsavoriteException("InitialUpdater not implemented for YCSB"); + + /// + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + => InPlaceUpdater(ref dstLogRecord, ref input, ref output, ref rmwInfo); + + /// + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + // This does not try to set ETag or Expiration + if (!logRecord.Info.ValueIsObject) // If !ValueIsObject, the destination data length, either inline or out-of-line, should already be sufficient + input.CopyTo(logRecord.ValueSpan); + else // Slice the input because it comes from a larger buffer + ((ObjectValue)logRecord.ValueObject).value = input.ReadOnlySpan.Slice(0, FixedLengthValue.Size).AsRef().value; + return true; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionSpanByteFunctions.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionSpanByteFunctions.cs index 7d9da2ad13d..892b474f194 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionSpanByteFunctions.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SessionSpanByteFunctions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using Tsavorite.core; namespace Tsavorite.benchmark @@ -8,18 +9,48 @@ namespace Tsavorite.benchmark public sealed class SessionSpanByteFunctions : SpanByteFunctions { /// - public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref ReadInfo readInfo) { // Copy only the first cache line for more interpretable results - value.CopySliceTo(32, ref dst, memoryPool); + srcLogRecord.ValueSpan.Slice(0, 32).CopyTo(output.SpanByte.Span); return true; } - /// - public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + // Note: Currently, only the ReadOnlySpan form of Upsert value is used here. + + /// + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - // Copy only the first cache line for more interpretable results - value.CopySliceTo(32, ref dst, memoryPool); + // This does not try to set ETag or Expiration + srcValue.CopyTo(logRecord.ValueSpan); + return true; + } + + /// + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration + srcValue.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + /// + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + => throw new TsavoriteException("InitialUpdater not implemented for YCSB"); + + /// + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + // This does not try to set ETag or Expiration + input.CopyTo(dstLogRecord.ValueSpan); + return true; + } + + /// + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + { + // This does not try to set ETag or Expiration + input.CopyTo(logRecord.ValueSpan); return true; } } diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SpanByteYcsbBenchmark.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SpanByteYcsbBenchmark.cs index 9649001aa65..ea879b821a5 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SpanByteYcsbBenchmark.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/SpanByteYcsbBenchmark.cs @@ -12,9 +12,15 @@ namespace Tsavorite.benchmark { #pragma warning disable IDE0065 // Misplaced using directive - using SpanByteStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; - internal class SpanByteYcsbBenchmark + internal static class SpanByteYcsbConstants + { + internal const int kValueDataSize = 96; + } + + internal class SpanByteYcsbBenchmark + where TAllocator : IAllocator { // Ensure sizes are aligned to chunk sizes static long InitCount; @@ -31,16 +37,15 @@ internal class SpanByteYcsbBenchmark readonly KeySpanByte[] txn_keys_; readonly IDevice device; - readonly TsavoriteKV> store; + readonly TsavoriteKV store; long idx_ = 0; long total_ops_done = 0; volatile bool done = false; - internal const int kKeySize = 16; - internal const int kValueSize = 100; + internal const int kValueDataSize = SpanByteYcsbConstants.kValueDataSize; - internal SpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys_, TestLoader testLoader) + internal SpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys_, TestLoader testLoader, Func allocatorFactory) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { @@ -73,7 +78,7 @@ internal SpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys_, Tes [ new RevivificationBin() { - RecordSize = RecordInfo.GetLength() + kKeySize + kValueSize + 8, // extra to ensure rounding up of value + RecordSize = RecordInfo.Size + KeySpanByte.DataSize + kValueDataSize + 8, // extra to ensure rounding up of value NumberOfRecords = testLoader.Options.RevivBinRecordCount, BestFitScanLimit = RevivificationBin.UseFirstFit } @@ -90,12 +95,12 @@ internal SpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys_, Tes device = Devices.CreateLogDevice(TestLoader.DevicePath, preallocateFile: true, deleteOnClose: !testLoader.RecoverMode, useIoCompletionPort: true); - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = testLoader.GetHashTableSize(), LogDevice = device, PreallocateLog = true, - MemorySize = 1L << 35, + LogMemorySize = 1L << 35, RevivificationSettings = revivificationSettings, CheckpointDir = testLoader.BackupPath }; @@ -104,12 +109,12 @@ internal SpanByteYcsbBenchmark(KeySpanByte[] i_keys_, KeySpanByte[] t_keys_, Tes { kvSettings.PageSize = 1L << 22; kvSettings.SegmentSize = 1L << 26; - kvSettings.MemorySize = 1L << 26; + kvSettings.LogMemorySize = 1L << 26; } store = new(kvSettings - , StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + , StoreFunctions.Create(SpanByteComparer.Instance, new SpanByteRecordTriggers()) + , allocatorFactory ); } @@ -134,19 +139,19 @@ private void RunYcsbUnsafeContext(int thread_idx) var sw = Stopwatch.StartNew(); - Span value = stackalloc byte[kValueSize]; - Span input = stackalloc byte[kValueSize]; - Span output = stackalloc byte[kValueSize]; + Span value = stackalloc byte[kValueDataSize]; + Span input = stackalloc byte[kValueDataSize]; + Span output = stackalloc byte[kValueDataSize]; - SpanByte _value = SpanByte.FromPinnedSpan(value); - SpanByte _input = SpanByte.FromPinnedSpan(input); + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory _output = SpanByteAndMemory.FromPinnedSpan(output); long reads_done = 0; long writes_done = 0; long deletes_done = 0; - using var session = store.NewSession(functions); + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); var uContext = session.UnsafeContext; uContext.BeginUnsafe(); @@ -170,28 +175,35 @@ private void RunYcsbUnsafeContext(int thread_idx) uContext.CompletePending(false); } - ref var key = ref SpanByte.Reinterpret(ref txn_keys_[idx]); - int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 - if (r < readPercent) - { - uContext.Read(ref key, ref _input, ref _output, Empty.Default); - ++reads_done; - continue; - } - if (r < upsertPercent) - { - uContext.Upsert(ref key, ref _value, Empty.Default); - ++writes_done; - continue; - } - if (r < rmwPercent) + unsafe { - uContext.RMW(ref key, ref _input, Empty.Default); - ++writes_done; - continue; + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = txn_keys_[idx]; + + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) + { + uContext.Read(key, ref pinnedInputSpan, ref _output, Empty.Default); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + uContext.Upsert(key, value, Empty.Default); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + uContext.RMW(key, ref pinnedInputSpan, Empty.Default); + ++writes_done; + continue; + } + uContext.Delete(key, Empty.Default); + if (di) + uContext.Upsert(key, value, Empty.Default); + ++deletes_done; } - uContext.Delete(ref key, Empty.Default); - ++deletes_done; } } @@ -223,19 +235,19 @@ private void RunYcsbSafeContext(int thread_idx) var sw = Stopwatch.StartNew(); - Span value = stackalloc byte[kValueSize]; - Span input = stackalloc byte[kValueSize]; - Span output = stackalloc byte[kValueSize]; + Span value = stackalloc byte[kValueDataSize]; + Span input = stackalloc byte[kValueDataSize]; + Span output = stackalloc byte[kValueDataSize]; - SpanByte _value = SpanByte.FromPinnedSpan(value); - SpanByte _input = SpanByte.FromPinnedSpan(input); + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory _output = SpanByteAndMemory.FromPinnedSpan(output); long reads_done = 0; long writes_done = 0; long deletes_done = 0; - using var session = store.NewSession(functions); + var di = testLoader.Options.DeleteAndReinsert; + using var session = store.NewSession(functions); var bContext = session.BasicContext; while (!done) @@ -257,27 +269,35 @@ private void RunYcsbSafeContext(int thread_idx) bContext.CompletePending(false); } - int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 - if (r < readPercent) - { - bContext.Read(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, ref _output, Empty.Default); - ++reads_done; - continue; - } - if (r < upsertPercent) + unsafe { - bContext.Upsert(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _value, Empty.Default); - ++writes_done; - continue; - } - if (r < rmwPercent) - { - bContext.RMW(ref SpanByte.Reinterpret(ref txn_keys_[idx]), ref _input, Empty.Default); - ++writes_done; - continue; + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + var key = txn_keys_[idx]; + + int r = (int)rng.Generate(100); // rng.Next() is not inclusive of the upper bound so this will be <= 99 + if (r < readPercent) + { + bContext.Read(key, ref pinnedInputSpan, ref _output, Empty.Default); + ++reads_done; + continue; + } + if (r < upsertPercent) + { + bContext.Upsert(key, value, Empty.Default); + ++writes_done; + continue; + } + if (r < rmwPercent) + { + bContext.RMW(key, ref pinnedInputSpan, Empty.Default); + ++writes_done; + continue; + } + bContext.Delete(key, Empty.Default); + if (di) + bContext.Upsert(key, value, Empty.Default); + ++deletes_done; } - bContext.Delete(ref SpanByte.Reinterpret(ref txn_keys_[idx]), Empty.Default); - ++deletes_done; } } @@ -373,7 +393,7 @@ internal unsafe (double insPerSec, double opsPerSec, long tailAddress) Run(TestL if (checkpointTaken < swatch.ElapsedMilliseconds / testLoader.Options.PeriodicCheckpointMilliseconds) { long start = swatch.ElapsedTicks; - if (store.TryInitiateHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType, testLoader.Options.PeriodicCheckpointTryIncremental)) + if (store.TryInitiateHybridLogCheckpoint(out _, testLoader.Options.PeriodicCheckpointType)) { store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); var timeTaken = (swatch.ElapsedTicks - start) / TimeSpan.TicksPerMillisecond; @@ -413,12 +433,11 @@ private void SetupYcsbUnsafeContext(int thread_idx) } waiter.Wait(); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var uContext = session.UnsafeContext; uContext.BeginUnsafe(); - Span value = stackalloc byte[kValueSize]; - ref SpanByte _value = ref SpanByte.Reinterpret(value); + Span value = stackalloc byte[kValueDataSize]; try { @@ -431,14 +450,12 @@ private void SetupYcsbUnsafeContext(int thread_idx) if (idx % 256 == 0) { uContext.Refresh(); - if (idx % 65536 == 0) - { uContext.CompletePending(false); - } } - uContext.Upsert(ref SpanByte.Reinterpret(ref init_keys_[idx]), ref _value, Empty.Default); + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + _ = uContext.Upsert(init_keys_[idx], value, Empty.Default); } } uContext.CompletePending(true); @@ -460,11 +477,10 @@ private void SetupYcsbSafeContext(int thread_idx) } waiter.Wait(); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var bContext = session.BasicContext; - Span value = stackalloc byte[kValueSize]; - ref SpanByte _value = ref SpanByte.Reinterpret(value); + Span value = stackalloc byte[kValueDataSize]; for (long chunk_idx = Interlocked.Add(ref idx_, YcsbConstants.kChunkSize) - YcsbConstants.kChunkSize; chunk_idx < InitCount; @@ -475,22 +491,18 @@ private void SetupYcsbSafeContext(int thread_idx) if (idx % 256 == 0) { bContext.Refresh(); - if (idx % 65536 == 0) - { bContext.CompletePending(false); - } } - bContext.Upsert(ref SpanByte.Reinterpret(ref init_keys_[idx]), ref _value, Empty.Default); + // The key vectors are not pinned, but we use only (ReadOnly)Span operations in SessionSpanByteFunctions and key compare. + _ = bContext.Upsert(init_keys_[idx], value, Empty.Default); } } bContext.CompletePending(true); } - #region Load Data - internal static void CreateKeyVectors(TestLoader testLoader, out KeySpanByte[] i_keys, out KeySpanByte[] t_keys) { InitCount = YcsbConstants.kChunkSize * (testLoader.InitCount / YcsbConstants.kChunkSize); @@ -499,16 +511,9 @@ internal static void CreateKeyVectors(TestLoader testLoader, out KeySpanByte[] i i_keys = new KeySpanByte[InitCount]; t_keys = new KeySpanByte[TxnCount]; } - - internal class KeySetter : IKeySetter - { - public unsafe void Set(KeySpanByte[] vector, long idx, long value) - { - vector[idx].length = kKeySize - 4; - vector[idx].value = value; - } - } - - #endregion + } + internal class SpanByteYcsbKeySetter : IKeySetter + { + public void Set(KeySpanByte[] vector, long idx, long value) => vector[idx].value = value; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestLoader.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestLoader.cs index 1b7da927d8b..fe040a586e0 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestLoader.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestLoader.cs @@ -13,6 +13,10 @@ namespace Tsavorite.benchmark { +#pragma warning disable IDE0065 // Misplaced using directive + using FixedLenStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; + internal interface IKeySetter { void Set(TKey[] vector, long idx, long value); @@ -22,8 +26,8 @@ class TestLoader { internal readonly Options Options; internal readonly string Distribution; - internal Key[] init_keys = default; - internal Key[] txn_keys = default; + internal FixedLengthKey[] init_keys = default; + internal FixedLengthKey[] txn_keys = default; internal KeySpanByte[] init_span_keys = default; internal KeySpanByte[] txn_span_keys = default; @@ -77,6 +81,21 @@ static bool verifyOption(bool isValid, string name, string info = null) if (!verifyOption(rumdPercents.Length == 4 && Options.RumdPercents.Sum() == 100 && !Options.RumdPercents.Any(x => x < 0), "rmud", "Percentages of [(r)eads,(u)pserts,r(m)ws,(d)eletes] must be empty or must sum to 100 with no negative elements")) return; + if (Options.UseOverflowValues && Options.UseObjectValues) + { + Console.WriteLine($"Cannot specify both UseOverflowValues and UseObjectValues"); + return; + } + if ((Options.UseOverflowValues || Options.UseObjectValues) && BenchmarkType != BenchmarkType.Object) + { + Console.WriteLine($"Can only specify UseOverflowValues or UseObjectValues with BenchmarkType.Object"); + return; + } + if (Options.UseSBA && BenchmarkType == BenchmarkType.Object) + { + Console.WriteLine($"SpanByteAllocator is not supported with BenchmarkType.Object"); + return; + } ReadPercent = rumdPercents[0]; UpsertPercent = ReadPercent + rumdPercents[1]; RmwPercent = UpsertPercent + rumdPercents[2]; @@ -106,15 +125,25 @@ private void LoadDataThreadProc() switch (BenchmarkType) { - case BenchmarkType.Ycsb: - Tsavorite_YcsbBenchmark.CreateKeyVectors(this, out init_keys, out txn_keys); - LoadData(this, init_keys, txn_keys, new Tsavorite_YcsbBenchmark.KeySetter()); + case BenchmarkType.FixedLen: + if (Options.UseSBA) + FixedLenYcsbBenchmark>.CreateKeyVectors(this, out init_keys, out txn_keys); + else + FixedLenYcsbBenchmark>.CreateKeyVectors(this, out init_keys, out txn_keys); + LoadData(this, init_keys, txn_keys, new FixedLenYcsbKeySetter()); break; case BenchmarkType.SpanByte: - SpanByteYcsbBenchmark.CreateKeyVectors(this, out init_span_keys, out txn_span_keys); - LoadData(this, init_span_keys, txn_span_keys, new SpanByteYcsbBenchmark.KeySetter()); + if (Options.UseSBA) + SpanByteYcsbBenchmark>.CreateKeyVectors(this, out init_span_keys, out txn_span_keys); + else + SpanByteYcsbBenchmark>.CreateKeyVectors(this, out init_span_keys, out txn_span_keys); + LoadData(this, init_span_keys, txn_span_keys, new SpanByteYcsbKeySetter()); + break; + case BenchmarkType.Object: + ObjectYcsbBenchmark.CreateKeyVectors(this, out init_keys, out txn_keys); + LoadData(this, init_keys, txn_keys, new ObjectYcsbBenchmark.KeySetter()); break; - case BenchmarkType.ConcurrentDictionaryYcsb: + case BenchmarkType.ConcurrentDictionary: ConcurrentDictionary_YcsbBenchmark.CreateKeyVectors(this, out init_keys, out txn_keys); LoadData(this, init_keys, txn_keys, new ConcurrentDictionary_YcsbBenchmark.KeySetter()); break; @@ -340,9 +369,9 @@ private static void LoadSyntheticData(string distribution, uin internal string BackupPath => $"{DataPath}/{Distribution}_{(Options.UseSyntheticData ? "synthetic" : "ycsb")}_{(Options.UseSmallData ? "2.5M_10M" : "250M_1000M")}"; - internal bool MaybeRecoverStore(TsavoriteKV store) - where SF : IStoreFunctions - where A : IAllocator + internal bool MaybeRecoverStore(TsavoriteKV store) + where SF : IStoreFunctions + where A : IAllocator { // Recover database for fast benchmark repeat runs. if (RecoverMode) @@ -371,9 +400,9 @@ internal bool MaybeRecoverStore(TsavoriteKV store) return false; } - internal void MaybeCheckpointStore(TsavoriteKV store) - where SF : IStoreFunctions - where A : IAllocator + internal void MaybeCheckpointStore(TsavoriteKV store) + where SF : IStoreFunctions + where A : IAllocator { // Checkpoint database for fast benchmark repeat runs. if (RecoverMode) diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestStats.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestStats.cs index 52837908d58..adb6707aa9e 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestStats.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/TestStats.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using Tsavorite.core; using static Tsavorite.benchmark.YcsbConstants; namespace Tsavorite.benchmark @@ -57,7 +58,7 @@ internal void ShowAllStats(AggregateType aggregateType, string discardMessage = AggregateType.FinalTrimmed => StatsLineNum.FinalTrimmedTail, _ => throw new InvalidOperationException("Unknown AggregateType") }; - ShowStats(statsLineNum, "TailAddress", tailAddresses); + ShowStats(statsLineNum, "TailAddress (abs)", tailAddresses); } private void ShowStats(StatsLineNum lineNum, string tag, List vec) @@ -105,7 +106,7 @@ internal static string GetLoadingTimeLine(double insertsPerSec, long elapsedMs) => $"##00; {InsPerSec}: {insertsPerSec:N2}; sec: {(double)elapsedMs / 1000:N3}"; internal static string GetAddressesLine(AddressLineNum lineNum, long begin, long head, long rdonly, long tail) - => $"##{(int)lineNum:00}; begin: {begin}; head: {head}; readonly: {rdonly}; tail: {tail}"; + => $"##{(int)lineNum:00}; begin: {LogAddress.AddressString(begin)}; head: {LogAddress.AddressString(head)}; readonly: {LogAddress.AddressString(rdonly)}; tail: {LogAddress.AddressString(tail)}"; internal static string GetStatsLine(StatsLineNum lineNum, string opsPerSecTag, double opsPerSec) => $"##{(int)lineNum:00}; {opsPerSecTag}: {opsPerSec:N2}; {OptionsString}"; diff --git a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/YcsbConstants.cs b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/YcsbConstants.cs index 15da29c3e2c..104da089b3b 100644 --- a/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/YcsbConstants.cs +++ b/libs/storage/Tsavorite/cs/benchmark/YCSB.benchmark/YcsbConstants.cs @@ -7,9 +7,10 @@ namespace Tsavorite.benchmark { enum BenchmarkType : byte { - Ycsb = 0, + FixedLen = 0, SpanByte, - ConcurrentDictionaryYcsb + Object, + ConcurrentDictionary }; enum AddressLineNum : int @@ -58,7 +59,6 @@ public static class YcsbConstants internal const string OpsPerSec = "ops/sec"; internal const CheckpointType kPeriodicCheckpointType = CheckpointType.FoldOver; - internal const bool kPeriodicCheckpointTryIncremental = false; internal const double SyntheticZipfTheta = 0.99; diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs index c95732f8f36..5cd52a5da22 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs @@ -3,32 +3,66 @@ using System; using System.Diagnostics; -using System.IO; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using static Tsavorite.core.Utility; namespace Tsavorite.core { +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + + /// + /// Type-free base class for hybrid log memory allocator. Contains utility methods that do not need type args and are not performance-critical + /// so can be virtual. + /// + public abstract class AllocatorBase + { + /// Create the circular buffers for flushing to device. Only implemented by ObjectAllocator. + internal virtual CircularDiskWriteBuffer CreateCircularFlushBuffers(IDevice objectLogDevice, ILogger logger) => default; + /// Create the circular flush buffers for object deserialization from device. Only implemented by ObjectAllocator. + internal virtual CircularDiskReadBuffer CreateCircularReadBuffers(IDevice objectLogDevice, ILogger logger) => default; + /// Create the circular flush buffers for object deserialization from device. Only implemented by ObjectAllocator. + internal virtual CircularDiskReadBuffer CreateCircularReadBuffers() => default; + + /// Returns the lowest segment in use in the object log; will be zero unless the database has been truncated. + internal virtual int LowestObjectLogSegmentInUse => 0; + /// Get the ObjectLog tail position, if this is ObjectAllocator. + internal virtual ObjectLogFilePositionInfo GetObjectLogTail() => new(); // This marks it as "unset" + /// Set the ObjectLog tail position, if this is ObjectAllocator. + internal virtual void SetObjectLogTail(ObjectLogFilePositionInfo tail) { } + } + /// /// Base class for hybrid log memory allocator. Contains utility methods, some of which are not performance-critical so can be virtual. /// - public abstract partial class AllocatorBase : IDisposable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public abstract unsafe partial class AllocatorBase : AllocatorBase, IDisposable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// The epoch we are operating with - protected readonly LightEpoch epoch; + internal readonly LightEpoch epoch; /// Whether we own (and thus must dispose) private readonly bool isEpochOwned; /// The store functions for this instance of TsavoriteKV - internal readonly TStoreFunctions _storeFunctions; + internal readonly TStoreFunctions storeFunctions; /// The fully-derived allocator struct wrapper (so calls on it are inlined rather than virtual) for this log. internal readonly TAllocator _wrapper; + /// The to hold the objects for transient instances. + internal ObjectIdMap transientObjectIdMap; + + /// Sometimes it's useful to know this explicitly rather than rely on method overrides etc. + internal bool IsObjectAllocator => transientObjectIdMap is not null; + + /// If true, then this allocator has as the first bytes on a page, so allocating a logical address + /// in must skip these bytes. + internal int pageHeaderSize; + #region Protected size definitions /// Buffer size internal readonly int BufferSize; @@ -45,38 +79,27 @@ public abstract partial class AllocatorBaseBuffer size mask protected readonly int BufferSizeMask; - /// Aligned page size in bytes + /// Aligned (to sector size) page size in bytes protected readonly int AlignedPageSizeBytes; - /// Total hybrid log size (bits) - protected readonly int LogTotalSizeBits; - - /// Total hybrid log size (bytes) - protected readonly long LogTotalSizeBytes; - /// Segment size in bits protected readonly int LogSegmentSizeBits; /// Segment size protected readonly long SegmentSize; - /// Segment buffer size - protected readonly int SegmentBufferSize; - - /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) - private int emptyPageCount; + /// Log mutable fraction + internal readonly double logMutableFraction; - /// Minimum number of empty pages in circular buffer to be maintained to account for non-power-of-two size - private int minEmptyPageCount; + /// Circular buffer definition + /// The long is actually a byte*, but storing as 'long' makes going through logicalAddress/physicalAddress translation more easily + protected long* pagePointers; - /// HeadAddress offset from tail (currently page-aligned) - internal long HeadAddressLagOffset; + /// Array of longs which are actually the byte* of the parallel index in + private long[] pagePointersArray; - /// Log mutable fraction - protected readonly double LogMutableFraction; - - /// ReadOnlyAddress offset from tail (currently page-aligned) - protected long ReadOnlyAddressLagOffset; + /// Array of pages kept to ensure the pinned pages are not garbage collected. + protected readonly byte[][] pageArrays; #endregion @@ -84,7 +107,9 @@ public abstract partial class AllocatorBaseThe maximum address of the immutable in-memory log region public long ReadOnlyAddress; - /// Safe read-only address + /// The lowest fuzzy mutable address. This is set by OnPagesMarkedReadOnly as the address to which we are setting the + /// prior to actually doing the flushes. If it is less than then it + /// is the low address of the "fuzzy region" ( is the high address of the "fuzzy region"). public long SafeReadOnlyAddress; /// @@ -111,18 +136,41 @@ public abstract partial class AllocatorBaseThe lowest valid address in the log public long BeginAddress; - /// The lowest valid address on disk - updated when truncating log - public long PersistedBeginAddress; - /// /// Address until which we are currently closing. Used to coordinate linear closing of pages. /// Only one thread will be closing pages at a time. /// long OngoingCloseUntilAddress; + /// + /// True while is rebuilding allocator state (page free + per-allocator + /// ). Operations that want to be resilient to a concurrent Reset + /// can observe this flag (after acquiring epoch protection) and bail out — Reset is a + /// wholesale wipe, so any pre-Reset work or cached state is no longer meaningful. Scan + /// iterators do exactly this: they terminate the iteration (return false from + /// GetNext) when this flag is set. + /// + /// IMPORTANT: this flag is opt-in defense. Tsavorite's hot-path RMW / Read / Upsert / + /// Delete do NOT consult it (cost on the hot path), so they remain unsafe to call + /// concurrently with Reset and can dereference freed pages mid-Initialize. Callers of + /// Reset must still quiesce all non-iterator operations on the store (per Reset's + /// docstring contract). For Garnet, VectorManager.PauseCleanupAsync serializes + /// the cleanup task — including its post-iterate RMWs — with Reset. + /// + internal volatile bool Initializing; + /// - public override string ToString() - => $"TA {GetTailAddress()}, ROA {ReadOnlyAddress}, SafeROA {SafeReadOnlyAddress}, HA {HeadAddress}, SafeHA {SafeHeadAddress}, CUA {ClosedUntilAddress}, FUA {FlushedUntilAddress}, BA {BeginAddress}"; + public override string ToString() => BaseToString(); + + protected string BaseToString(string fuaDetails = "") + { + var tailAddress = UnstableGetTailAddress(out var isUnstable); + var unstableTailStr = isUnstable ? "(u)" : ""; + return $"TA {AddressString(tailAddress)}{unstableTailStr}, ROA {AddressString(ReadOnlyAddress)}, SafeROA {AddressString(SafeReadOnlyAddress)}, HA {AddressString(HeadAddress)}," + + $" SafeHA {AddressString(SafeHeadAddress)}, CUA {AddressString(ClosedUntilAddress)}," + + $" FUA {AddressString(FlushedUntilAddress)}{fuaDetails}," + + $" BA {AddressString(BeginAddress)}, PgSz {PageSize}, BufSz {BufferSize}, APC {AllocatedPageCount}, MAPC {MaxAllocatedPageCount}"; + } #endregion #region Protected device info @@ -140,7 +188,7 @@ public override string ToString() internal readonly PendingFlushList[] PendingFlush; /// Global address of the current tail (next element to be allocated from the circular buffer) - private PageOffset TailPageOffset; + internal PageOffset TailPageOffset; /// Whether log is disposed private bool disposed = false; @@ -154,9 +202,7 @@ public override string ToString() /// Buffer pool internal SectorAlignedBufferPool bufferPool; - /// This hlog is an instance of a Read cache - protected readonly bool IsReadCache = false; - + /// Address type for this hlog's records' /// Read cache eviction callback protected readonly Action EvictCallback = null; @@ -170,26 +216,31 @@ public override string ToString() private readonly ErrorList errorList = new(); /// Observer for records entering read-only region - internal IObserver> OnReadOnlyObserver; + internal IObserver onReadOnlyObserver; - /// Observer for records getting evicted from memory (page closed) - internal IObserver> OnEvictionObserver; + /// Observer for records getting evicted from memory (page closed). May be the same object as . + internal IObserver onEvictionObserver; - /// Observer for records brought into memory by deserializing pages - internal IObserver> OnDeserializationObserver; + /// + /// Whether this allocator is the read cache (as opposed to the main hybrid log). + /// Set once at construction from . + /// + internal readonly bool IsReadCache; + + /// Log size tracker; called when an operation at the Tsavorite-internal level adds or removes heap memory size + /// (e.g. copying to log tail or read cache, which do not call ). + /// May be the same object as . + internal LogSizeTracker logSizeTracker; /// The "event" to be waited on for flush completion by the initiator of an operation - internal CompletionEvent FlushEvent; + internal CompletionEvent flushEvent; /// If set, this is a function to call to determine whether the object size tracker reports maximum memory size has been exceeded. public Func IsSizeBeyondLimit; #endregion #region Abstract and virtual methods - /// Initialize fully derived allocator - public abstract void Initialize(); - - /// Write async to device + /// Write async to device for snapshot checkpoint /// /// /// @@ -198,155 +249,166 @@ public override string ToString() /// /// /// - /// /// Start address of fuzzy region, which contains old and new version records (we use this to selectively flush only old-version records during snapshot checkpoint) - protected abstract void WriteAsyncToDevice(long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, PageAsyncFlushResult result, IDevice device, IDevice objectLogDevice, long[] localSegmentOffsets, long fuzzyStartLogicalAddress); - - /// Read objects to memory (async) - protected abstract unsafe void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default); + protected abstract void WriteAsyncToDeviceForSnapshot(long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, + PageAsyncFlushResult result, IDevice device, IDevice objectLogDevice, long fuzzyStartLogicalAddress); /// Read page from device (async) - protected abstract void ReadAsync(ulong alignedSourceAddress, int destinationPageIndex, uint aligned_read_length, DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device, IDevice objlogDevice); + protected abstract void ReadAsync(ulong alignedSourceAddress, IntPtr destinationPtr, uint aligned_read_length, + DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device); /// Write page to device (async) protected abstract void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult); - /// Flush checkpoint Delta to the Device - internal virtual unsafe void AsyncFlushDeltaToDevice(long startAddress, long endAddress, long prevEndAddress, long version, DeltaLog deltaLog, out SemaphoreSlim completedSemaphore, int throttleCheckpointFlushDelayMs) + /// + /// Reset the hybrid log to empty. + /// + /// Concurrent-safety contract: + /// * SCAN ITERATORS are safe end-to-end. The two-phase epoch cascade below + /// (PR #1765) protects the page-free section, and the + /// flag (set across this whole method including the per-allocator + /// rewind) lets iterators terminate cleanly during the + /// post-Phase-2 non-monotonic Initialize that would otherwise expose freed + /// pagePointers. + /// * RMW / Read / Upsert / Delete are NOT safe — Tsavorite's hot paths do not + /// consult . They can race with Initialize and + /// dereference freed pages. Callers MUST quiesce all non-iterator operations + /// on the store before invoking Reset (per the original docstring contract). + /// For Garnet, VectorManager.PauseCleanupAsync serializes the cleanup + /// task — including its post-iterate RMWs — with Reset. + /// + /// Phase breakdown (executed by ): + /// + /// Phase 1: publish new ReadOnlyAddress synchronously, then under + /// BumpCurrentEpoch — i.e. after writers caching the OLD ReadOnlyAddress + /// have drained — publish SafeReadOnlyAddress and FlushedUntilAddress. + /// Mirrors OnPagesMarkedReadOnly's invariant that "by the time + /// SafeReadOnlyAddress advances, no thread is mutating below it". + /// + /// Phase 2: publish new HeadAddress synchronously (now safe — writers have observed + /// the new ReadOnlyAddress, so no writer holds a cached old ReadOnlyAddress + /// that would leave HeadAddress > cached ReadOnlyAddress). Then under + /// BumpCurrentEpoch — i.e. after readers caching the OLD HeadAddress have + /// drained — close pages (advancing SafeHeadAddress and ClosedUntilAddress) + /// and free pages. Mirrors OnPagesClosed's invariant. + /// + /// Final: publish new BeginAddress synchronously. Publishing it last (rather than + /// up front) means an iterator with a stale nextAddress sees + /// currentAddress > OLD BeginAddress and does not snap forward into the + /// just-freed in-memory range — instead the currentAddress < NEW HeadAddress + /// check routes it through LoadPageIfNeeded's disk-frame branch (frame is + /// iterator-owned, disk segment is intact). The invariant + /// BeginAddress <= HeadAddress holds throughout. + /// + /// Then per-allocator runs (re-allocates pages 0/1, rewinds + /// addresses to FirstValidAddress). The whole sequence is wrapped in + /// Initializing = true / false so iterators that opt in to checking the flag + /// terminate during the rewind window. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + public void Reset() { - logger?.LogTrace("Starting async delta log flush with throttling {throttlingEnabled}", throttleCheckpointFlushDelayMs >= 0 ? $"enabled ({throttleCheckpointFlushDelayMs}ms)" : "disabled"); - - var _completedSemaphore = new SemaphoreSlim(0); - completedSemaphore = _completedSemaphore; - - if (throttleCheckpointFlushDelayMs >= 0) - _ = Task.Run(FlushRunner); - else - FlushRunner(); + // Gate Initializing-aware operations (e.g., scan iterators) for the entire + // reset+initialize sequence. Cleared in finally so a throw cannot leave them + // permanently terminating their next call. + Initializing = true; + try + { + ResetCore(); - void FlushRunner() + // Per-allocator Initialize re-allocates pages 0/1, resets all addresses to + // FirstValidAddress, and resets TailPageOffset. This non-monotonically rewinds + // multiple fields and is unsafe for concurrent operations — that's exactly + // what the Initializing flag guards iterator-aware callers against. + Initialize(); + } + finally { - long startPage = GetPage(startAddress); - long endPage = GetPage(endAddress); - if (endAddress > _wrapper.GetStartLogicalAddress(endPage)) - endPage++; + // Volatile.Write semantics (the field is volatile) ensure all our state + // mutations are visible BEFORE callers observe Initializing == false. + Initializing = false; + } + } - long prevEndPage = GetPage(prevEndAddress); - deltaLog.Allocate(out int entryLength, out long destPhysicalAddress); - int destOffset = 0; + [MethodImpl(MethodImplOptions.NoInlining)] + private void ResetCore() + { + var newBeginAddress = GetTailAddress(); - // We perform delta capture under epoch protection with page-wise refresh for latency reasons - bool epochTaken = false; - if (!epoch.ThisInstanceProtected()) - { - epochTaken = true; - epoch.Resume(); - } + // To use BumpCurrentEpoch we must be epoch-protected; conversely to wait for the + // queued action to drain we must NOT be holding the prior epoch. We toggle the + // protection per phase. If the caller arrived already protected, restore at the end. + var wasProtected = epoch.ThisInstanceProtected(); + if (wasProtected) + epoch.Suspend(); + // -------- Phase 1: ReadOnly -> wait for writer drain -> SafeReadOnly + FlushedUntil -------- + _ = MonotonicUpdate(ref ReadOnlyAddress, newBeginAddress, out _); + + using (var phase1Done = new ManualResetEventSlim(initialState: false)) + { + epoch.Resume(); try { - for (long p = startPage; p < endPage; p++) + epoch.BumpCurrentEpoch(() => { - // Check if we have the page safely available to process in memory - if (HeadAddress >= (p << LogPageSizeBits) + PageSize) - continue; - - // All RCU pages need to be added to delta - // For IPU-only pages, prune based on dirty bit - if ((p < prevEndPage || endAddress == prevEndAddress) && PageStatusIndicator[p % BufferSize].Dirty < version) - continue; - - var logicalAddress = p << LogPageSizeBits; - var physicalAddress = _wrapper.GetPhysicalAddress(logicalAddress); - - var endLogicalAddress = logicalAddress + PageSize; - if (endAddress < endLogicalAddress) endLogicalAddress = endAddress; - Debug.Assert(endLogicalAddress > logicalAddress); - var endPhysicalAddress = physicalAddress + (endLogicalAddress - logicalAddress); - - if (p == startPage) + try { - physicalAddress += (int)(startAddress & PageSizeMask); - logicalAddress += (int)(startAddress & PageSizeMask); + _ = MonotonicUpdate(ref SafeReadOnlyAddress, newBeginAddress, out _); + _ = MonotonicUpdate(ref FlushedUntilAddress, newBeginAddress, out _); } - - while (physicalAddress < endPhysicalAddress) - { - ref var info = ref _wrapper.GetInfo(physicalAddress); - var (_, alignedRecordSize) = _wrapper.GetRecordSize(physicalAddress); - if (info.Dirty) - { - info.ClearDirtyAtomic(); // there may be read locks being taken, hence atomic - int size = sizeof(long) + sizeof(int) + alignedRecordSize; - if (destOffset + size > entryLength) - { - deltaLog.Seal(destOffset); - deltaLog.Allocate(out entryLength, out destPhysicalAddress); - destOffset = 0; - if (destOffset + size > entryLength) - { - deltaLog.Seal(0); - deltaLog.Allocate(out entryLength, out destPhysicalAddress); - } - if (destOffset + size > entryLength) - throw new TsavoriteException("Insufficient page size to write delta"); - } - *(long*)(destPhysicalAddress + destOffset) = logicalAddress; - destOffset += sizeof(long); - *(int*)(destPhysicalAddress + destOffset) = alignedRecordSize; - destOffset += sizeof(int); - Buffer.MemoryCopy((void*)physicalAddress, (void*)(destPhysicalAddress + destOffset), alignedRecordSize, alignedRecordSize); - destOffset += alignedRecordSize; - } - physicalAddress += alignedRecordSize; - logicalAddress += alignedRecordSize; - } - epoch.ProtectAndDrain(); - } - } - finally - { - if (epochTaken) - epoch.Suspend(); + finally { phase1Done.Set(); } + }); } - - if (destOffset > 0) - deltaLog.Seal(destOffset); - _completedSemaphore.Release(); + finally { epoch.Suspend(); } + phase1Done.Wait(); } - } - - /// Delete in-memory portion of the log - internal abstract void DeleteFromMemory(); - /// Reset the hybrid log. WARNING: assumes that threads have drained out at this point. - public virtual void Reset() - { - var newBeginAddress = GetTailAddress(); + // -------- Phase 2: HeadAddress -> wait for reader drain -> OnPagesClosed + FreeAllPages -------- + var headShifted = MonotonicUpdate(ref HeadAddress, newBeginAddress, out _); - // Shift read-only addresses to tail without flushing - _ = Utility.MonotonicUpdate(ref ReadOnlyAddress, newBeginAddress, out _); - _ = Utility.MonotonicUpdate(ref SafeReadOnlyAddress, newBeginAddress, out _); - - // Shift head address to tail - if (Utility.MonotonicUpdate(ref HeadAddress, newBeginAddress, out _)) + using (var phase2Done = new ManualResetEventSlim(initialState: false)) { - // Close addresses - OnPagesClosed(newBeginAddress); - - // Wait for pages to get closed - while (ClosedUntilAddress < newBeginAddress) + epoch.Resume(); + try { - _ = Thread.Yield(); - if (epoch.ThisInstanceProtected()) - epoch.ProtectAndDrain(); + epoch.BumpCurrentEpoch(() => + { + try + { + if (headShifted) + OnPagesClosed(newBeginAddress); + + // Wait for ClosedUntilAddress to catch up to newBeginAddress before + // freeing remaining pages. Two scenarios make this necessary: + // (a) headShifted==true: OnPagesClosed may have returned immediately + // because another thread already owned OnPagesClosedWorker for our + // range — that worker is still freeing pages on the other thread. + // (b) headShifted==false: a concurrent Reset (or other ShiftHeadAddress + // caller) already advanced HeadAddress past newBeginAddress and its + // OnPagesClosedWorker may still be running. + // In both cases, calling FreeAllAllocatedPages while the worker is mid-flight + // would race with its FreePage calls and corrupt page state. + while (ClosedUntilAddress < newBeginAddress) + _ = Thread.Yield(); + + FreeAllAllocatedPages(); + } + finally { phase2Done.Set(); } + }); } + finally { epoch.Suspend(); } + phase2Done.Wait(); } - // Update begin address to tail - _ = Utility.MonotonicUpdate(ref BeginAddress, newBeginAddress, out _); + // Restore caller's epoch state if they were protected on entry. + if (wasProtected) + epoch.Resume(); + + // -------- Final: publish BeginAddress (see XML doc on Reset for why this happens last) -------- + _ = MonotonicUpdate(ref BeginAddress, newBeginAddress, out _); - FlushEvent.Initialize(); + flushEvent.Initialize(); Array.Clear(PageStatusIndicator, 0, BufferSize); if (PendingFlush != null) { @@ -356,14 +418,24 @@ public virtual void Reset() device.Reset(); } - /// Wraps when an allocator potentially has to interact with multiple devices - protected virtual void TruncateUntilAddress(long toAddress) + /// + /// Free any pages still allocated after has run. Subclasses + /// override to call their per-allocator FreePage. Invoked from inside Reset's + /// epoch.BumpCurrentEpoch action so it is safe against concurrent iterators. + /// + protected virtual void FreeAllAllocatedPages() { } + + /// Asynchronously wraps ; fires + /// AFTER the device truncation completes when + /// is true. + internal void TruncateUntilAddress(long toAddress) => _ = Task.Run(() => { - PersistedBeginAddress = toAddress; - _ = Task.Run(() => device.TruncateUntilAddress(toAddress)); - } + TruncateUntilAddressBlocking(toAddress); + if (storeFunctions.CallOnTruncate) + storeFunctions.OnTruncate(toAddress); + }); - /// Wraps when an allocator potentially has to interact with multiple devices + /// Synchronously (blocking) wraps ; overridden when an allocator potentially has to interact with multiple devices protected virtual void TruncateUntilAddressBlocking(long toAddress) => device.TruncateUntilAddress(toAddress); /// Remove disk segment @@ -380,11 +452,13 @@ public virtual void Dispose() epoch.Dispose(); bufferPool.Free(); - FlushEvent.Dispose(); - notifyFlushedUntilAddressSemaphore?.Dispose(); + flushEvent.Dispose(); + notifyFlushedUntilAddressTcs?.TrySetCanceled(); + notifyFlushedUntilAddressTcs = null; - OnReadOnlyObserver?.OnCompleted(); - OnEvictionObserver?.OnCompleted(); + onReadOnlyObserver?.OnCompleted(); + onEvictionObserver?.OnCompleted(); + logSizeTracker?.Stop(); } #endregion abstract and virtual methods @@ -395,98 +469,26 @@ private protected void VerifyCompatibleSectorSize(IDevice device) throw new TsavoriteException($"Allocator with sector size {sectorSize} cannot flush to device with sector size {device.SectorSize}"); } - internal unsafe void ApplyDelta(DeltaLog log, long startPage, long endPage, long recoverTo) - { - if (log == null) return; - - long startLogicalAddress = _wrapper.GetStartLogicalAddress(startPage); - long endLogicalAddress = _wrapper.GetStartLogicalAddress(endPage); - - log.Reset(); - while (log.GetNext(out long physicalAddress, out int entryLength, out var type)) - { - switch (type) - { - case DeltaLogEntryType.DELTA: - // Delta records - long endAddress = physicalAddress + entryLength; - while (physicalAddress < endAddress) - { - var address = *(long*)physicalAddress; - physicalAddress += sizeof(long); - var size = *(int*)physicalAddress; - physicalAddress += sizeof(int); - if (address >= startLogicalAddress && address < endLogicalAddress) - { - var destination = _wrapper.GetPhysicalAddress(address); - - // Clear extra space (if any) in old record - var oldSize = _wrapper.GetRecordSize(destination).Item2; - if (oldSize > size) - new Span((byte*)(destination + size), oldSize - size).Clear(); - - // Update with new record - Buffer.MemoryCopy((void*)physicalAddress, (void*)destination, size, size); - - // Clean up temporary bits when applying the delta log - ref var destInfo = ref _wrapper.GetInfo(destination); - destInfo.ClearBitsForDiskImages(); - } - physicalAddress += size; - } - break; - case DeltaLogEntryType.CHECKPOINT_METADATA: - if (recoverTo != -1) - { - // Only read metadata if we need to stop at a specific version - var metadata = new byte[entryLength]; - unsafe - { - fixed (byte* m = metadata) - Buffer.MemoryCopy((void*)physicalAddress, m, entryLength, entryLength); - } - - HybridLogRecoveryInfo recoveryInfo = new(); - using StreamReader s = new(new MemoryStream(metadata)); - recoveryInfo.Initialize(s); - // Finish recovery if only specific versions are requested - if (recoveryInfo.version == recoverTo) return; - } - - break; - default: - throw new TsavoriteException("Unexpected entry type"); - - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void MarkPage(long logicalAddress, long version) - { - var offset = (logicalAddress >> LogPageSizeBits) % BufferSize; - if (PageStatusIndicator[offset].Dirty < version) - PageStatusIndicator[offset].Dirty = version; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void MarkPageAtomic(long logicalAddress, long version) - { - var offset = (logicalAddress >> LogPageSizeBits) % BufferSize; - Utility.MonotonicUpdate(ref PageStatusIndicator[offset].Dirty, version, out _); - } - - internal void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite, - DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult, - IDevice device) + /// + /// This writes data from a page (or pages) for allocators that support only inline data. + /// + /// The source address, aligned to start of allocator page + /// The destination address, aligned to start of allocator page + /// Number of bytes to be written, based on allocator page range + /// The callback for the operation + /// The callback state information, including information for the flush operation + /// The device to write to + [MethodImpl(MethodImplOptions.NoInlining)] + internal void WriteInlinePageAsync(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite, + DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult, IDevice device) { if (asyncResult.partial) { // Write only required bytes within the page - int aligned_start = (int)((asyncResult.fromAddress - (asyncResult.page << LogPageSizeBits))); + int aligned_start = (int)(asyncResult.fromAddress - GetLogicalAddressOfStartOfPage(asyncResult.page)); aligned_start = (aligned_start / sectorSize) * sectorSize; - int aligned_end = (int)(asyncResult.untilAddress - (asyncResult.page << LogPageSizeBits)); + int aligned_end = (int)(asyncResult.untilAddress - GetLogicalAddressOfStartOfPage(asyncResult.page)); aligned_end = (aligned_end + (sectorSize - 1)) & ~(sectorSize - 1); numBytesToWrite = (uint)(aligned_end - aligned_start); @@ -494,53 +496,79 @@ internal void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDes } else { - device.WriteAsync(alignedSourceAddress, alignedDestinationAddress, - numBytesToWrite, callback, asyncResult); + // Write the whole page + device.WriteAsync(alignedSourceAddress, alignedDestinationAddress, numBytesToWrite, callback, asyncResult); } } - internal long GetReadOnlyAddressLagOffset() => ReadOnlyAddressLagOffset; - - protected readonly ILogger logger; + internal readonly ILogger logger; /// Instantiate base allocator implementation - private protected AllocatorBase(LogSettings settings, TStoreFunctions storeFunctions, Func wrapperCreator, Action evictCallback, LightEpoch epoch, Action flushCallback, ILogger logger = null) + [MethodImpl(MethodImplOptions.NoInlining)] + private protected AllocatorBase(AllocatorSettings allocatorSettings, TStoreFunctions storeFunctions, Func wrapperCreator, + ILogger logger = null, ObjectIdMap transientObjectIdMap = null) { - _storeFunctions = storeFunctions; + var logSettings = allocatorSettings.LogSettings; + var evictCallback = allocatorSettings.evictCallback; + var epoch = allocatorSettings.epoch; + var flushCallback = allocatorSettings.flushCallback; + IsReadCache = allocatorSettings.IsReadCache; + + this.storeFunctions = storeFunctions; _wrapper = wrapperCreator(this); + this.transientObjectIdMap = transientObjectIdMap; + // Validation - if (settings.PageSizeBits < LogSettings.kMinPageSizeBits || settings.PageSizeBits > LogSettings.kMaxPageSizeBits) - throw new TsavoriteException($"{nameof(settings.PageSizeBits)} must be between {LogSettings.kMinPageSizeBits} and {LogSettings.kMaxPageSizeBits}"); - if (settings.SegmentSizeBits < LogSettings.kMinSegmentSizeBits || settings.SegmentSizeBits > LogSettings.kMaxSegmentSizeBits) - throw new TsavoriteException($"{nameof(settings.SegmentSizeBits)} must be between {LogSettings.kMinSegmentSizeBits} and {LogSettings.kMaxSegmentSizeBits}"); - if (settings.MemorySizeBits != 0 && (settings.MemorySizeBits < LogSettings.kMinMemorySizeBits || settings.MemorySizeBits > LogSettings.kMaxMemorySizeBits)) - throw new TsavoriteException($"{nameof(settings.MemorySizeBits)} must be between {LogSettings.kMinMemorySizeBits} and {LogSettings.kMaxMemorySizeBits}, or may be 0 for ReadOnly TsavoriteLog"); - if (settings.MutableFraction < 0.0 || settings.MutableFraction > 1.0) - throw new TsavoriteException($"{nameof(settings.MutableFraction)} must be >= 0.0 and <= 1.0"); - if (settings.ReadCacheSettings is not null) - { - var rcs = settings.ReadCacheSettings; + if (logSettings.PageCount == 0 && logSettings.MemorySize == 0) + throw new TsavoriteException($"{nameof(logSettings.PageCount)} or {nameof(logSettings.MemorySize)} must be specified"); + if (logSettings.PageSizeBits < LogSettings.kMinPageSizeBits || logSettings.PageSizeBits > LogSettings.kMaxPageSizeBits) + throw new TsavoriteException($"{nameof(logSettings.PageSizeBits)} must be between {LogSettings.kMinPageSizeBits} and {LogSettings.kMaxPageSizeBits}"); + if (logSettings.PageSizeBits < PageHeader.SizeBits) + throw new TsavoriteException($"{nameof(logSettings.PageSizeBits)} must be >= PageHeader.SizeBits {PageHeader.SizeBits}"); + if (logSettings.PageCount > MemoryUtils.ArrayMaxLength) + throw new TsavoriteException($"{nameof(logSettings.PageCount)} must be less than or equal to the maximum array length ({MemoryUtils.ArrayMaxLength})"); + if (logSettings.SegmentSizeBits < LogSettings.kMinMainLogSegmentSizeBits || logSettings.SegmentSizeBits > LogSettings.kMaxSegmentSizeBits) + throw new TsavoriteException($"{nameof(logSettings.SegmentSizeBits)} must be between {LogSettings.kMinMainLogSegmentSizeBits} and {LogSettings.kMaxSegmentSizeBits}"); + if (logSettings.MemorySize != 0 && (logSettings.MemorySize < 1L << LogSettings.kMinMemorySizeBits || logSettings.MemorySize > 1L << LogSettings.kMaxMemorySizeBits)) + throw new TsavoriteException($"{nameof(logSettings.MemorySize)} must be between {1L << LogSettings.kMinMemorySizeBits} and {1L << LogSettings.kMaxMemorySizeBits}, or may be 0 for ReadOnly TsavoriteLog"); + if ((logSettings.MemorySize != 0) && (logSettings.MemorySize < (1L << logSettings.PageSizeBits) * 2)) + throw new TsavoriteException($"{nameof(logSettings.MemorySize)} must be at least twice the page size ({1L << logSettings.PageSizeBits})"); + if (logSettings.MutableFraction < 0.0 || logSettings.MutableFraction > 1.0) + throw new TsavoriteException($"{nameof(logSettings.MutableFraction)} must be >= 0.0 and <= 1.0"); + if (logSettings.ReadCacheSettings is not null) + { + var rcs = logSettings.ReadCacheSettings; + if (rcs.PageCount == 0 && rcs.MemorySize == 0) + throw new TsavoriteException($"{nameof(rcs.PageCount)} or {nameof(rcs.MemorySize)} must be specified"); if (rcs.PageSizeBits < LogSettings.kMinPageSizeBits || rcs.PageSizeBits > LogSettings.kMaxPageSizeBits) throw new TsavoriteException($"{nameof(rcs.PageSizeBits)} must be between {LogSettings.kMinPageSizeBits} and {LogSettings.kMaxPageSizeBits}"); - if (rcs.MemorySizeBits < LogSettings.kMinMemorySizeBits || rcs.MemorySizeBits > LogSettings.kMaxMemorySizeBits) - throw new TsavoriteException($"{nameof(rcs.MemorySizeBits)} must be between {LogSettings.kMinMemorySizeBits} and {LogSettings.kMaxMemorySizeBits}"); + if (rcs.PageCount > MemoryUtils.ArrayMaxLength) + throw new TsavoriteException($"{nameof(rcs.PageCount)} must be less than or equal to the maximum array length ({MemoryUtils.ArrayMaxLength})"); + if (rcs.MemorySize != 0 && (rcs.MemorySize < 1L << LogSettings.kMinMemorySizeBits || rcs.MemorySize > 1L << LogSettings.kMaxMemorySizeBits)) + throw new TsavoriteException($"{nameof(rcs.MemorySize)} must be between {1L << LogSettings.kMinMemorySizeBits} and {1L << LogSettings.kMaxMemorySizeBits}"); + if ((rcs.MemorySize != 0) && (rcs.MemorySize < (1L << rcs.PageSizeBits) * 2)) + throw new TsavoriteException($"{nameof(logSettings.MemorySize)} must be at least twice the page size ({1L << rcs.PageSizeBits})"); if (rcs.SecondChanceFraction < 0.0 || rcs.SecondChanceFraction > 1.0) - throw new TsavoriteException($"{(rcs.SecondChanceFraction)} must be >= 0.0 and <= 1.0"); + throw new TsavoriteException($"{rcs.SecondChanceFraction} must be >= 0.0 and <= 1.0"); } + if (logSettings.MaxInlineKeySizeBits < LogSettings.kLowestMaxInlineSizeBits || logSettings.PageSizeBits > LogSettings.kMaxStringSizeBits - 1) + throw new TsavoriteException($"{nameof(logSettings.MaxInlineKeySizeBits)} must be between {LogSettings.kMinPageSizeBits} and {LogSettings.kMaxStringSizeBits - 1}"); + if (logSettings.MaxInlineValueSizeBits < LogSettings.kLowestMaxInlineSizeBits || logSettings.PageSizeBits > LogSettings.kMaxStringSizeBits - 1) + throw new TsavoriteException($"{nameof(logSettings.MaxInlineValueSizeBits)} must be between {LogSettings.kMinPageSizeBits} and {LogSettings.kMaxStringSizeBits - 1}"); + this.logger = logger; - if (settings.LogDevice == null) + if (logSettings.LogDevice == null) throw new TsavoriteException("LogSettings.LogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)"); - IsReadCache = evictCallback != null; EvictCallback = evictCallback; FlushCallback = flushCallback; - PreallocateLog = settings.PreallocateLog; - FlushEvent.Initialize(); + PreallocateLog = logSettings.PreallocateLog; + flushEvent.Initialize(); - IsNullDevice = settings.LogDevice is NullDevice; + IsNullDevice = logSettings.LogDevice is NullDevice; if (epoch == null) { @@ -550,40 +578,42 @@ private protected AllocatorBase(LogSettings settings, TStoreFunctions storeFunct else this.epoch = epoch; - settings.LogDevice.Initialize(1L << settings.SegmentSizeBits, epoch); - settings.ObjectLogDevice?.Initialize(-1, epoch); + logSettings.LogDevice.Initialize(1L << logSettings.SegmentSizeBits, epoch); + logSettings.ObjectLogDevice?.Initialize(1L << logSettings.ObjectLogSegmentSizeBits, epoch); // Page size - LogPageSizeBits = settings.PageSizeBits; + LogPageSizeBits = logSettings.PageSizeBits; PageSize = 1 << LogPageSizeBits; PageSizeMask = PageSize - 1; - // Total HLOG size - LogTotalSizeBits = settings.MemorySizeBits; - LogTotalSizeBytes = 1L << LogTotalSizeBits; - BufferSize = (int)(LogTotalSizeBytes / (1L << LogPageSizeBits)); + // Total HLOG size and MaxAllocatedPageCount. There are a couple ways MaxAllocatedPageCount can be set here; once set it + // will never be exceeded by AllocatedPageCount, even if memory usage falls below logSettings.MemorySize. Memory + // size tracking will be enforced only if this.logSizeTracker is set; otherwise, we set the MaxAllocatedPageCount here and + // do not control heap memory usage. + if (logSettings.MemorySize > 0) + { + // If LogSettings.PageCount is specified it becomes MaxAllocatedPageCount; otherwise MaxAllocatedPageCount will be + // MaxMemorySize divided by page size. + MaxAllocatedPageCount = logSettings.PageCount > 0 ? logSettings.PageCount : (int)(logSettings.MemorySize / PageSize); + } + else + { + if (logSettings.PageCount <= 0) + throw new TsavoriteException($"Log Memory size or PageCount must be specified"); + MaxAllocatedPageCount = logSettings.PageCount; + } + BufferSize = (int)NextPowerOf2(MaxAllocatedPageCount); + BufferSizeMask = BufferSize - 1; - LogMutableFraction = settings.MutableFraction; + logMutableFraction = logSettings.MutableFraction; // Segment size - LogSegmentSizeBits = settings.SegmentSizeBits; + LogSegmentSizeBits = logSettings.SegmentSizeBits; SegmentSize = 1L << LogSegmentSizeBits; - SegmentBufferSize = 1 + (LogTotalSizeBytes / SegmentSize < 1 ? 1 : (int)(LogTotalSizeBytes / SegmentSize)); - if (SegmentSize < PageSize) throw new TsavoriteException($"Segment ({SegmentSize}) must be at least of page size ({PageSize})"); - if ((LogTotalSizeBits != 0) && (LogTotalSizeBytes < PageSize * 2)) - throw new TsavoriteException($"Memory size ({LogTotalSizeBytes}) must be at least twice the page size ({PageSize})"); - - // Readonlymode has MemorySizeBits 0 => skip the check - if (settings.MemorySizeBits > 0 && settings.MinEmptyPageCount > MaxEmptyPageCount) - throw new TsavoriteException($"MinEmptyPageCount ({settings.MinEmptyPageCount}) can't be more than MaxEmptyPageCount ({MaxEmptyPageCount})"); - - MinEmptyPageCount = settings.MinEmptyPageCount; - EmptyPageCount = settings.MinEmptyPageCount; - PageStatusIndicator = new FullPageStatus[BufferSize]; if (!IsNullDevice) @@ -592,34 +622,65 @@ private protected AllocatorBase(LogSettings settings, TStoreFunctions storeFunct for (int i = 0; i < BufferSize; i++) PendingFlush[i] = new PendingFlushList(); } - device = settings.LogDevice; + device = logSettings.LogDevice; sectorSize = (int)device.SectorSize; if (PageSize < sectorSize) throw new TsavoriteException($"Page size must be at least of device sector size ({sectorSize} bytes). Set PageSizeBits accordingly."); - AlignedPageSizeBytes = (PageSize + (sectorSize - 1)) & ~(sectorSize - 1); + AlignedPageSizeBytes = RoundUp(PageSize, sectorSize); + + if (BufferSize > 0) + { + pageArrays = new byte[BufferSize][]; + pagePointersArray = GC.AllocateArray(BufferSize, pinned: true); + pagePointers = (long*)Unsafe.AsPointer(ref pagePointersArray[0]); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal long GetPhysicalAddress(long logicalAddress) + { + // if (disposed) // TODO: Clean up dispose sequence + // ThrowTsavoriteException("GetPhysicalAddress called when disposed"); + + // Index of page within the circular buffer, and offset on the page. + var pageIndex = GetPageIndexForAddress(logicalAddress); + var offset = GetOffsetOnPage(logicalAddress); + return *(pagePointers + pageIndex) + offset; + } + internal bool IsAllocated(int pageIndex) => pageArrays[pageIndex] is not null; + + internal virtual void ClearPage(long page, int offset = 0) + { + var pageArray = pageArrays[page % BufferSize]; + + // If the offset is 0, we can clear everything in the array including the cache-alignment padding. + // Otherwise, we have to adjust the offset for the initial cache alignment. + if (offset != 0) + offset += (int)(pagePointers[page % BufferSize] - (long)Unsafe.AsPointer(ref pageArray[0])); + Array.Clear(pageArray, offset, pageArray.Length - offset); } + [MethodImpl(MethodImplOptions.NoInlining)] internal void VerifyRecoveryInfo(HybridLogCheckpointInfo recoveredHLCInfo, bool trimLog = false) { // Note: trimLog is unused right now. Can be used to trim the log to the minimum // segment range necessary for recovery to given checkpoint var diskBeginAddress = recoveredHLCInfo.info.beginAddress; - var diskFlushedUntilAddress = - recoveredHLCInfo.info.useSnapshotFile == 0 ? - recoveredHLCInfo.info.finalLogicalAddress : - recoveredHLCInfo.info.flushedLogicalAddress; + var diskFlushedUntilAddress = recoveredHLCInfo.info.useSnapshotFile == 0 + ? recoveredHLCInfo.info.finalLogicalAddress + : recoveredHLCInfo.info.flushedLogicalAddress; // Delete disk segments until specified disk begin address // First valid disk segment required for recovery - long firstValidSegment = (int)(diskBeginAddress >> LogSegmentSizeBits); + long firstValidSegment = (int)GetSegment(diskBeginAddress); // Last valid disk segment required for recovery - var lastValidSegment = (int)(diskFlushedUntilAddress >> LogSegmentSizeBits); - if ((diskFlushedUntilAddress & ((1L << LogSegmentSizeBits) - 1)) == 0) + var lastValidSegment = (int)GetSegment(diskFlushedUntilAddress); + if (GetOffsetOnSegment(diskFlushedUntilAddress) == 0) lastValidSegment--; logger?.LogInformation("Recovery requires disk segments in range [{firstSegment}--{tailStartSegment}]", firstValidSegment, lastValidSegment); @@ -627,10 +688,11 @@ internal void VerifyRecoveryInfo(HybridLogCheckpointInfo recoveredHLCInfo, bool var firstAvailSegment = device.StartSegment; var lastAvailSegment = device.EndSegment; - if (FlushedUntilAddress > _wrapper.GetFirstValidLogicalAddress(0)) + if (FlushedUntilAddress > GetFirstValidLogicalAddressOnPage(0)) { - int currTailSegment = (int)(FlushedUntilAddress >> LogSegmentSizeBits); - if ((FlushedUntilAddress & ((1L << LogSegmentSizeBits) - 1)) == 0) + var flushedUntilAddress = FlushedUntilAddress; + int currTailSegment = (int)GetSegment(flushedUntilAddress); + if (GetOffsetOnSegment(flushedUntilAddress) == 0) currTailSegment--; if (currTailSegment > lastAvailSegment) @@ -648,7 +710,10 @@ internal void VerifyRecoveryInfo(HybridLogCheckpointInfo recoveredHLCInfo, bool if (trimLog) { logger?.LogInformation("Trimming disk segments until (not including) {firstSegment}", firstValidSegment); - TruncateUntilAddressBlocking(firstValidSegment << LogSegmentSizeBits); + var toAddress = GetStartLogicalAddressOfSegment(firstValidSegment); + TruncateUntilAddressBlocking(toAddress); + if (storeFunctions.CallOnTruncate) + storeFunctions.OnTruncate(toAddress); for (int s = lastValidSegment + 1; s <= lastAvailSegment; s++) { @@ -658,32 +723,42 @@ internal void VerifyRecoveryInfo(HybridLogCheckpointInfo recoveredHLCInfo, bool } } - /// Initialize allocator - protected void Initialize(long firstValidAddress) + /// Allocate a pinned byte[] for the page at + protected void AllocatePinnedPageArray(int index) { - Debug.Assert(firstValidAddress <= PageSize, $"firstValidAddress {firstValidAddress} shoulld be <= PageSize {PageSize}"); + var adjustedSize = PageSize + 2 * sectorSize; + var tmp = GC.AllocateArray(adjustedSize, true); + var p = (long)Unsafe.AsPointer(ref tmp[0]); + pagePointersArray[index] = (p + (sectorSize - 1)) & ~((long)sectorSize - 1); + pageArrays[index] = tmp; + } + /// Initialize allocator + [MethodImpl(MethodImplOptions.NoInlining)] + protected internal virtual void Initialize() + { bufferPool ??= new SectorAlignedBufferPool(1, sectorSize); + var firstValidAddress = FirstValidAddress; if (BufferSize > 0) { - long tailPage = firstValidAddress >> LogPageSizeBits; - int tailPageIndex = (int)(tailPage % BufferSize); - if (!_wrapper.IsAllocated(tailPageIndex)) + long tailPage = GetPage(firstValidAddress); + int tailPageIndex = GetPageIndexForPage(tailPage); + if (!IsAllocated(tailPageIndex)) _wrapper.AllocatePage(tailPageIndex); // Allocate next page as well - int nextPageIndex = (int)(tailPage + 1) % BufferSize; - if (!_wrapper.IsAllocated(nextPageIndex)) + int nextPageIndex = GetPageIndexForPage(tailPage + 1); + if (!IsAllocated(nextPageIndex)) _wrapper.AllocatePage(nextPageIndex); } if (PreallocateLog) { - for (int i = 0; i < BufferSize; i++) + for (int pageIndex = 0; pageIndex < BufferSize; pageIndex++) { - if (!_wrapper.IsAllocated(i)) - _wrapper.AllocatePage(i); + if (!IsAllocated(pageIndex)) + _wrapper.AllocatePage(pageIndex); } } @@ -695,97 +770,50 @@ protected void Initialize(long firstValidAddress) FlushedUntilAddress = firstValidAddress; BeginAddress = firstValidAddress; - TailPageOffset.Page = (int)(firstValidAddress >> LogPageSizeBits); - TailPageOffset.Offset = (int)(firstValidAddress & PageSizeMask); + // Initialize TailAddress (the address of the next allocaton); this will always be nonzero. + TailPageOffset.Page = (int)GetPage(firstValidAddress); + TailPageOffset.Offset = (int)GetOffsetOnPage(firstValidAddress); } - /// Number of pages in circular buffer that are allocated - public int AllocatedPageCount; - - /// Max number of pages that have been allocated at any point in time - public int MaxAllocatedPageCount; + /// + /// Max allocated page count; less than or equal to . will never exceed this, + /// even if is set and memory usage falls below its TargetSize. This is also used to handle non-powerOf2 + /// inline log sizes.. + /// + internal int MaxAllocatedPageCount; - /// Maximum possible number of empty pages in circular buffer - public int MaxEmptyPageCount => BufferSize - 1; + /// + /// The number of memory pages that are currently allocated in the circular buffer. Will never exceed . + /// If there is a then it manages combined inline and heap memory, and + /// may increase or decrease (but again, will never exceed ). + /// + public int AllocatedPageCount; - /// Minimum number of empty pages in circular buffer to be maintained to account for non-power-of-two size - public int MinEmptyPageCount - { - get => minEmptyPageCount; - set - { - minEmptyPageCount = value; - if (emptyPageCount != minEmptyPageCount) - { - EmptyPageCount = minEmptyPageCount; - } - } - } + /// High-water mark of the number of memory pages that were allocated in the circular buffer + public int HighWaterAllocatedPageCount; /// Maximum memory size in bytes - public long MaxMemorySizeBytes => (BufferSize - MinEmptyPageCount) * (long)PageSize; - - /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) - public int EmptyPageCount - { - get => emptyPageCount; - - set - { - // HeadOffset lag (from tail). - var headOffsetLagSize = MaxEmptyPageCount; - if (value > headOffsetLagSize) return; - if (value < MinEmptyPageCount) return; - - int oldEPC; - lock (this) // linearize all setters of EmptyPageCount - { - oldEPC = emptyPageCount; - emptyPageCount = value; - headOffsetLagSize -= emptyPageCount; - - // Address lag offsets correspond to the number of pages "behind" TailPageOffset (the tail in the circular buffer). - ReadOnlyAddressLagOffset = (long)(LogMutableFraction * headOffsetLagSize) << LogPageSizeBits; - HeadAddressLagOffset = (long)headOffsetLagSize << LogPageSizeBits; - } - - // Force eviction now if empty page count has increased - if (value >= oldEPC) - { - bool prot = epoch.ThisInstanceProtected(); - - if (!prot) epoch.Resume(); - try - { - // These shifts adjust via application of the lag addresses. - var _tailAddress = GetTailAddress(); - PageAlignedShiftReadOnlyAddress(_tailAddress); - PageAlignedShiftHeadAddress(_tailAddress); - } - finally - { - if (!prot) epoch.Suspend(); - } - } - } - } + public long MaxMemorySizeBytes => BufferSize * PageSize; /// Increments AllocatedPageCount. Updates MaxAllocatedPageCount if a higher number of pages have been allocated. [MethodImpl(MethodImplOptions.AggressiveInlining)] protected void IncrementAllocatedPageCount() { var newAllocatedPageCount = Interlocked.Increment(ref AllocatedPageCount); - var currMaxAllocatedPageCount = MaxAllocatedPageCount; - while (currMaxAllocatedPageCount < newAllocatedPageCount) + var currHighWaterCount = HighWaterAllocatedPageCount; + while (currHighWaterCount < newAllocatedPageCount) { - if (Interlocked.CompareExchange(ref MaxAllocatedPageCount, newAllocatedPageCount, currMaxAllocatedPageCount) == currMaxAllocatedPageCount) + if (Interlocked.CompareExchange(ref HighWaterAllocatedPageCount, newAllocatedPageCount, currHighWaterCount) == currHighWaterCount) return; - currMaxAllocatedPageCount = MaxAllocatedPageCount; + currHighWaterCount = HighWaterAllocatedPageCount; } } - /// Segment size - public long GetSegmentSize() => SegmentSize; + /// Main log segment size + public long GetMainLogSegmentSize() => SegmentSize; + + /// Object log segment size + public virtual long GetObjectLogSegmentSize() => -1; /// Get tail address [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -793,9 +821,8 @@ public long GetTailAddress() { var local = TailPageOffset; - // Handle corner cases during page overflow - // The while loop is guaranteed to terminate because HandlePageOverflow - // ensures that it fixes the unstable TailPageOffset immediately. + // Handle corner cases during page overflow. + // The while loop is guaranteed to terminate because HandlePageOverflow ensures that it fixes the unstable TailPageOffset immediately. while (local.Offset >= PageSize) { if (local.Offset == PageSize) @@ -805,130 +832,289 @@ public long GetTailAddress() break; } // Offset is being adjusted by overflow thread, spin-wait - Thread.Yield(); + _ = Thread.Yield(); local = TailPageOffset; } - return ((long)local.Page << LogPageSizeBits) | (uint)local.Offset; + return GetLogicalAddressOfStartOfPage(local.Page) | (uint)local.Offset; + } + + /// Get tail address without considering whether it is unstable; this is called during HandlePageOverflow as part of determining + /// whether we must evict, which we may not be able to do; if is called on the thread that owns the tail-address + /// stabilization, it will infinite-loop. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal long UnstableGetTailAddress(out bool isUnstable) + { + var local = TailPageOffset; + var address = GetLogicalAddressOfStartOfPage(local.Page); + isUnstable = local.Offset >= PageSize; + if (!isUnstable) + return address | (uint)local.Offset; + + // It is unstable so stay on the same page, because we will likely restabilize before size tracker evictions commence. + return address | (uint)(PageSize - Constants.kRecordAlignment); } /// Get page index from - public long GetPage(long logicalAddress) => logicalAddress >> LogPageSizeBits; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetPage(long logicalAddress) => GetPageOfAddress(logicalAddress, LogPageSizeBits); /// Get page index for page + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetPageIndexForPage(long page) => (int)(page % BufferSize); /// Get page index for address - public int GetPageIndexForAddress(long address) => (int)((address >> LogPageSizeBits) % BufferSize); - - /// Get capacity (number of pages) - public int GetCapacityNumPages() => BufferSize; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetPageIndexForAddress(long logicalAddress) => GetPageIndexForPage(GetPageOfAddress(logicalAddress, LogPageSizeBits)); /// Get page size + [MethodImpl(MethodImplOptions.AggressiveInlining)] public long GetPageSize() => PageSize; + /// Get logical address of start of page + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetAddressOfStartOfPageOfAddress(long address) => address & ~PageSizeMask; + /// Get offset in page - public long GetOffsetInPage(long address) => address & PageSizeMask; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetOffsetOnPage(long address) => address & PageSizeMask; + + /// Get start logical address; this is the 0'th byte on the page, i.e. the start; it is *not* a valid record address + /// (for that see ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetLogicalAddressOfStartOfPage(long page) => page << LogPageSizeBits; + + /// Get first valid address on a page (which is the start of the page plus sizeof()). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetFirstValidLogicalAddressOnPage(long page) => (page << LogPageSizeBits) + FirstValidAddress; + + /// Get log segment index from + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetSegment(long logicalAddress) => logicalAddress >> LogSegmentSizeBits; + + /// Get offset in page + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetOffsetOnSegment(long address) => address & (SegmentSize - 1); + + /// Get start logical address + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetStartLogicalAddressOfSegment(long segment) => segment << LogSegmentSizeBits; /// Get sector size for main hlog device + [MethodImpl(MethodImplOptions.AggressiveInlining)] public int GetDeviceSectorSize() => sectorSize; + /// Have the derived allocator allocate the current and next page, if needed. + /// The derived allocator "owns" the actual memory for the pages in its circular buffer, as it knows the details of their use, + /// although for efficiency we keep the and some utility operations here in + /// (as well as manage ). [MethodImpl(MethodImplOptions.NoInlining)] void AllocatePagesWithException(int pageIndex, PageOffset localTailPageOffset, int numSlots) { try { // Allocate this page, if needed - if (!_wrapper.IsAllocated(pageIndex % BufferSize)) + if (!IsAllocated(pageIndex % BufferSize)) _wrapper.AllocatePage(pageIndex % BufferSize); // Allocate next page in advance, if needed - if (!_wrapper.IsAllocated((pageIndex + 1) % BufferSize)) + if (!IsAllocated((pageIndex + 1) % BufferSize)) _wrapper.AllocatePage((pageIndex + 1) % BufferSize); } catch { // Reset to previous tail localTailPageOffset.PageAndOffset -= numSlots; - Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); + _ = Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); throw; } } /// - /// Throw Tsavorite exception with message. We use a method wrapper so that - /// the caller method can execute inlined. + /// Shift log read-only address, with an optional wait /// - /// - /// - [MethodImpl(MethodImplOptions.NoInlining)] - static void ThrowTsavoriteException(string message) - => throw new TsavoriteException(message); + /// Address to shift read-only until + /// Wait to ensure shift is complete (may involve page flushing) + internal void ShiftReadOnlyAddressWithWait(long newReadOnlyAddress, bool wait) + { + // If we don't have the epoch, acquire it only long enough to launch the shift. + if (epoch.ResumeIfNotProtected()) + { + try + { + _ = ShiftReadOnlyAddress(newReadOnlyAddress); + } + finally + { + epoch.Suspend(); + } + + // Wait for flush to complete + while (wait && FlushedUntilAddress < newReadOnlyAddress) + _ = Thread.Yield(); + return; + } + + // Epoch already protected, so launch the shift and wait for flush to complete + _ = ShiftReadOnlyAddress(newReadOnlyAddress); + while (wait && FlushedUntilAddress < newReadOnlyAddress) + epoch.ProtectAndDrain(); + } + + /// + /// Shift log readonly and head addresses, with an optional wait on the head address shift + /// + /// New ReadOnlyAddress + /// New HeadAddress + /// Wait for eviction to complete, i.e., until ClosedUntilAddress catches up (may involve page flushing, closing, and eviction callbacks) + public void ShiftAddressesWithWait(long newReadOnlyAddress, long newHeadAddress, bool waitForEviction) + { + Debug.Assert(newHeadAddress <= newReadOnlyAddress, $"new HeadAddress {newHeadAddress} must not be ahead of newReadOnlyAddress {newReadOnlyAddress}"); + + // First shift read-only; force wait so that we do not close unflushed page + ShiftReadOnlyAddressWithWait(newReadOnlyAddress, wait: true); + + // Then shift head address. If we don't have the epoch, acquire it only long enough to launch the shift. + if (epoch.ResumeIfNotProtected()) + { + try + { + _ = ShiftHeadAddress(newHeadAddress); + } + finally + { + epoch.Suspend(); + } + + while (waitForEviction && ClosedUntilAddress < newHeadAddress) + _ = Thread.Yield(); + return; + } + + // Epoch already protected, so launch the shift and wait for eviction to complete + _ = ShiftHeadAddress(newHeadAddress); + + // We wait for ClosedUntilAddress here to ensure eviction scan is complete + while (waitForEviction && ClosedUntilAddress < newHeadAddress) + epoch.ProtectAndDrain(); + } + + /// + /// Whether we need to shift HeadAddress and ReadOnlyAddress to higher addresses when turning the page. + /// + /// The page we are turning to; it has just been allocated and TailAddress will be moving to this page + /// Local copy of PageOffset (includes the addition of numSlots) + /// Size of new allocation + /// + bool NeedToShiftAddress(long pageIndex, PageOffset localTailPageOffset, int numSlots) + { + var tailAddress = GetLogicalAddressOfStartOfPage(localTailPageOffset.Page) | ((long)(localTailPageOffset.Offset - numSlots)); + var shiftAddress = GetLogicalAddressOfStartOfPage(pageIndex); + + // First check whether we need to shift HeadAddress. If we have a logSizeTracker that's over budget then we have already issued + // a shift if needed (and allowed by allocated page count); otherwise make sure we stay in the MaxAllocatedPageCount (which may be less than BufferSize). + var desiredHeadAddress = HeadAddress; + if (logSizeTracker is null || !logSizeTracker.IsBeyondSizeLimit) + { + var headPage = GetPage(desiredHeadAddress); + if (pageIndex - headPage >= MaxAllocatedPageCount) + { + desiredHeadAddress = GetFirstValidLogicalAddressOnPage(headPage + 1); + if (desiredHeadAddress > tailAddress) + desiredHeadAddress = tailAddress; + return desiredHeadAddress > HeadAddress; + } + } + + // Check whether we need to shift ROA based on desiredHeadAddress. + var desiredReadOnlyAddress = CalculateReadOnlyAddress(shiftAddress, desiredHeadAddress); + if (desiredReadOnlyAddress > tailAddress) + desiredReadOnlyAddress = tailAddress; + return desiredReadOnlyAddress > ReadOnlyAddress; + } /// - /// Whether we need to shift addresses when turning the page. + /// Shift log addresses when turning the page. /// /// The page we are turning to - /// Local copy of PageOffset (includes the addition of numSlots) - /// Size of new allocation - /// - bool NeedToShiftAddress(long pageIndex, PageOffset localTailPageOffset, int numSlots) + /// If true, we have determined that we must call to Close and evict a + /// page before we can allocate a new one. This is done for checks that do not issue a signal to the size tracker, such as a + /// Flush or Close via normal wrapping operations. + void IssueShiftAddress(long pageIndex, bool needSHA) { - var tailAddress = (((long)localTailPageOffset.Page) << LogPageSizeBits) | ((long)(localTailPageOffset.Offset - numSlots)); - var shiftAddress = pageIndex << LogPageSizeBits; + // Issue the shift of address + var shiftAddress = GetLogicalAddressOfStartOfPage(pageIndex); + var tailAddress = GetTailAddress(); + + // First check whether we need to shift HeadAddress. If we are not forcing for flush and have a logSizeTracker that's over budget then we have already issued + // a shift if needed (and allowed by allocated page count); otherwise make sure we stay in the MaxAllocatedPageCount (which may be less than BufferSize). + var desiredHeadAddress = HeadAddress; + if (needSHA || logSizeTracker is null || !logSizeTracker.IsBeyondSizeLimit) + { + var headPage = GetPage(desiredHeadAddress); + if (pageIndex - headPage >= MaxAllocatedPageCount) + { + // Snapping to start of page rather than PageHeader.Size means that HA being middle-of-page implies a partial page. + desiredHeadAddress = GetLogicalAddressOfStartOfPage(headPage + 1); + if (desiredHeadAddress > tailAddress) + desiredHeadAddress = tailAddress; + } + } - // Check whether we need to shift ROA - var desiredReadOnlyAddress = shiftAddress - ReadOnlyAddressLagOffset; + // Check whether we need to shift ROA based on desiredHeadAddress. + var desiredReadOnlyAddress = CalculateReadOnlyAddress(shiftAddress, desiredHeadAddress); if (desiredReadOnlyAddress > tailAddress) desiredReadOnlyAddress = tailAddress; if (desiredReadOnlyAddress > ReadOnlyAddress) - return true; + _ = ShiftReadOnlyAddress(desiredReadOnlyAddress); - // Check whether we need to shift HA - var desiredHeadAddress = shiftAddress - HeadAddressLagOffset; - var currentFlushedUntilAddress = FlushedUntilAddress; - if (desiredHeadAddress > currentFlushedUntilAddress) - desiredHeadAddress = currentFlushedUntilAddress; - if (desiredHeadAddress > tailAddress) - desiredHeadAddress = tailAddress; + // Now shift HeadAddress if needed if (desiredHeadAddress > HeadAddress) - return true; - - return false; + _ = ShiftHeadAddress(desiredHeadAddress); } /// - /// Shift log addresses when turning the page. + /// If the page we are trying to allocate is past the last page with an unflushed address region, we have to wait for the flushEvent. /// - /// The page we are turning to - void IssueShiftAddress(long pageIndex) - { - // Issue the shift of address - var shiftAddress = pageIndex << LogPageSizeBits; - var tailAddress = GetTailAddress(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool NeedToWaitForFlush(int page) + => page >= BufferSize + GetPage(FlushedUntilAddress); // wraps around the BufferSize - long desiredReadOnlyAddress = shiftAddress - ReadOnlyAddressLagOffset; - if (desiredReadOnlyAddress > tailAddress) - desiredReadOnlyAddress = tailAddress; - ShiftReadOnlyAddress(desiredReadOnlyAddress); + /// + /// If the page we are trying to allocate is past the last page with an unclosed address region, then we can retry immediately + /// because this is called after NeedToWait, so we know we've completed the wait on flushEvent for the necessary pages to be flushed, + /// and are waiting for OnPagesClosed to be completed. Similarly, if the log size tracker is over budget, it has already issued + /// the ShiftHeadAddress that will close pages, so we can retry immediately. + /// + /// The page we are about to move to + /// Returns whether we need to call to advance HeadAddress so ClosedUntilAddress will advance + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool NeedToWaitForClose(int page, out bool needSHA) + { + if (page >= BufferSize + GetPage(ClosedUntilAddress)) // wraps around the BufferSize + { + needSHA = HeadAddress == ClosedUntilAddress; // Need SHA to advance HeadAddress to advance ClosedUntilAddress + return true; + } - long desiredHeadAddress = shiftAddress - HeadAddressLagOffset; - if (desiredHeadAddress > tailAddress) - desiredHeadAddress = tailAddress; - ShiftHeadAddress(desiredHeadAddress); + needSHA = false; + if (logSizeTracker is null || !logSizeTracker.IsBeyondSizeLimitAndCanEvict(addingPage: true)) + return false; + logSizeTracker.Signal(); + return true; } [MethodImpl(MethodImplOptions.NoInlining)] long HandlePageOverflow(ref PageOffset localTailPageOffset, int numSlots) { - int pageIndex = localTailPageOffset.Page + 1; + var pageIndex = localTailPageOffset.Page + 1; - // This thread is trying to allocate at an offset past where one or more previous threads - // already overflowed; exit and allow the first overflow thread to proceed. Do not try to remove - // the update to TailPageOffset that was done by this thread; that will be overwritten when + // See if this thread is trying to allocate at an offset past where one or more previous threads + // already overflowed; if so, exit and allow the first overflow thread to proceed. Do not try to + // remove the update to TailPageOffset that was done by this thread; that will be overwritten when // the first overflow thread finally completes and updates TailPageOffset. if (localTailPageOffset.Offset - numSlots > PageSize) { - if (NeedToWait(pageIndex)) + if (NeedToWaitForFlush(pageIndex)) return 0; // RETRY_LATER return -1; // RETRY_NOW } @@ -936,15 +1122,16 @@ long HandlePageOverflow(ref PageOffset localTailPageOffset, int numSlots) // The single thread that "owns" the page-increment proceeds below. This is the thread for which: // 1. Old image of offset (pre-Interlocked.Increment) is <= PageSize, and // 2. New image of offset (post-Interlocked.Increment) is > PageSize. - if (NeedToWait(pageIndex)) + + // If we need to wait for the flushEvent, we have to RETRY_LATER + if (NeedToWaitForFlush(pageIndex)) { // Reset to previous tail so that next attempt can retry localTailPageOffset.PageAndOffset -= numSlots; - Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); + _ = Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); // Shift only after TailPageOffset is reset to a valid state - IssueShiftAddress(pageIndex); - + IssueShiftAddress(pageIndex, needSHA: true); return 0; // RETRY_LATER } @@ -953,49 +1140,58 @@ long HandlePageOverflow(ref PageOffset localTailPageOffset, int numSlots) // 2. We have issued any necessary address shifting at the page-turn boundary. // If either cannot be verified, we can ask the caller to retry now (immediately), because it is // an ephemeral state. - if (CannotAllocate(pageIndex) || NeedToShiftAddress(pageIndex, localTailPageOffset, numSlots)) + if (NeedToWaitForClose(pageIndex, out bool needSHA) || NeedToShiftAddress(pageIndex, localTailPageOffset, numSlots)) { // Reset to previous tail so that next attempt can retry localTailPageOffset.PageAndOffset -= numSlots; - Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); + _ = Interlocked.Exchange(ref TailPageOffset.PageAndOffset, localTailPageOffset.PageAndOffset); // Shift only after TailPageOffset is reset to a valid state - IssueShiftAddress(pageIndex); - + IssueShiftAddress(pageIndex, needSHA); return -1; // RETRY_NOW } // Allocate next page and set new tail - if (!_wrapper.IsAllocated(pageIndex % BufferSize) || !_wrapper.IsAllocated((pageIndex + 1) % BufferSize)) + if (!IsAllocated(pageIndex % BufferSize) || !IsAllocated((pageIndex + 1) % BufferSize)) AllocatePagesWithException(pageIndex, localTailPageOffset, numSlots); + // Set up the TailPageOffset to account for the page header and then this allocation. localTailPageOffset.Page++; - localTailPageOffset.Offset = numSlots; + localTailPageOffset.Offset = numSlots + pageHeaderSize; TailPageOffset = localTailPageOffset; - // At this point, the slot is allocated and we are not allowed to refresh epochs any longer. + // If the logSizeTracker is active then we may have expanded the AllocatedPageCount, and thus may be able to increase ReadOnlyAddress + // and flush, which will allow the logSizeTracker to evict more records when needed, such as when we add the page's size to the tracker. + if (logSizeTracker is not null) + { + var newReadOnlyAddress = CalculateReadOnlyAddress(GetTailAddress(), HeadAddress); + if (newReadOnlyAddress > ReadOnlyAddress) + _ = ShiftReadOnlyAddress(newReadOnlyAddress); + } - // Offset is zero, for the first allocation on the new page - return ((long)localTailPageOffset.Page) << LogPageSizeBits; + // At this point the slot is allocated and we are not allowed to refresh epochs any longer. + // Return the first logical address after the page header. + return GetLogicalAddressOfStartOfPage(localTailPageOffset.Page) + pageHeaderSize; // Same as GetFirstValidLogicalAddressOnPage(localTailPageOffset.Page) but faster } /// Try allocate, no thread spinning allowed /// Number of slots to allocate /// The allocated logical address, or 0 in case of inability to allocate [MethodImpl(MethodImplOptions.AggressiveInlining)] - long TryAllocate(int numSlots = 1) + private long TryAllocate(int numSlots = 1) { if (numSlots > PageSize) ThrowTsavoriteException("Entry does not fit on page"); PageOffset localTailPageOffset = default; localTailPageOffset.PageAndOffset = TailPageOffset.PageAndOffset; + Debug.Assert(localTailPageOffset.Offset >= pageHeaderSize, $"TailPageOffset consistency error: Offset {localTailPageOffset.Offset} should equal be >= pageHeaderSize {pageHeaderSize}"); - // Necessary to check because threads keep retrying and we do not - // want to overflow the offset more than once per thread + // If the TailAddress.Offset is already past PageSize, another thread has incremented it and is working in HandlePageOverflow. + // Necessary to check because threads keep retrying and we do not want to overflow the offset more than once per thread. if (localTailPageOffset.Offset > PageSize) { - if (NeedToWait(localTailPageOffset.Page + 1)) + if (NeedToWaitForFlush(localTailPageOffset.Page + 1)) return 0; // RETRY_LATER return -1; // RETRY_NOW } @@ -1013,51 +1209,85 @@ long TryAllocate(int numSlots = 1) // before performing any epoch bumps or system calls. return HandlePageOverflow(ref localTailPageOffset, numSlots); } - return (((long)localTailPageOffset.Page) << LogPageSizeBits) | ((long)(localTailPageOffset.Offset - numSlots)); + + return GetLogicalAddressOfStartOfPage(localTailPageOffset.Page) | ((long)(localTailPageOffset.Offset - numSlots)); } /// Try allocate, spin for RETRY_NOW (logicalAddress is less than 0) case /// Number of slots to allocate - /// The allocated logical address, or 0 in case of inability to allocate + /// Returned address, or RETRY_LATER (if 0) indicator + /// True if we were able to allocate, else false [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long TryAllocateRetryNow(int numSlots = 1) + public bool TryAllocateRetryNow(int numSlots, out long logicalAddress) { - long logicalAddress; while ((logicalAddress = TryAllocate(numSlots)) < 0) { + // -1: RETRY_NOW _ = TryComplete(); epoch.ProtectAndDrain(); - Thread.Yield(); + _ = Thread.Yield(); } - return logicalAddress; + + // 0: RETRY_LATER + return logicalAddress != 0; } /// - /// If the page we are trying to allocate is past the last page with an unclosed address region, - /// then we can retry immediately because this is called after NeedToWait, so we know we've - /// completed the wait on flushEvent for the necessary pages to be flushed, and are waiting for - /// OnPagesClosed to be completed. + /// Calculate the new ReadOnlyAddress from the inputs. This may be called when has changed due to memory + /// size limits, so we cannot use a constant lag approach. /// + /// Either the next TailAddress if doing an allocation, or the current TailAddress if trimming memory size. + /// Either the current HeadAddress if doing an allocation, or the calculated HeadAddress if trimming memory size. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool CannotAllocate(int page) => page >= BufferSize + (ClosedUntilAddress >> LogPageSizeBits); + internal long CalculateReadOnlyAddress(long tailAddress, long headAddress) + { + // Snap ReadOnlyAddress to the start of the page that this calculation on logMutableFraction ends up in. If this is below HeadAddress, + // then make it the end of the HeadAddress page. If tailAddress is still on the first page, return HeadAddress. + if (tailAddress <= PageSize + PageHeader.Size) + return headAddress; + + // First we try to set ReadOnlyAddress to the lower page boundary, to maximize mutable space. + var readOnlyAddress = RoundDown(headAddress + (long)((1.0 - logMutableFraction) * (tailAddress - headAddress)), PageSize); + + // If are at the beginning we will have only a small number of pages, so it's even more important to maximize mutable space. + // If the new readOnlyAddress is less than or equal to the current one, make sure we have handled the boundary case where + // we had to tweak it to handle this. + // TODO: Currently we keep ReadOnlyAddress at page boundaries unless we can't (HeadAddress has grown but TailAddress is still + // on the first page); consider changing to fine-grained ReadOnlyAddress. + if (readOnlyAddress <= headAddress) + { + var headPage = GetPage(headAddress); - /// - /// If the page we are trying to allocate is past the last page with an unflushed address region, - /// we have to wait for the flushEvent. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool NeedToWait(int page) => page >= BufferSize + (FlushedUntilAddress >> LogPageSizeBits); + // If HeadAddress hasn't moved, currentReadOnlyAddress is either HeadAddress or at a page boundary, unless we have the case where + // we've gone beyond size budget on a single page; in that case we must remain at headAddress. + if ((GetOffsetOnPage(headAddress) <= PageHeader.Size) || GetPage(tailAddress) == headPage) + readOnlyAddress = headAddress; + else + { + // HeadAddress has moved and tailAddress is on another page. + readOnlyAddress = GetLogicalAddressOfStartOfPage(headPage + 1); + } + } + + // The HeadAddress page is always calculated to be below the TailAddress page once we have more than a page of records. + Debug.Assert(readOnlyAddress <= tailAddress, $"ReadOnlyAddress {readOnlyAddress} must not exceed TailAddress {tailAddress}"); + Debug.Assert(readOnlyAddress >= headAddress, $"ReadOnlyAddress {readOnlyAddress} must not be less than HeadAddress {headAddress}"); + + //Debug.WriteLine($"Calcalating ROA: tailAddress={tailAddress}, headAddress={headAddress}, readOnlyAddress={readOnlyAddress}"); + return readOnlyAddress; + } /// Used by applications to make the current state of the database immutable quickly - public bool ShiftReadOnlyToTail(out long tailAddress, out SemaphoreSlim notifyDone) + public bool ShiftReadOnlyToTail(out long tailAddress, out Task notifyDone) { notifyDone = null; tailAddress = GetTailAddress(); - long localTailAddress = tailAddress; - if (Utility.MonotonicUpdate(ref ReadOnlyAddress, tailAddress, out _)) + var localTailAddress = tailAddress; + if (MonotonicUpdate(ref ReadOnlyAddress, tailAddress, out _)) { - notifyFlushedUntilAddressSemaphore = new SemaphoreSlim(0); - notifyDone = notifyFlushedUntilAddressSemaphore; + notifyFlushedUntilAddressTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + notifyDone = notifyFlushedUntilAddressTcs.Task; notifyFlushedUntilAddress = localTailAddress; epoch.BumpCurrentEpoch(() => OnPagesMarkedReadOnly(localTailAddress)); return true; @@ -1068,7 +1298,7 @@ public bool ShiftReadOnlyToTail(out long tailAddress, out SemaphoreSlim notifyDo /// Used by applications to move read-only forward public bool ShiftReadOnlyAddress(long newReadOnlyAddress, bool noFlush = false) { - if (Utility.MonotonicUpdate(ref ReadOnlyAddress, newReadOnlyAddress, out _)) + if (MonotonicUpdate(ref ReadOnlyAddress, newReadOnlyAddress, out _)) { epoch.BumpCurrentEpoch(() => OnPagesMarkedReadOnly(newReadOnlyAddress, noFlush)); return true; @@ -1080,7 +1310,7 @@ public bool ShiftReadOnlyAddress(long newReadOnlyAddress, bool noFlush = false) public void ShiftBeginAddress(long newBeginAddress, bool truncateLog, bool noFlush = false) { // First update the begin address - if (!Utility.MonotonicUpdate(ref BeginAddress, newBeginAddress, out _)) + if (!MonotonicUpdate(ref BeginAddress, newBeginAddress, out _)) { if (truncateLog) epoch.BumpCurrentEpoch(() => TruncateUntilAddress(newBeginAddress)); @@ -1088,7 +1318,7 @@ public void ShiftBeginAddress(long newBeginAddress, bool truncateLog, bool noFlu } // Shift read-only address - var flushEvent = FlushEvent; + var localFlushEvent = flushEvent; _ = ShiftReadOnlyAddress(newBeginAddress, noFlush); if (!noFlush) @@ -1107,19 +1337,18 @@ public void ShiftBeginAddress(long newBeginAddress, bool truncateLog, bool noFlu try { epoch.Suspend(); - flushEvent.Wait(); + localFlushEvent.Wait(); } finally { epoch.Resume(); } - flushEvent = FlushEvent; + localFlushEvent = flushEvent; } } // Then shift head address - var h = Utility.MonotonicUpdate(ref HeadAddress, newBeginAddress, out _); - + var h = MonotonicUpdate(ref HeadAddress, newBeginAddress, out _); if (h || truncateLog) { epoch.BumpCurrentEpoch(() => @@ -1133,45 +1362,58 @@ public void ShiftBeginAddress(long newBeginAddress, bool truncateLog, bool noFlu } /// Invokes eviction observer if set and then frees the page. - internal virtual void EvictPage(long page) + internal void EvictPageForRecovery(long page) { - var start = page << LogPageSizeBits; - var end = (page + 1) << LogPageSizeBits; - if (OnEvictionObserver is not null) - MemoryPageScan(start, end, OnEvictionObserver); + var start = GetLogicalAddressOfStartOfPage(page); + var end = GetLogicalAddressOfStartOfPage(page + 1); + + var source = IsReadCache ? EvictionSource.ReadCache : EvictionSource.MainLog; + + // Per-record eviction walk handles internal heap accounting (key + value via + // logSizeTracker) and optionally notifies the application via OnEvict. + if (logSizeTracker is not null || storeFunctions.CallOnEvict) + { + _wrapper.EvictRecordsInRange(start, end, source); + } + if (onEvictionObserver is not null) + { + MemoryPageScan(start, end, onEvictionObserver); + } + _wrapper.FreePage(page); } /// + /// Action to be performed when pages move into the immutable region. /// Seal: make sure there are no longer any threads writing to the page /// Flush: send page to secondary store /// - private void OnPagesMarkedReadOnly(long newSafeReadOnlyAddress, bool noFlush = false) + internal virtual void OnPagesMarkedReadOnly(long newSafeReadOnlyAddress, bool noFlush = false) { - if (Utility.MonotonicUpdate(ref SafeReadOnlyAddress, newSafeReadOnlyAddress, out long oldSafeReadOnlyAddress)) + if (MonotonicUpdate(ref SafeReadOnlyAddress, newSafeReadOnlyAddress, out var oldSafeReadOnlyAddress)) { // Debug.WriteLine("SafeReadOnly shifted from {0:X} to {1:X}", oldSafeReadOnlyAddress, newSafeReadOnlyAddress); - if (OnReadOnlyObserver != null) + if (onReadOnlyObserver != null) { // This scan does not need a store because it does not lock; it is epoch-protected so by the time it runs no current thread // will have seen a record below the new ReadOnlyAddress as "in mutable region". - using var iter = Scan(store: null, oldSafeReadOnlyAddress, newSafeReadOnlyAddress, ScanBufferingMode.NoBuffering); - OnReadOnlyObserver?.OnNext(iter); + using var iter = Scan(store: null, oldSafeReadOnlyAddress, newSafeReadOnlyAddress, DiskScanBufferingMode.NoBuffering); + onReadOnlyObserver?.OnNext(iter); } - AsyncFlushPages(oldSafeReadOnlyAddress, newSafeReadOnlyAddress, noFlush); + AsyncFlushPagesForReadOnly(oldSafeReadOnlyAddress, newSafeReadOnlyAddress, noFlush); } } - /// Action to be performed for when all threads have agreed that a page range is closed. + /// Action to be performed when all threads have agreed that a page range is closed. private void OnPagesClosed(long newSafeHeadAddress) { Debug.Assert(newSafeHeadAddress > 0); - if (Utility.MonotonicUpdate(ref SafeHeadAddress, newSafeHeadAddress, out _)) + if (MonotonicUpdate(ref SafeHeadAddress, newSafeHeadAddress, out _ /*oldSafeHeadAddress*/)) { // This thread is responsible for [oldSafeHeadAddress -> newSafeHeadAddress] - for (; ; Thread.Yield()) + while (true) { - long _ongoingCloseUntilAddress = OngoingCloseUntilAddress; + var _ongoingCloseUntilAddress = OngoingCloseUntilAddress; // If we are closing in the middle of an ongoing OPCWorker loop, exit. if (_ongoingCloseUntilAddress >= newSafeHeadAddress) @@ -1180,56 +1422,62 @@ private void OnPagesClosed(long newSafeHeadAddress) // We'll continue the loop if we fail the CAS here; that means another thread extended the Ongoing range. if (Interlocked.CompareExchange(ref OngoingCloseUntilAddress, newSafeHeadAddress, _ongoingCloseUntilAddress) == _ongoingCloseUntilAddress) { + // If _ongoingCloseUntilAddress != 0 then another thread is runnning the OPCWorker loop and will see the OngoingCloseUntilAddress increment to + // include newSafeHeadAddress so we are done here. Otherwise, this thread is responsible for closing [ClosedUntilAddress -> newSafeHeadAddress] + // and any other ranges that OngoingCloseUntilAddress is incremented to, and we are done here when that concludes. if (_ongoingCloseUntilAddress == 0) - { - // There was no other thread running the OPCWorker loop, so this thread is responsible for closing [ClosedUntilAddress -> newSafeHeadAddress] OnPagesClosedWorker(); - } - else - { - // There was another thread runnning the OPCWorker loop, and its ongoing close operation was successfully extended to include the new safe - // head address; we have no further work here. - } return; } + _ = Thread.Yield(); } } } private void OnPagesClosedWorker() { - for (; ; Thread.Yield()) + while (true) { - long closeStartAddress = ClosedUntilAddress; - long closeEndAddress = OngoingCloseUntilAddress; + var closeStartAddress = ClosedUntilAddress; + var closeEndAddress = OngoingCloseUntilAddress; - if (IsReadCache) + if (EvictCallback is not null) EvictCallback(closeStartAddress, closeEndAddress); - for (long closePageAddress = closeStartAddress & ~PageSizeMask; closePageAddress < closeEndAddress; closePageAddress += PageSize) + // Process a page (possibly fragment) at a time. + for (var closePageAddress = GetAddressOfStartOfPageOfAddress(closeStartAddress); closePageAddress < closeEndAddress; closePageAddress += PageSize) { - long start = closeStartAddress > closePageAddress ? closeStartAddress : closePageAddress; - long end = closeEndAddress < closePageAddress + PageSize ? closeEndAddress : closePageAddress + PageSize; - - // This scan does not need a store because it does not lock; it is epoch-protected so by the time it runs no current thread - // will have seen a record below the eviction range as "in mutable region". - if (OnEvictionObserver is not null) - MemoryPageScan(start, end, OnEvictionObserver); - - // If we are using a null storage device, we must also shift BeginAddress + // Get the range on this page: the start may be 0 or greater, and the end may be end-of-page or less. + var start = closeStartAddress > closePageAddress ? closeStartAddress : closePageAddress; + var end = closeEndAddress < closePageAddress + PageSize ? closeEndAddress : closePageAddress + PageSize; + + // Legacy observer path — skip if the observer IS the logSizeTracker, since + // EvictRecordsInRange below already handles heap accounting via logSizeTracker. + if (onEvictionObserver is not null) + MemoryPageScan(start, end, onEvictionObserver); + + // Per-record eviction walk: handles internal heap-size accounting (key overflow + // and value heap via logSizeTracker) and optionally notifies the application + // via OnEvict for app-level cleanup. + var evictSource = IsReadCache ? EvictionSource.ReadCache : EvictionSource.MainLog; + if (logSizeTracker is not null || storeFunctions.CallOnEvict) + _wrapper.EvictRecordsInRange(start, end, evictSource); + + // If we are using a null storage device, we must also shift BeginAddress (leave it in-memory) if (IsNullDevice) - _ = Utility.MonotonicUpdate(ref BeginAddress, end, out _); + _ = MonotonicUpdate(ref BeginAddress, end, out _); // If the end of the closing range is at the end of the page, free the page if (end == closePageAddress + PageSize) - _wrapper.FreePage((int)(closePageAddress >> LogPageSizeBits)); + _wrapper.FreePage((int)GetPage(closePageAddress)); - _ = Utility.MonotonicUpdate(ref ClosedUntilAddress, end, out _); + _ = MonotonicUpdate(ref ClosedUntilAddress, end, out _); } - // End if we have exhausted co-operative work + // End if we have exhausted co-operative work. This includes the case where OngoingCloseUntilAddress and closeEndAddress are already 0. if (Interlocked.CompareExchange(ref OngoingCloseUntilAddress, 0, closeEndAddress) == closeEndAddress) break; + _ = Thread.Yield(); } } @@ -1244,51 +1492,14 @@ private void DebugPrintAddresses() var _readonly = ReadOnlyAddress; var _tail = GetTailAddress(); - Console.WriteLine("BeginAddress: {0}.{1}", GetPage(_begin), GetOffsetInPage(_begin)); - Console.WriteLine("ClosedUntilAddress: {0}.{1}", GetPage(_closedUntil), GetOffsetInPage(_closedUntil)); - Console.WriteLine("SafeHead: {0}.{1}", GetPage(_safehead), GetOffsetInPage(_safehead)); - Console.WriteLine("Head: {0}.{1}", GetPage(_head), GetOffsetInPage(_head)); - Console.WriteLine("FlushedUntil: {0}.{1}", GetPage(_flush), GetOffsetInPage(_flush)); - Console.WriteLine("SafeReadOnly: {0}.{1}", GetPage(_safereadonly), GetOffsetInPage(_safereadonly)); - Console.WriteLine("ReadOnly: {0}.{1}", GetPage(_readonly), GetOffsetInPage(_readonly)); - Console.WriteLine("Tail: {0}.{1}", GetPage(_tail), GetOffsetInPage(_tail)); - } - - /// - /// Called every time a new tail page is allocated. Here the read-only is shifted only to page boundaries - /// unlike ShiftReadOnlyToTail where shifting can happen to any fine-grained address. - /// - private void PageAlignedShiftReadOnlyAddress(long currentTailAddress) - { - long pageAlignedTailAddress = currentTailAddress & ~PageSizeMask; - long desiredReadOnlyAddress = pageAlignedTailAddress - ReadOnlyAddressLagOffset; - if (Utility.MonotonicUpdate(ref ReadOnlyAddress, desiredReadOnlyAddress, out _)) - { - // Debug.WriteLine("Allocate: Moving read-only offset from {0:X} to {1:X}", oldReadOnlyAddress, desiredReadOnlyAddress); - epoch.BumpCurrentEpoch(() => OnPagesMarkedReadOnly(desiredReadOnlyAddress)); - } - } - - /// - /// Called whenever a new tail page is allocated or when the user is checking for a failed memory allocation - /// Tries to shift head address based on the head offset lag size. - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void PageAlignedShiftHeadAddress(long currentTailAddress) - { - var desiredHeadAddress = (currentTailAddress & ~PageSizeMask) - HeadAddressLagOffset; - - // Obtain local values of variables that can change - var currentFlushedUntilAddress = FlushedUntilAddress; - if (desiredHeadAddress > currentFlushedUntilAddress) - desiredHeadAddress = currentFlushedUntilAddress & ~PageSizeMask; - - if (Utility.MonotonicUpdate(ref HeadAddress, desiredHeadAddress, out _)) - { - // Debug.WriteLine("Allocate: Moving head offset from {0:X} to {1:X}", oldHeadAddress, newHeadAddress); - epoch.BumpCurrentEpoch(() => OnPagesClosed(desiredHeadAddress)); - } + Console.WriteLine("BeginAddress: {0}.{1}", GetPage(_begin), GetOffsetOnPage(_begin)); + Console.WriteLine("ClosedUntilAddress: {0}.{1}", GetPage(_closedUntil), GetOffsetOnPage(_closedUntil)); + Console.WriteLine("SafeHead: {0}.{1}", GetPage(_safehead), GetOffsetOnPage(_safehead)); + Console.WriteLine("Head: {0}.{1}", GetPage(_head), GetOffsetOnPage(_head)); + Console.WriteLine("FlushedUntil: {0}.{1}", GetPage(_flush), GetOffsetOnPage(_flush)); + Console.WriteLine("SafeReadOnly: {0}.{1}", GetPage(_safereadonly), GetOffsetOnPage(_safereadonly)); + Console.WriteLine("ReadOnly: {0}.{1}", GetPage(_readonly), GetOffsetOnPage(_readonly)); + Console.WriteLine("Tail: {0}.{1}", GetPage(_tail), GetOffsetOnPage(_tail)); } /// @@ -1298,17 +1509,15 @@ private void PageAlignedShiftHeadAddress(long currentTailAddress) public long ShiftHeadAddress(long desiredHeadAddress) { // Obtain local values of variables that can change - long currentFlushedUntilAddress = FlushedUntilAddress; + var currentFlushedUntilAddress = FlushedUntilAddress; - long newHeadAddress = desiredHeadAddress; + // Cap the new head address at the last flushed address. + var newHeadAddress = desiredHeadAddress; if (newHeadAddress > currentFlushedUntilAddress) newHeadAddress = currentFlushedUntilAddress; - if (newHeadAddress % (1 << LogPageSizeBits) != 0) - { - - } - if (Utility.MonotonicUpdate(ref HeadAddress, newHeadAddress, out _)) + // Note: Currently nothing needs to be done if HeadAddress advancement is at a finer grain than page-level. + if (MonotonicUpdate(ref HeadAddress, newHeadAddress, out _)) { // Debug.WriteLine("Allocate: Moving head offset from {0:X} to {1:X}", oldHeadAddress, newHeadAddress); epoch.BumpCurrentEpoch(() => OnPagesClosed(newHeadAddress)); @@ -1319,16 +1528,16 @@ public long ShiftHeadAddress(long desiredHeadAddress) /// /// Every async flush callback tries to update the flushed until address to the latest value possible - /// Is there a better way to do this with enabling fine-grained addresses (not necessarily at page boundaries)? + /// TODO: Is there a better way to do this with enabling fine-grained addresses (not necessarily at page boundaries)? /// protected void ShiftFlushedUntilAddress() { - long currentFlushedUntilAddress = FlushedUntilAddress; - long page = GetPage(currentFlushedUntilAddress); + var currentFlushedUntilAddress = FlushedUntilAddress; + var page = GetPage(currentFlushedUntilAddress); - bool update = false; - long pageLastFlushedAddress = PageStatusIndicator[page % BufferSize].LastFlushedUntilAddress; - while (pageLastFlushedAddress >= currentFlushedUntilAddress && currentFlushedUntilAddress >= (page << LogPageSizeBits)) + var update = false; + var pageLastFlushedAddress = PageStatusIndicator[page % BufferSize].LastFlushedUntilAddress; + while (pageLastFlushedAddress >= currentFlushedUntilAddress && currentFlushedUntilAddress >= GetLogicalAddressOfStartOfPage(page)) { currentFlushedUntilAddress = pageLastFlushedAddress; update = true; @@ -1340,7 +1549,7 @@ protected void ShiftFlushedUntilAddress() { // Anything here must be valid flushes because error flushes do not set LastFlushedUntilAddress, which // prevents future ranges from being marked as flushed - if (Utility.MonotonicUpdate(ref FlushedUntilAddress, currentFlushedUntilAddress, out long oldFlushedUntilAddress)) + if (MonotonicUpdate(ref FlushedUntilAddress, currentFlushedUntilAddress, out long oldFlushedUntilAddress)) { FlushCallback?.Invoke( new CommitInfo @@ -1350,10 +1559,10 @@ protected void ShiftFlushedUntilAddress() ErrorCode = 0 }); - FlushEvent.Set(); + flushEvent.Set(); if ((oldFlushedUntilAddress < notifyFlushedUntilAddress) && (currentFlushedUntilAddress >= notifyFlushedUntilAddress)) - _ = notifyFlushedUntilAddressSemaphore.Release(); + _ = notifyFlushedUntilAddressTcs?.TrySetResult(true); } } @@ -1372,25 +1581,30 @@ protected void ShiftFlushedUntilAddress() /// Address for notification of flushed-until public long notifyFlushedUntilAddress; - /// Semaphore for notification of flushed-until - public SemaphoreSlim notifyFlushedUntilAddressSemaphore; + /// TaskCompletionSource for notification of flushed-until + public TaskCompletionSource notifyFlushedUntilAddressTcs; /// Reset for recovery - public void RecoveryReset(long tailAddress, long headAddress, long beginAddress, long readonlyAddress) + [MethodImpl(MethodImplOptions.NoInlining)] + protected internal virtual void RecoveryReset(long tailAddress, long headAddress, long beginAddress, long readonlyAddress) { long tailPage = GetPage(tailAddress); - long offsetInPage = GetOffsetInPage(tailAddress); + long offsetInPage = GetOffsetOnPage(tailAddress); TailPageOffset.Page = (int)tailPage; TailPageOffset.Offset = (int)offsetInPage; + // Sometimes the tailAddress calculation ends on a page boundary and this gets into the RecoveryInfo. + // Don't change GetTailAddress() as that may affect other calculations; instead, ensure it's set correctly here. + if (pageHeaderSize > 0 && TailPageOffset.Offset == 0) + TailPageOffset.Offset = pageHeaderSize; // Allocate current page if necessary var pageIndex = TailPageOffset.Page % BufferSize; - if (!_wrapper.IsAllocated(pageIndex)) + if (!IsAllocated(pageIndex)) _wrapper.AllocatePage(pageIndex); // Allocate next page as well - this is an invariant in the allocator! var nextPageIndex = (pageIndex + 1) % BufferSize; - if (!_wrapper.IsAllocated(nextPageIndex)) + if (!IsAllocated(nextPageIndex)) _wrapper.AllocatePage(nextPageIndex); BeginAddress = beginAddress; @@ -1405,7 +1619,7 @@ public void RecoveryReset(long tailAddress, long headAddress, long beginAddress, pageIndex = GetPageIndexForAddress(tailAddress); // clear the last page starting from tail address - _wrapper.ClearPage(pageIndex, (int)GetOffsetInPage(tailAddress)); + ClearPage(pageIndex, (int)GetOffsetOnPage(tailAddress)); // Printing debug info logger?.LogInformation("******* Recovered HybridLog Stats *******"); @@ -1416,219 +1630,208 @@ public void RecoveryReset(long tailAddress, long headAddress, long beginAddress, logger?.LogInformation("Tail Address: {tailAddress}", tailAddress); } - /// Invoked by users to obtain a record from disk. It uses sector aligned memory to read the record efficiently into memory. - internal unsafe void AsyncReadRecordToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, ref AsyncIOContext context) + /// Read a main log record to - used for RUMD operations. + [MethodImpl(MethodImplOptions.NoInlining)] + internal void AsyncReadRecordToMemory(long fromLogicalAddress, int numBytes, DeviceIOCompletionCallback callback, ref AsyncIOContext context) + { + context.record = GetAndPopulateReadBuffer(fromLogicalAddress, numBytes, out var alignedFileOffset, out var alignedReadLength); + var asyncResult = new AsyncGetFromDiskResult { context = context }; + device.ReadAsync(alignedFileOffset, (IntPtr)asyncResult.context.record.aligned_pointer, alignedReadLength, callback, asyncResult); + } + + /// Read inline blittable record to - simple read context version. Used by TsavoriteLog. + [MethodImpl(MethodImplOptions.NoInlining)] + internal void AsyncReadBlittableRecordToMemory(long fromLogicalAddress, int numBytes, DeviceIOCompletionCallback callback, ref SimpleReadContext context) { - var fileOffset = (ulong)(AlignedPageSizeBytes * (fromLogical >> LogPageSizeBits) + (fromLogical & PageSizeMask)); - var alignedFileOffset = (ulong)(((long)fileOffset / sectorSize) * sectorSize); + context.record = GetAndPopulateReadBuffer(fromLogicalAddress, numBytes, out var alignedFileOffset, out var alignedReadLength); + device.ReadAsync(alignedFileOffset, (IntPtr)context.record.aligned_pointer, alignedReadLength, callback, context); + } - var alignedReadLength = (uint)((long)fileOffset + numBytes - (long)alignedFileOffset); - alignedReadLength = (uint)((alignedReadLength + (sectorSize - 1)) & ~(sectorSize - 1)); + private SectorAlignedMemory GetAndPopulateReadBuffer(long fromLogicalAddress, int numBytes, out ulong alignedFileOffset, out uint alignedReadLength) + { + var fileOffset = (ulong)(AlignedPageSizeBytes * GetPage(fromLogicalAddress) + GetOffsetOnPage(fromLogicalAddress)); + alignedFileOffset = (ulong)RoundDown((long)fileOffset, sectorSize); + alignedReadLength = (uint)((long)fileOffset + numBytes - (long)alignedFileOffset); + alignedReadLength = (uint)RoundUp(alignedReadLength, sectorSize); var record = bufferPool.Get((int)alignedReadLength); record.valid_offset = (int)(fileOffset - alignedFileOffset); - record.available_bytes = (int)(alignedReadLength - (fileOffset - alignedFileOffset)); + record.available_bytes = (int)(alignedReadLength - record.valid_offset); record.required_bytes = numBytes; - - var asyncResult = default(AsyncGetFromDiskResult>); - asyncResult.context = context; - asyncResult.context.record = record; - device.ReadAsync(alignedFileOffset, - (IntPtr)asyncResult.context.record.aligned_pointer, - alignedReadLength, - callback, - asyncResult); + return record; } - /// - /// Read record to memory - simple read context version - /// - /// - /// - /// - /// - internal unsafe void AsyncReadRecordToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, ref SimpleReadContext context) - { - var fileOffset = (ulong)(AlignedPageSizeBytes * (fromLogical >> LogPageSizeBits) + (fromLogical & PageSizeMask)); - var alignedFileOffset = (ulong)(((long)fileOffset / sectorSize) * sectorSize); - - var alignedReadLength = (uint)((long)fileOffset + numBytes - (long)alignedFileOffset); - alignedReadLength = (uint)((alignedReadLength + (sectorSize - 1)) & ~(sectorSize - 1)); - - context.record = bufferPool.Get((int)alignedReadLength); - context.record.valid_offset = (int)(fileOffset - alignedFileOffset); - context.record.available_bytes = (int)(alignedReadLength - (fileOffset - alignedFileOffset)); - context.record.required_bytes = numBytes; - - device.ReadAsync(alignedFileOffset, - (IntPtr)context.record.aligned_pointer, - alignedReadLength, - callback, - context); - } + /// Read pages from specified device(s) for recovery, with no output of the countdown event (but it is still created in the + /// and thus must be Dispose()d). + public void AsyncReadPagesForRecovery(long readPageStart, int numPages, long untilAddress, TContext context, + long devicePageOffset = 0, IDevice logDevice = null, IDevice objectLogDevice = null) + => AsyncReadPagesForRecovery(readPageStart, numPages, untilAddress, context, out _, devicePageOffset, logDevice, objectLogDevice); - /// Read pages from specified device - public void AsyncReadPagesFromDevice( - long readPageStart, - int numPages, - long untilAddress, - DeviceIOCompletionCallback callback, - TContext context, - long devicePageOffset = 0, - IDevice logDevice = null, IDevice objectLogDevice = null) - => AsyncReadPagesFromDevice(readPageStart, numPages, untilAddress, callback, context, out _, devicePageOffset, logDevice, objectLogDevice); - - /// Read pages from specified device - private void AsyncReadPagesFromDevice( - long readPageStart, - int numPages, - long untilAddress, - DeviceIOCompletionCallback callback, - TContext context, - out CountdownEvent completed, - long devicePageOffset = 0, - IDevice device = null, IDevice objectLogDevice = null) + /// Read pages from specified device for recovery, returning the countdown event + [MethodImpl(MethodImplOptions.NoInlining)] + private void AsyncReadPagesForRecovery(long readPageStart, int numPages, long untilAddress, TContext context, + out CountdownEvent completed, long devicePageOffset = 0, IDevice logDevice = null, IDevice objectLogDevice = null) { - var usedDevice = device ?? this.device; - IDevice usedObjlogDevice = objectLogDevice; + var usedDevice = logDevice ?? this.device; completed = new CountdownEvent(numPages); for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++) { var pageIndex = (int)(readPage % BufferSize); - if (!_wrapper.IsAllocated(pageIndex)) + if (!IsAllocated(pageIndex)) _wrapper.AllocatePage(pageIndex); else - _wrapper.ClearPage(readPage); + ClearPage(readPage, offset: 0); var asyncResult = new PageAsyncReadResult() { page = readPage, - offset = devicePageOffset, + devicePageOffset = devicePageOffset, context = context, handle = completed, - maxPtr = PageSize + maxAddressOffsetOnPage = PageSize, + isForRecovery = true }; var offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); var readLength = (uint)AlignedPageSizeBytes; - long adjustedUntilAddress = AlignedPageSizeBytes * (untilAddress >> LogPageSizeBits) + (untilAddress & PageSizeMask); + long adjustedUntilAddress = AlignedPageSizeBytes * GetPage(untilAddress) + GetOffsetOnPage(untilAddress); if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) { readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); - asyncResult.maxPtr = readLength; + asyncResult.maxAddressOffsetOnPage = readLength; readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); } - if (device != null) + // If device != null then it is the snapshot file device. In that case we may have an offset into it due to FlushedUntilAddress + // having advanced; see Recovery.cs:RecoverHybridLog. + if (logDevice != null) offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); - ReadAsync(offsetInFile, pageIndex, readLength, callback, asyncResult, usedDevice, usedObjlogDevice); + // Create separate readBuffers for each main-log page, as each page launches its own async read and callbacks are on different threads. + // Do *not* use "using" here as we need it to survive to the ReadAsync AsyncReadPagesForRecoveryCallback. + asyncResult.readBuffers = CreateCircularReadBuffers(objectLogDevice, logger); + + // Call the overridden ReadAsync for the derived allocator class + ReadAsync(offsetInFile, (IntPtr)pagePointers[pageIndex], readLength, AsyncReadPagesForRecoveryCallback, asyncResult, usedDevice); } } /// - /// Flush page range to disk - /// Called when all threads have agreed that a page range is sealed. + /// Flush page range to disk. Called when all threads have agreed that a page range is sealed. /// - /// - /// - /// - public void AsyncFlushPages(long fromAddress, long untilAddress, bool noFlush = false) + /// + /// This is called synchronously from OnPagesMarkedReadOnly to kick off a flush sequence of (possibly multiple and/or partial) pages. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + internal virtual void AsyncFlushPagesForReadOnly(long fromAddress, long untilAddress, bool noFlush = false) { - long startPage = fromAddress >> LogPageSizeBits; - long endPage = untilAddress >> LogPageSizeBits; - var numPages = (int)(endPage - startPage); - - long offsetInEndPage = GetOffsetInPage(untilAddress); - - // Extra (partial) page being flushed - if (offsetInEndPage > 0) - numPages++; + // This is the base implementation, used by TsavoriteLog and SpanByteAllocator; it is overridden by ObjectAllocatorImpl to handle object log flushes. + GetFlushPageRange(fromAddress, untilAddress, out var startPage, out var numPages); - /* Request asynchronous writes to the device. If waitForPendingFlushComplete - * is set, then a CountDownEvent is set in the callback handle. - */ - for (long flushPage = startPage; flushPage < (startPage + numPages); flushPage++) + // Write each page (or partial page) in the range. + for (var flushPage = startPage; flushPage < (startPage + numPages); flushPage++) { - long pageStartAddress = flushPage << LogPageSizeBits; - long pageEndAddress = (flushPage + 1) << LogPageSizeBits; - - var asyncResult = new PageAsyncFlushResult - { - page = flushPage, - count = 1, - partial = false, - fromAddress = pageStartAddress, - untilAddress = pageEndAddress - }; - if ( - ((fromAddress > pageStartAddress) && (fromAddress < pageEndAddress)) || - ((untilAddress > pageStartAddress) && (untilAddress < pageEndAddress)) - ) + if (!PrepareFlushAsyncResult(fromAddress, untilAddress, noFlush, flushPage, out var asyncResult)) + continue; + + // If there are partial pages, we need to wait until the ongoing prior adjacent flush is completed to ensure correctness; otherwise, given + // that we write in multiples of sector size, if the adjacent fragments do not end/start on sector alignment there may be a race where the + // final sector in the prior fragment (which is incomplete) overwrites the same sector (which was completed) in the next adjacent fragment. + // To accomplish this we use PendingFlush and AsyncFlushPageCallback chains to the next fragment in the sequence. TsavoriteLog in particular + // does fine-grained flushing and this is critical for its performance and correctness. + var index = GetPageIndexForAddress(asyncResult.fromAddress); + if (GetOffsetOnPage(asyncResult.fromAddress) > 0) { - asyncResult.partial = true; + // Try to merge request with existing adjacent (earlier) pending requests (these have not yet begun or they would not be in the queue). + while (PendingFlush[index].RemovePreviousAdjacent(asyncResult.fromAddress, out var existingRequest)) + asyncResult.fromAddress = existingRequest.fromAddress; - if (untilAddress < pageEndAddress) - asyncResult.untilAddress = untilAddress; + // Enqueue the (possibly merged) new work item into the queue. + PendingFlush[index].Add(asyncResult); - if (fromAddress > pageStartAddress) - asyncResult.fromAddress = fromAddress; + // Perform work from shared queue if possible: When a flush completes it updates FlushedUntilAddress. If there is an item in the shared + // queue that starts at FlushedUntilAddress, it can now be flushed. Flush callbacks will RemoveNextAdjacent(FlushedUntilAddress, ...) + // to continue the chain of flushes until the queue is empty. This will issue a write that completes in the background as we move to the + // next adjacent chunk (or page if this is the last chunk on the current page). + if (PendingFlush[index].RemoveNextAdjacent(FlushedUntilAddress, out PageAsyncFlushResult request)) + WriteAsync(GetPage(request.fromAddress), AsyncFlushPageCallback, request); // Call the overridden WriteAsync for the derived allocator class + continue; } - bool skip = false; - if (asyncResult.untilAddress <= BeginAddress) - { - // Short circuit as no flush needed - _ = Utility.MonotonicUpdate(ref PageStatusIndicator[flushPage % BufferSize].LastFlushedUntilAddress, BeginAddress, out _); - ShiftFlushedUntilAddress(); - skip = true; - } + // This is the start of a possibly partial page range; if partial, there may be elements in the PendingFlush array for this index. + // The flush will issue a write that completes in the background as we move to the next range to flush; if this was a partial range + // then the PendingFlush list will be drained via chained WriteAsync in the callbacks. + WriteAsync(flushPage, AsyncFlushPageCallback, asyncResult); // Call the overridden WriteAsync for the derived allocator class + } + } - if (IsNullDevice || noFlush) - { - // Short circuit as no flush needed - _ = Utility.MonotonicUpdate(ref PageStatusIndicator[flushPage % BufferSize].LastFlushedUntilAddress, asyncResult.untilAddress, out _); - ShiftFlushedUntilAddress(); - skip = true; - } + private protected void GetFlushPageRange(long fromAddress, long untilAddress, out long startPage, out long numPages) + { + startPage = GetPage(fromAddress); + var endPage = GetPage(untilAddress); + numPages = (int)(endPage - startPage); - if (skip) continue; + // Extra (partial) page being flushed + if (GetOffsetOnPage(untilAddress) > 0) + numPages++; + } - // Partial page starting point, need to wait until the - // ongoing adjacent flush is completed to ensure correctness - if (GetOffsetInPage(asyncResult.fromAddress) > 0) - { - var index = GetPageIndexForAddress(asyncResult.fromAddress); + private protected bool PrepareFlushAsyncResult(long fromAddress, long untilAddress, bool noFlush, long flushPage, out PageAsyncFlushResult asyncResult) + { + // Default to writing the full page. + var pageStartAddress = GetLogicalAddressOfStartOfPage(flushPage); + var pageEndAddress = GetLogicalAddressOfStartOfPage(flushPage + 1); - // Try to merge request with existing adjacent (earlier) pending requests - while (PendingFlush[index].RemovePreviousAdjacent(asyncResult.fromAddress, out var existingRequest)) - asyncResult.fromAddress = existingRequest.fromAddress; + asyncResult = new PageAsyncFlushResult + { + page = flushPage, + count = 1, + partial = false, + fromAddress = pageStartAddress, + untilAddress = pageEndAddress + }; + + // If either fromAddress or untilAddress is in the middle of the page, this will be a partial page flush. + asyncResult.partial = ((fromAddress > pageStartAddress) && (fromAddress < pageEndAddress)) + || ((untilAddress > pageStartAddress) && (untilAddress < pageEndAddress)); + if (asyncResult.partial) + { + if (untilAddress < pageEndAddress) + asyncResult.untilAddress = untilAddress; + if (fromAddress > pageStartAddress) + asyncResult.fromAddress = fromAddress; + } - // Enqueue work in shared queue - PendingFlush[index].Add(asyncResult); + // We skip if either of the following, and need to check both. + var skip = false; + if (asyncResult.untilAddress <= BeginAddress) + { + // Short circuit as no flush needed; just advance the flushed until address to BeginAddress. + _ = MonotonicUpdate(ref PageStatusIndicator[flushPage % BufferSize].LastFlushedUntilAddress, BeginAddress, out _); + ShiftFlushedUntilAddress(); + skip = true; + } - // Perform work from shared queue if possible - if (PendingFlush[index].RemoveNextAdjacent(FlushedUntilAddress, out PageAsyncFlushResult request)) - WriteAsync(request.fromAddress >> LogPageSizeBits, AsyncFlushPageCallback, request); - } - else - WriteAsync(flushPage, AsyncFlushPageCallback, asyncResult); + if (IsNullDevice || noFlush) + { + // Short circuit as no flush needed; just advance the flushed until address to untilAddress. + _ = MonotonicUpdate(ref PageStatusIndicator[flushPage % BufferSize].LastFlushedUntilAddress, asyncResult.untilAddress, out _); + ShiftFlushedUntilAddress(); + skip = true; } + + return !skip; } /// - /// Flush pages asynchronously + /// Flush pages asynchronously for recovery (such as when we have invalidated v+1 records). /// - /// - /// - /// - /// - /// - public void AsyncFlushPages(long flushPageStart, int numPages, DeviceIOCompletionCallback callback, TContext context) + public void AsyncFlushPagesForRecovery(long scanFromAddress, long flushPageStart, int numPages, DeviceIOCompletionCallback callback, TContext context) { - for (long flushPage = flushPageStart; flushPage < (flushPageStart + numPages); flushPage++) + Debug.Assert(scanFromAddress < GetLogicalAddressOfStartOfPage(flushPageStart + 1), $"scanFromAddress ({scanFromAddress}) must be on flushPageStart ({flushPageStart})"); + for (var flushPage = flushPageStart; flushPage < (flushPageStart + numPages); flushPage++) { var asyncResult = new PageAsyncFlushResult() { @@ -1636,34 +1839,40 @@ public void AsyncFlushPages(long flushPageStart, int numPages, DeviceI context = context, count = 1, partial = false, - untilAddress = (flushPage + 1) << LogPageSizeBits + fromAddress = Math.Max(scanFromAddress, GetLogicalAddressOfStartOfPage(flushPage)), + untilAddress = GetLogicalAddressOfStartOfPage(flushPage + 1), + flushRequestState = FlushRequestState.Recovery }; + // For OA, we do not use FlushBuffers here; we set isForRecovery to reuse the stored lengths rather than re-serializing objects, + // using the lengths filled in during deserialization in RecoverHybridLog(Async), and when that is complete we fill in objectLogTail. WriteAsync(flushPage, callback, asyncResult); } } /// - /// Flush pages from startPage (inclusive) to endPage (exclusive) - /// to specified log device and obj device + /// Flush pages from startPage (inclusive) to endPage (exclusive) to specified log device and obj device for a snapshot checkpoint. /// + /// /// /// + /// /// /// - /// + /// /// - /// + /// Task that completes when all pages are flushed, or faults if an exception occurs /// - public void AsyncFlushPagesToDevice(long startPage, long endPage, long endLogicalAddress, long fuzzyStartLogicalAddress, IDevice device, IDevice objectLogDevice, out SemaphoreSlim completedSemaphore, int throttleCheckpointFlushDelayMs) + [MethodImpl(MethodImplOptions.NoInlining)] + public void AsyncFlushPagesForSnapshot(CircularDiskWriteBuffer flushBuffers, long startPage, long endPage, long startLogicalAddress, long endLogicalAddress, + long fuzzyStartLogicalAddress, IDevice logDevice, IDevice objectLogDevice, out Task completedTask, int throttleCheckpointFlushDelayMs) { logger?.LogTrace("Starting async full log flush with throttling {throttlingEnabled}", throttleCheckpointFlushDelayMs >= 0 ? $"enabled ({throttleCheckpointFlushDelayMs}ms)" : "disabled"); - var _completedSemaphore = new SemaphoreSlim(0); - completedSemaphore = _completedSemaphore; + var completionTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + completedTask = completionTcs.Task; - // If throttled, convert rest of the method into a truly async task run - // because issuing IO can take up synchronous time + // If throttled, convert rest of the method into a truly async task run because issuing IO can take up synchronous time if (throttleCheckpointFlushDelayMs >= 0) _ = Task.Run(FlushRunner); else @@ -1673,40 +1882,85 @@ void FlushRunner() { var totalNumPages = (int)(endPage - startPage); - var flushCompletionTracker = new FlushCompletionTracker(_completedSemaphore, throttleCheckpointFlushDelayMs >= 0 ? new SemaphoreSlim(0) : null, totalNumPages); - var localSegmentOffsets = new long[SegmentBufferSize]; + var flushCompletionTracker = new FlushCompletionTracker(completionTcs, enableThrottling: throttleCheckpointFlushDelayMs >= 0, totalNumPages); - for (long flushPage = startPage; flushPage < endPage; flushPage++) + try { - long flushPageAddress = flushPage << LogPageSizeBits; - var pageSize = PageSize; - if (flushPage == endPage - 1) - pageSize = (int)(endLogicalAddress - flushPageAddress); - - var asyncResult = new PageAsyncFlushResult + // Flush each page in sequence + for (long flushPage = startPage; flushPage < endPage; flushPage++) { - flushCompletionTracker = flushCompletionTracker, - page = flushPage, - fromAddress = flushPageAddress, - untilAddress = flushPageAddress + pageSize, - count = 1 - }; - - // Intended destination is flushPage - WriteAsyncToDevice(startPage, flushPage, pageSize, AsyncFlushPageToDeviceCallback, asyncResult, device, objectLogDevice, localSegmentOffsets, fuzzyStartLogicalAddress); + // For the first page, startLogicalAddress may be in the middle of the page; for the last page, endLogicalAddress may be in the middle of the page; + // for middle pages, we flush the entire page. + var flushStartAddress = GetLogicalAddressOfStartOfPage(flushPage); + if (startLogicalAddress > flushStartAddress) + flushStartAddress = startLogicalAddress; + var flushEndAddress = GetLogicalAddressOfStartOfPage(flushPage + 1); + if (endLogicalAddress < flushEndAddress) + flushEndAddress = endLogicalAddress; + var flushSize = flushEndAddress - flushStartAddress; + if (flushSize <= 0) + { + // No data to flush for this page. Signal completion and drain the + // throttle semaphore so the next real page's WaitOneFlush is not + // satisfied by this page's release. + flushCompletionTracker.CompleteFlush(); + flushCompletionTracker.WaitOneFlush(); + continue; + } - if (throttleCheckpointFlushDelayMs >= 0) - { - flushCompletionTracker.WaitOneFlush(); - Thread.Sleep(throttleCheckpointFlushDelayMs); + var asyncResult = new PageAsyncFlushResult + { + flushCompletionTracker = flushCompletionTracker, + page = flushPage, + fromAddress = flushStartAddress, + untilAddress = flushEndAddress, + count = 1, + flushRequestState = FlushRequestState.Snapshot, + flushBuffers = flushBuffers + }; + + // Intended destination is flushPage + WriteAsyncToDeviceForSnapshot(startPage, flushPage, (int)flushSize, AsyncFlushPageForSnapshotCallback, asyncResult, logDevice, objectLogDevice, fuzzyStartLogicalAddress); + + // If we did not issue a flush write (due to HeadAddress moving past flushPage), then WriteAsync set isForSnapshot false and we release the asyncResult here; + // otherwise, we wait for the completion of the flush (and the callback will release the asyncResult). + if (asyncResult.flushRequestState != FlushRequestState.WriteNotIssued) + { + if (throttleCheckpointFlushDelayMs >= 0) + { + flushCompletionTracker.WaitOneFlush(); + Thread.Sleep(throttleCheckpointFlushDelayMs); + } + } + else + { + _ = asyncResult.Release(); + // Release() called CompleteFlush() which released the throttle semaphore. + // Drain it so the next real page's WaitOneFlush is not satisfied by this no-op. + flushCompletionTracker.WaitOneFlush(); + } } } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed while flushing snapshot pages from {startPage} to {endPage}", nameof(AsyncFlushPagesForSnapshot), startPage, endPage); + flushCompletionTracker.SetException(ex); + } } } - internal void AsyncGetFromDisk(long fromLogical, int numBytes, AsyncIOContext context, SectorAlignedMemory result = default) + /// + /// Get a single record from the disk. + /// + /// Start of the record + /// Number of bytes to be read (may be less than actual record size) + /// The of the operation. This is passed by value, not reference; in the iterator case, it is + /// the completionEvent's contained request, and populating it will result in prematurely freeing the record. + internal void AsyncGetFromDisk(long fromLogicalAddress, int numBytes, AsyncIOContext context) { - if (epoch.ThisInstanceProtected()) // Do not spin for unprotected IO threads + // If this is a protected thread, we must wait to issue the Read operation. Spin until the device is not throttled, + // draining events on each iteration, but do not release the epoch. + if (epoch.ThisInstanceProtected()) { while (device.Throttle()) { @@ -1716,66 +1970,184 @@ internal void AsyncGetFromDisk(long fromLogical, int numBytes, AsyncIOContext + /// Read pages from specified device + /// + [MethodImpl(MethodImplOptions.NoInlining)] + internal void AsyncReadPageFromDeviceToFrame(CircularDiskReadBuffer readBuffers, + long readPage, + long untilAddress, + DeviceIOCompletionCallback callback, + TContext context, + BlittableFrame frame, + out CountdownEvent completed, + long devicePageOffset = 0, + IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) + { + var usedDevice = device ?? this.device; + + completed = new CountdownEvent(1); + + int pageIndex = (int)(readPage % frame.frameSize); + if (frame.frame[pageIndex] == null) + frame.Allocate(pageIndex); else - AsyncReadRecordObjectsToMemory(fromLogical, numBytes, AsyncGetFromDiskCallback, context, result); + frame.Clear(pageIndex); + + var asyncResult = new PageAsyncReadResult() + { + page = readPage, + context = context, + handle = completed, + cts = cts, + readBuffers = readBuffers + }; + + ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); + uint readLength = (uint)AlignedPageSizeBytes; + long adjustedUntilAddress = AlignedPageSizeBytes * GetPage(untilAddress) + GetOffsetOnPage(untilAddress); + + if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) + { + readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); + readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); + } + + if (device != null) + offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); + + ReadAsync(offsetInFile, (IntPtr)frame.GetPhysicalAddress(pageIndex), readLength, callback, asyncResult, usedDevice); + } + + /// + /// Checks to see if we have a full record, or at least enough to compare the key. + /// + /// The context from the IO operation + /// If we return false, the address to issue the next IO for + /// If we return false, the number of bytes to issue the next IO for + /// True if we have the full record and the key was the requested key; if the record is fully inline, then the ctx.diskLogRecord is set and the ctx.record is transferred to it. + /// Otherwise it is false, and: + /// + /// If the key was present, it did not match ctx.requestKey; is recordInfo.PreviousAddress, and + /// is the initial IO size. + /// Otherwise, the data we have is not sufficient to determine record length, or we know the length and it is greater than the data we have now. + /// is the same address we just read, and is one of: + /// + /// If we did not have enough data to determine required length, we use the initial IO size. This should seldom happen as we issue the initial + /// IO request with this size, but perhaps this is called with a partial buffer. + /// Otherwise, we know the data length needed, and we set to that. + /// + /// + /// + /// If we have a complete record and the key passes the comparison and we have overflow or objects, then this will be overridden by a derived class (see + /// ) which will issue additional reads to retrieve those objects. + private protected virtual bool VerifyRecordFromDiskCallback(ref AsyncIOContext ctx, out long prevAddressToRead, out int prevLengthToRead) + { + // TODO: Optimize for non-ReadAtAddress tombstoned records to not have to retrieve the full record or, if we have it, not deserialize objects. + + // Initialize to "key is not present (data too small) or does not match so get previous record" length to read + prevLengthToRead = IStreamBuffer.InitialIOSize; + + // See if we have a complete record. + var currentLength = ctx.record.available_bytes; + if (currentLength >= RecordInfo.Size + RecordDataHeader.MinHeaderBytes) + { + var ptr = ctx.record.GetValidPointer(); + var recordInfo = *(RecordInfo*)ptr; + var dataHeader = new RecordDataHeader(ptr + RecordInfo.Size); + var (numKeyLengthBytes, numRecordLengthBytes) = dataHeader.DeconstructKVByteLengths(out var headerLength); + + // GetRecordLength is always safe, because it is in the second sizeof(ulong) and we round up to 8-byte alignment. + var recordLength = dataHeader.GetRecordLength(numRecordLengthBytes); + if (currentLength <= headerLength) + { + prevLengthToRead = recordLength; + goto RereadCurrent; + } + + // Initialize to "invalid record or key does not match so get previous record" address to read + prevAddressToRead = recordInfo.PreviousAddress; + + if (recordInfo.Invalid) // includes IsNull + return false; + + var offsetToKeyStart = dataHeader.GetOffsetToKeyStart(headerLength); + + // If the length is up to offsetToKeyStart, we can read the full lengths. If not, we'll fall through to reread the current record. + if (currentLength >= offsetToKeyStart) + { + var keyLength = dataHeader.GetKeyLength(numKeyLengthBytes, numRecordLengthBytes); + var keyStartPtr = ptr + offsetToKeyStart; + + // We have the full key if it is inline, so check for a match if we had a requested key, and return if not. + if (!ctx.requestKey.IsEmpty && recordInfo.KeyIsInline && !storeFunctions.KeysEqual(ctx.requestKey, dataHeader)) + return false; + + // Keys match. If we have the full record, return success; otherwise we'll drop through to read the full record with the length we now know. + if (currentLength >= recordLength) + { + ctx.diskLogRecord = DiskLogRecord.TransferFrom(ref ctx.record, transientObjectIdMap); + ctx.diskLogRecord.InfoRef.ClearBitsForDiskImages(); + if (storeFunctions.CallOnDiskRead) + storeFunctions.OnDiskRead(ref ctx.diskLogRecord.logRecord); + return true; + } + } + } + + RereadCurrent: + // Either we didn't have the full record size, or we didn't have enough bytes to even read the full record size. Either way, prevLengthToRead + // is set for a re-read of the same record. + prevAddressToRead = ctx.logicalAddress; + return false; } - private unsafe void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, object context) + [MethodImpl(MethodImplOptions.NoInlining)] + private void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, object context) { if (errorCode != 0) - logger?.LogError("AsyncGetFromDiskCallback error: {0}", errorCode); + logger?.LogError("AsyncGetFromDiskCallback error: {errorCode}", errorCode); - var result = (AsyncGetFromDiskResult>)context; + var result = (AsyncGetFromDiskResult)context; var ctx = result.context; try { - var record = ctx.record.GetValidPointer(); - int requiredBytes = _wrapper.GetRequiredRecordSize((long)record, ctx.record.available_bytes); - if (ctx.record.available_bytes >= requiredBytes) - { - Debug.Assert(!_wrapper.GetInfoFromBytePointer(record).Invalid, "Invalid records should not be in the hash chain for pending IO"); + // Note: don't test for (numBytes >= ctx.record.required_bytes) for this initial read, as the file may legitimately end before the + // InitialIOSize request can be fulfilled. + ctx.record.available_bytes = (int)numBytes; - // We have all the required bytes. If we don't have the complete record, RetrievedFullRecord calls AsyncGetFromDisk. - if (!_wrapper.RetrievedFullRecord(record, ref ctx)) - return; + Debug.Assert(!(*(RecordInfo*)ctx.record.GetValidPointer()).Invalid, $"Invalid records should not be in the hash chain for pending IO; address {ctx.logicalAddress}"); - // If request_key is null we're called from ReadAtAddress, so it is an implicit match. - if (ctx.request_key is not null && !_storeFunctions.KeysEqual(ref ctx.request_key.Get(), ref _wrapper.GetContextRecordKey(ref ctx))) + if (!VerifyRecordFromDiskCallback(ref ctx, out var prevAddressToRead, out var prevLengthToRead)) + { + // Either we had an incomplete record and we're re-reading the current record, or the record Key didn't match and we're reading the previous record + // in the chain. If the record to read is in the range to resolve then issue the read, else fall through to signal "IO complete". + ctx.logicalAddress = prevAddressToRead; + if (ctx.logicalAddress >= BeginAddress && ctx.logicalAddress >= ctx.minAddress) { - // Keys don't match so request the previous record in the chain if it is in the range to resolve. - ctx.logicalAddress = _wrapper.GetInfoFromBytePointer(record).PreviousAddress; - if (ctx.logicalAddress >= BeginAddress && ctx.logicalAddress >= ctx.minAddress) - { - ctx.record.Return(); - ctx.record = ctx.objBuffer = default; - AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, ctx); - return; - } + _wrapper.OnDisposeDiskRecord(ref ctx.diskLogRecord, DisposeReason.DeserializedFromDisk); + ctx.DisposeRecord(); + AsyncGetFromDisk(ctx.logicalAddress, prevLengthToRead, ctx); + return; } - - // Either the keys match or we are below the range to retrieve (which ContinuePending* will detect), so we're done. - if (ctx.completionEvent is not null) - ctx.completionEvent.Set(ref ctx); - else if (ctx.callbackQueue is not null) - ctx.callbackQueue.Enqueue(ctx); - else - _ = ctx.asyncOperation.TrySetResult(ctx); } + + // Either we have a full record with a key match or we are below the range to retrieve (which ContinuePending* will detect), so we're done. + if (ctx.completionEvent is not null) + ctx.completionEvent.Set(ref ctx); else - { - ctx.record.Return(); - AsyncGetFromDisk(ctx.logicalAddress, requiredBytes, ctx); - } + ctx.callbackQueue.Enqueue(ctx); } catch (Exception e) { logger?.LogError(e, "AsyncGetFromDiskCallback error"); + _wrapper.OnDisposeDiskRecord(ref ctx.diskLogRecord, DisposeReason.DeserializedFromDisk); + ctx.DisposeRecord(); if (ctx.completionEvent is not null) ctx.completionEvent.SetException(e); - else if (ctx.asyncOperation is not null) - _ = ctx.asyncOperation.TrySetException(e); else throw; } @@ -1787,39 +2159,40 @@ private unsafe void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, obje /// /// /// - private void AsyncFlushPageCallback(uint errorCode, uint numBytes, object context) + [MethodImpl(MethodImplOptions.NoInlining)] + private protected void AsyncFlushPageCallback(uint errorCode, uint numBytes, object context) { try { if (errorCode != 0) - logger?.LogError("AsyncFlushPageCallback error: {0}", errorCode); + logger?.LogError("AsyncFlushPageCallback error: {errorCode}", errorCode); // Set the page status to flushed var result = (PageAsyncFlushResult)context; - if (Interlocked.Decrement(ref result.count) == 0) + if (result.Release() == 0) { if (errorCode != 0) { - // Note down error details and trigger handling only when we are certain this is the earliest - // error among currently issued flushes + // Note down error details and trigger handling only when we are certain this is the earliest error among currently issued flushes errorList.Add(new CommitInfo { FromAddress = result.fromAddress, UntilAddress = result.untilAddress, ErrorCode = errorCode }); } else { - // Update the page's last flushed until address only if there is no failure. - _ = Utility.MonotonicUpdate( - ref PageStatusIndicator[result.page % BufferSize].LastFlushedUntilAddress, - result.untilAddress, out _); + // There is no failure so update the page's last flushed until address. + _ = MonotonicUpdate(ref PageStatusIndicator[result.page % BufferSize].LastFlushedUntilAddress, result.untilAddress, out _); } ShiftFlushedUntilAddress(); - result.Free(); } + // Continue the chained flushes, popping the next request from the queue if it is adjacent. var _flush = FlushedUntilAddress; - if (GetOffsetInPage(_flush) > 0 && PendingFlush[GetPage(_flush) % BufferSize].RemoveNextAdjacent(_flush, out PageAsyncFlushResult request)) - WriteAsync(request.fromAddress >> LogPageSizeBits, AsyncFlushPageCallback, request); + if (GetOffsetOnPage(_flush) > 0 && PendingFlush[GetPage(_flush) % BufferSize].RemoveNextAdjacent(_flush, out PageAsyncFlushResult request)) + { + request.flushBuffers = result.flushBuffers; // Reuse the flush buffers from the completed flush to continue the flush chain + WriteAsync(GetPage(request.fromAddress), AsyncFlushPageCallback, request); // Call the overridden WriteAsync for the derived allocator class + } } catch when (disposed) { } } @@ -1829,44 +2202,43 @@ internal void UnsafeSkipError(CommitInfo info) try { errorList.TruncateUntil(info.UntilAddress); - var page = info.FromAddress >> PageSizeMask; - _ = Utility.MonotonicUpdate(ref PageStatusIndicator[page % BufferSize].LastFlushedUntilAddress, info.UntilAddress, out _); + var page = GetPage(info.FromAddress); + _ = MonotonicUpdate(ref PageStatusIndicator[page % BufferSize].LastFlushedUntilAddress, info.UntilAddress, out _); ShiftFlushedUntilAddress(); var _flush = FlushedUntilAddress; - if (GetOffsetInPage(_flush) > 0 && PendingFlush[GetPage(_flush) % BufferSize].RemoveNextAdjacent(_flush, out PageAsyncFlushResult request)) - WriteAsync(request.fromAddress >> LogPageSizeBits, AsyncFlushPageCallback, request); + if (GetOffsetOnPage(_flush) > 0 && PendingFlush[GetPage(_flush) % BufferSize].RemoveNextAdjacent(_flush, out PageAsyncFlushResult request)) + { + // Reuse the flush buffers from the completed flush to continue the flush chain + WriteAsync(GetPage(request.fromAddress), AsyncFlushPageCallback, request); // Call the overridden WriteAsync for the derived allocator class + } } catch when (disposed) { } } /// - /// IOCompletion callback for page flush + /// IOCompletion callback for page flush for snapshot checkpoint /// /// /// /// - protected void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, object context) + protected void AsyncFlushPageForSnapshotCallback(uint errorCode, uint numBytes, object context) { try { if (errorCode != 0) - logger?.LogError("AsyncFlushPageToDeviceCallback error: {0}", errorCode); + logger?.LogError("AsyncFlushPageToDeviceCallback error: {errorCode}", errorCode); var result = (PageAsyncFlushResult)context; + var epochTaken = epoch.ResumeIfNotProtected(); - var epochTaken = false; - if (!epoch.ThisInstanceProtected()) - { - epochTaken = true; - epoch.Resume(); - } - - // Unset dirty bit for flushed pages try { - var startAddress = result.page << LogPageSizeBits; + var startAddress = GetLogicalAddressOfStartOfPage(result.page); var endAddress = startAddress + PageSize; + // First make sure we're not trying to process a logical address that's in a page header. + startAddress += PageHeader.Size; + if (result.fromAddress > startAddress) startAddress = result.fromAddress; if (result.untilAddress < endAddress) @@ -1882,16 +2254,15 @@ protected void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, obj if (flushWidth > 0) { - var physicalAddress = _wrapper.GetPhysicalAddress(startAddress); + var physicalAddress = GetPhysicalAddress(startAddress); var endPhysicalAddress = physicalAddress + flushWidth; while (physicalAddress < endPhysicalAddress) { - ref var info = ref _wrapper.GetInfo(physicalAddress); - var (_, alignedRecordSize) = _wrapper.GetRecordSize(physicalAddress); - if (info.Dirty) - info.ClearDirtyAtomic(); // there may be read locks being taken, hence atomic + var logRecord = _wrapper.CreateLogRecord(startAddress); + var alignedRecordSize = logRecord.AllocatedSize; physicalAddress += alignedRecordSize; + startAddress += alignedRecordSize; } } } @@ -1899,14 +2270,12 @@ protected void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, obj { if (epochTaken) epoch.Suspend(); + _ = result.Release(); } - - if (Interlocked.Decrement(ref result.count) == 0) - result.Free(); } catch when (disposed) { } } - internal string PrettyPrintLogicalAddress(long logicalAddress) => $"{logicalAddress}:{GetPage(logicalAddress)}.{GetOffsetInPage(logicalAddress)}"; + internal string PrettyPrintLogicalAddress(long logicalAddress) => $"{logicalAddress}:{GetPage(logicalAddress)}.{GetOffsetOnPage(logicalAddress)}"; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorRecord.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorRecord.cs deleted file mode 100644 index 17543c839e0..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorRecord.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.InteropServices; - -#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member - -namespace Tsavorite.core -{ - [StructLayout(LayoutKind.Sequential, Pack = 1)] - public struct AllocatorRecord - { - public RecordInfo info; - public TKey key; - public TValue value; - - public override string ToString() - { - var keyString = key?.ToString() ?? "null"; - if (keyString.Length > 20) - keyString = keyString.Substring(0, 20) + "..."; - var valueString = value?.ToString() ?? "null"; ; - if (valueString.Length > 20) - valueString = valueString.Substring(0, 20) + "..."; - return $"{keyString} | {valueString} | {info}"; - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs index d3df96a17b2..fdc0580cdda 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorScan.cs @@ -8,32 +8,39 @@ namespace Tsavorite.core { - public abstract partial class AllocatorBase : IDisposable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + + public abstract partial class AllocatorBase : IDisposable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Pull-based scan interface for HLOG; user calls GetNext() which advances through the address range. /// /// Pull Scan iterator instance - public abstract ITsavoriteScanIterator Scan(TsavoriteKV store, long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering, bool includeClosedRecords = false); + public abstract ITsavoriteScanIterator Scan(TsavoriteKV store, long beginAddress, long endAddress, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool includeClosedRecords = false); /// /// Push-based scan interface for HLOG, called from LogAccessor; scan the log given address range, calling for each record. /// /// True if Scan completed; false if Scan ended early due to one of the TScanIterator reader functions returning false - internal abstract bool Scan(TsavoriteKV store, long beginAddress, long endAddress, ref TScanFunctions scanFunctions, - ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering) - where TScanFunctions : IScanIteratorFunctions; + internal abstract bool Scan(TsavoriteKV store, long beginAddress, long endAddress, ref TScanFunctions scanFunctions, + DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering) + where TScanFunctions : IScanIteratorFunctions; /// /// Push-based iteration of key versions, calling for each record. /// /// True if Scan completed; false if Scan ended early due to one of the TScanIterator reader functions returning false - internal bool IterateKeyVersions(TsavoriteKV store, ref TKey key, ref TScanFunctions scanFunctions) - where TScanFunctions : IScanIteratorFunctions + internal bool IterateKeyVersions(TsavoriteKV store, TKey key, ref TScanFunctions scanFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TScanFunctions : IScanIteratorFunctions { - OperationStackContext stackCtx = new(_storeFunctions.GetKeyHashCode64(ref key)); + OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(key)); if (!store.FindTag(ref stackCtx.hei)) return false; stackCtx.SetRecordSourceToHashEntry(store.hlogBase); @@ -41,41 +48,41 @@ internal bool IterateKeyVersions(TsavoriteKV /// Push-based iteration of key versions, calling for each record. /// /// True if Scan completed; false if Scan ended early due to one of the TScanIterator reader functions returning false - internal abstract bool IterateKeyVersions(TsavoriteKV store, ref TKey key, long beginAddress, ref TScanFunctions scanFunctions) - where TScanFunctions : IScanIteratorFunctions; + internal abstract bool IterateKeyVersions(TsavoriteKV store, TKey key, long beginAddress, ref TScanFunctions scanFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TScanFunctions : IScanIteratorFunctions; /// /// Implementation for push-scanning Tsavorite log /// internal bool PushScanImpl(long beginAddress, long endAddress, ref TScanFunctions scanFunctions, TScanIterator iter) - where TScanFunctions : IScanIteratorFunctions - where TScanIterator : ITsavoriteScanIterator, IPushScanIterator + where TScanFunctions : IScanIteratorFunctions + where TScanIterator : ITsavoriteScanIterator, IPushScanIterator { if (!scanFunctions.OnStart(beginAddress, endAddress)) return false; - var headAddress = HeadAddress; long numRecords = 1; var stop = false; - for (; !stop && iter.GetNext(out var recordInfo); ++numRecords) + for (; !stop && iter.GetNext(); numRecords++) { try { - // Pull Iter records are in temp storage so do not need locks, but we'll call ConcurrentReader because, for example, GenericAllocator - // may need to know the object is in that region. - if (recordInfo.IsClosed) // Iterator checks this but it may have changed since + if (iter.Info.IsClosed) // Iterator checks this but it may have changed since continue; - if (iter.CurrentAddress >= headAddress) - stop = !scanFunctions.ConcurrentReader(ref iter.GetKey(), ref iter.GetValue(), new RecordMetadata(recordInfo, iter.CurrentAddress), numRecords, out _); - else - stop = !scanFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), new RecordMetadata(recordInfo, iter.CurrentAddress), numRecords, out _); + + // Pull Iter records are in temp storage so do not need locks. + stop = !scanFunctions.Reader(in iter, new RecordMetadata(iter.CurrentAddress), numRecords, out _); } catch (Exception ex) { @@ -91,29 +98,30 @@ internal bool PushScanImpl(long beginAddress, lon /// /// Implementation for push-iterating key versions /// - internal bool IterateKeyVersionsImpl(TsavoriteKV store, ref TKey key, long beginAddress, ref TScanFunctions scanFunctions, TScanIterator iter) - where TScanFunctions : IScanIteratorFunctions - where TScanIterator : ITsavoriteScanIterator, IPushScanIterator + internal bool IterateHashChain(TsavoriteKV store, TKey key, long beginAddress, ref TScanFunctions scanFunctions, TScanIterator iter) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TScanFunctions : IScanIteratorFunctions + where TScanIterator : ITsavoriteScanIterator, IPushScanIterator { - if (!scanFunctions.OnStart(beginAddress, Constants.kInvalidAddress)) + if (!scanFunctions.OnStart(beginAddress, kInvalidAddress)) return false; - var headAddress = HeadAddress; + var readOnlyAddress = ReadOnlyAddress; long numRecords = 1; bool stop = false, continueOnDisk = false; - for (; !stop && iter.BeginGetPrevInMemory(ref key, out var recordInfo, out continueOnDisk); ++numRecords) + for (; !stop && iter.BeginGetPrevInMemory(key, out var logRecord, out continueOnDisk); numRecords++) { - OperationStackContext stackCtx = default; + OperationStackContext stackCtx = default; try { - // Iter records above headAddress will be in log memory and must be locked. - if (iter.CurrentAddress >= headAddress && !recordInfo.IsClosed) - { - store.LockForScan(ref stackCtx, ref key); - stop = !scanFunctions.ConcurrentReader(ref key, ref iter.GetValue(), new RecordMetadata(recordInfo, iter.CurrentAddress), numRecords, out _); - } - else - stop = !scanFunctions.SingleReader(ref key, ref iter.GetValue(), new RecordMetadata(recordInfo, iter.CurrentAddress), numRecords, out _); + // Iter records above readOnlyAddress will be in mutable log memory so the chain must be locked. + // We hold the epoch so iter does not need to copy, so do not use iter's ISourceLogRecord implementation; create a local LogRecord around the address. + if (iter.CurrentAddress >= readOnlyAddress && !logRecord.Info.IsClosed) + store.LockForScan(ref stackCtx, key); + stop = !scanFunctions.Reader(in logRecord, new RecordMetadata(iter.CurrentAddress), numRecords, out _); } catch (Exception ex) { @@ -130,11 +138,11 @@ internal bool IterateKeyVersionsImpl(TsavoriteKV< if (continueOnDisk) { - AsyncIOContextCompletionEvent completionEvent = new(); + AsyncIOContextCompletionEvent completionEvent = new(); try { var logicalAddress = iter.CurrentAddress; - while (!stop && GetFromDiskAndPushToReader(ref key, ref logicalAddress, ref scanFunctions, numRecords, completionEvent, out stop)) + while (!stop && GetFromDiskAndPushToReader(key, ref logicalAddress, ref scanFunctions, numRecords, completionEvent, out stop)) ++numRecords; } catch (Exception ex) @@ -152,28 +160,42 @@ internal bool IterateKeyVersionsImpl(TsavoriteKV< return !stop; } - internal unsafe bool GetFromDiskAndPushToReader(ref TKey key, ref long logicalAddress, ref TScanFunctions scanFunctions, long numRecords, - AsyncIOContextCompletionEvent completionEvent, out bool stop) - where TScanFunctions : IScanIteratorFunctions + internal bool GetFromDiskAndPushToReader(TKey key, ref long logicalAddress, ref TScanFunctions scanFunctions, long numRecords, + AsyncIOContextCompletionEvent completionEvent, out bool stop) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TScanFunctions : IScanIteratorFunctions { - completionEvent.Prepare(_wrapper.GetKeyContainer(ref key), logicalAddress); + stop = false; + if (logicalAddress < BeginAddress) + return false; - AsyncGetFromDisk(logicalAddress, _wrapper.GetAverageRecordSize(), completionEvent.request); + completionEvent.Prepare(key, logicalAddress, bufferPool); + AsyncGetFromDisk(logicalAddress, IStreamBuffer.InitialIOSize, completionEvent.request); completionEvent.Wait(); - stop = false; - if (completionEvent.exception is not null) + ref var request = ref completionEvent.request; + try { - scanFunctions.OnException(completionEvent.exception, numRecords); - return false; - } - if (completionEvent.request.logicalAddress < BeginAddress) - return false; + if (completionEvent.exception is not null) + { + scanFunctions.OnException(completionEvent.exception, numRecords); + return false; + } - RecordInfo recordInfo = _wrapper.GetInfoFromBytePointer(completionEvent.request.record.GetValidPointer()); - recordInfo.ClearBitsForDiskImages(); - stop = !scanFunctions.SingleReader(ref key, ref _wrapper.GetContextRecordValue(ref completionEvent.request), new RecordMetadata(recordInfo, completionEvent.request.logicalAddress), numRecords, out _); - logicalAddress = recordInfo.PreviousAddress; + request.diskLogRecord.InfoRef.ClearBitsForDiskImages(); + if (storeFunctions.CallOnDiskRead) + storeFunctions.OnDiskRead(ref request.diskLogRecord.logRecord); + stop = !scanFunctions.Reader(in request.diskLogRecord, new RecordMetadata(request.logicalAddress), numRecords, out _); + logicalAddress = request.diskLogRecord.Info.PreviousAddress; + } + finally + { + _wrapper.OnDisposeDiskRecord(ref request.diskLogRecord, DisposeReason.DeserializedFromDisk); + request.DisposeRecord(); + } return !stop; } @@ -184,23 +206,25 @@ internal unsafe bool GetFromDiskAndPushToReader(ref TKey key, re /// True if Scan completed and pushed records; false if Scan ended early due to finding less than records /// or one of the TScanIterator reader functions returning false /// Currently we load an entire page, which while inefficient in performance, allows us to make the cursor safe (by ensuring we align to a valid record) if it is not - /// the last one returned. We could optimize this to load only the subset of a page that is pointed to by the cursor and do GetRequiredRecordSize/RetrievedFullRecord as in + /// the last one returned. We could optimize this to load only the subset of a page that is pointed to by the cursor and use DiskLogRecord.GetSerializedRecordLength as in /// AsyncGetFromDiskCallback. However, this would not validate the cursor and would therefore require maintaining a cursor history. - internal abstract bool ScanCursor(TsavoriteKV store, ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions; - - private protected bool ScanLookup(TsavoriteKV store, - ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, TScanIterator iter, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - where TScanIterator : ITsavoriteScanIterator, IPushScanIterator + internal abstract bool ScanCursor(TsavoriteKV store, ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, + long endAddress, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions; + + private protected bool ScanLookup(TsavoriteKV store, + ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, TScanIterator iter, bool validateCursor, long maxAddress, + bool resetCursor = true, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + where TScanIterator : ITsavoriteScanIterator, IPushScanIterator { - using var session = store.NewSession>(new LogScanCursorFunctions()); + using var session = store.NewSession>(new NoOpSessionFunctions()); var bContext = session.BasicContext; if (cursor < BeginAddress) // This includes 0, which means to start the Scan cursor = BeginAddress; - else if (validateCursor) - iter.SnapCursorToLogicalAddress(ref cursor); + else if (validateCursor && !iter.SnapCursorToLogicalAddress(ref cursor)) + goto IterationComplete; if (!scanFunctions.OnStart(cursor, iter.EndAddress)) return false; @@ -211,19 +235,17 @@ private protected bool ScanLookup 256) { - bContext.CompletePending(wait: true); + _ = bContext.CompletePending(wait: true); numPending = 0; } } @@ -257,49 +279,46 @@ private protected bool ScanLookup(TSessionFunctionsWrapper sessionFunctions, ScanCursorState scanCursorState, RecordInfo recordInfo, - ref TKey key, ref TValue value, long currentAddress, long minAddress, long maxAddress) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ConditionalScanPush(TSessionFunctionsWrapper sessionFunctions, + ScanCursorState scanCursorState, in TSourceLogRecord srcLogRecord, long originalAddress, long currentAddress, long minAddress, long maxAddress) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { Debug.Assert(epoch.ThisInstanceProtected(), "This is called only from ScanLookup so the epoch should be protected"); - TsavoriteKV.PendingContext pendingContext = new(_storeFunctions.GetKeyHashCode64(ref key)); + var pendingContext = new TsavoriteKV.PendingContext(storeFunctions.GetKeyHashCode64(srcLogRecord)); OperationStatus internalStatus; - OperationStackContext stackCtx = new(pendingContext.keyHash); + OperationStackContext stackCtx = new(pendingContext.keyHash); bool needIO; do { // If a more recent version of the record exists, do not push this one. Start by searching in-memory. - if (sessionFunctions.Store.TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx, currentAddress, minAddress, maxAddress, out internalStatus, out needIO)) + if (sessionFunctions.Store.TryFindRecordInMainLogForConditionalOperation(sessionFunctions, srcLogRecord, ref stackCtx, + currentAddress, minAddress, maxAddress, out internalStatus, out needIO)) return Status.CreateFound(); } while (sessionFunctions.Store.HandleImmediateNonPendingRetryStatus(internalStatus, sessionFunctions)); - TInput input = default; - TOutput output = default; if (needIO) { // A more recent version of the key was not (yet) found and we need another IO to continue searching. - internalStatus = PrepareIOForConditionalScan(sessionFunctions, ref pendingContext, ref key, ref input, ref value, ref output, default, - ref stackCtx, minAddress, maxAddress, scanCursorState); + internalStatus = PrepareIOForConditionalScan(sessionFunctions, ref pendingContext, in srcLogRecord, ref stackCtx, originalAddress, minAddress, maxAddress, scanCursorState); } else { - // A more recent version of the key was not found. recSrc.LogicalAddress is the correct address, because minAddress was examined - // and this is the previous record in the tag chain. Push this record to the user. + // A more recent version of the key was not found, so push the original record (with its originalAddress). + RecordMetadata recordMetadata = new(originalAddress); + epoch.Suspend(); try { - RecordMetadata recordMetadata = new(recordInfo, stackCtx.recSrc.LogicalAddress); - var stop = (stackCtx.recSrc.LogicalAddress >= HeadAddress) - ? !scanCursorState.functions.ConcurrentReader(ref key, ref value, recordMetadata, scanCursorState.acceptedCount, out var cursorRecordResult) - : !scanCursorState.functions.SingleReader(ref key, ref value, recordMetadata, scanCursorState.acceptedCount, out cursorRecordResult); + var stop = !scanCursorState.functions.Reader(in srcLogRecord, recordMetadata, scanCursorState.acceptedCount, out var cursorRecordResult); if (stop) scanCursorState.stop = true; else { if ((cursorRecordResult & CursorRecordResult.Accept) != 0) - Interlocked.Increment(ref scanCursorState.acceptedCount); + _ = Interlocked.Increment(ref scanCursorState.acceptedCount); if ((cursorRecordResult & CursorRecordResult.EndBatch) != 0) scanCursorState.endBatch = true; if ((cursorRecordResult & CursorRecordResult.RetryLastRecord) != 0) @@ -316,96 +335,27 @@ internal Status ConditionalScanPush(TSessionFunctionsWrapper sessionFunctions, - ref TsavoriteKV.PendingContext pendingContext, - ref TKey key, ref TInput input, ref TValue value, ref TOutput output, TContext userContext, - ref OperationStackContext stackCtx, long minAddress, long maxAddress, ScanCursorState scanCursorState) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal static OperationStatus PrepareIOForConditionalScan( + TSessionFunctionsWrapper sessionFunctions, + ref TsavoriteKV.PendingContext pendingContext, in TSourceLogRecord srcLogRecord, + ref OperationStackContext stackCtx, long originalAddress, long minAddress, long maxAddress, ScanCursorState scanCursorState) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - // WriteReason is not surfaced for this operation, so pick anything. - var status = sessionFunctions.Store.PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, ref key, ref input, ref value, ref output, - userContext, ref stackCtx, minAddress, maxAddress, WriteReason.Compaction, OperationType.CONDITIONAL_SCAN_PUSH); + var store = sessionFunctions.Store; + + var status = store.PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, in srcLogRecord, ref stackCtx, minAddress, maxAddress, OperationType.CONDITIONAL_SCAN_PUSH); pendingContext.scanCursorState = scanCursorState; + pendingContext.originalAddress = originalAddress; return status; } - internal struct LogScanCursorFunctions : ISessionFunctions - { - public bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo) => true; - public bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) => true; - public void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { } - - public bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - public void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo) { } - public bool ConcurrentDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - public void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - public bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) => true; - public void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason) { } - public bool ConcurrentWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) => true; - public void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - public bool InPlaceUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - - public bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo) => true; - public bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - public bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo) => true; - - public bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; - public bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - public void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo) { } - - public void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - public void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { } - - public int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => 0; - public int GetRMWInitialValueLength(ref TInput input) => 0; - public int GetUpsertValueLength(ref TValue value, ref TInput input) => 0; - - public void ConvertOutputToHeap(ref TInput input, ref TOutput output) { } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal long SnapToFixedLengthLogicalAddressBoundary(ref long logicalAddress, int recordSize) - { - // Get the initial offset on the page - int offset = (int)(logicalAddress & PageSizeMask); - long pageStart = logicalAddress - offset; - - int recordStartOffset; - if (logicalAddress < PageSize) - { - // We are on the first page so must account for BeginAddress. - if (offset < BeginAddress) - return logicalAddress = BeginAddress; - recordStartOffset = (int)(((offset - BeginAddress) / recordSize) * recordSize + BeginAddress); - } - else - { - // Not the first page, so just find the highest recordStartOffset <= offset. - recordStartOffset = (offset / recordSize) * recordSize; - } - - // If there is not enough room for a full record, advance logicalAddress to the next page start. - if (PageSize - recordStartOffset >= recordSize) - logicalAddress = pageStart + recordStartOffset; - else - logicalAddress = pageStart + PageSize; - return logicalAddress; - } - /// /// Scan page guaranteed to be in memory /// /// Begin address /// End address /// Observer of scan - internal abstract void MemoryPageScan(long beginAddress, long endAddress, IObserver> observer); + internal abstract void MemoryPageScan(long beginAddress, long endAddress, IObserver observer); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorSettings.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorSettings.cs index 935859aa374..81f2d8e4b94 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorSettings.cs @@ -7,22 +7,28 @@ namespace Tsavorite.core { /// - /// This class is created by to pass parameters to the allocator factory function. + /// This class is created by to pass parameters to the allocator factory function. /// public struct AllocatorSettings { - /// The Log settings, usually from + /// The Log settings, usually from internal LogSettings LogSettings; - /// The epoch created for the + /// The epoch created for the internal LightEpoch epoch; - /// The logger to use, either from or created by + /// The logger to use, either from or created by internal ILogger logger; /// The action to call on page eviction; used only for readcache internal Action evictCallback; + /// + /// Whether this allocator is the read cache (as opposed to the main hybrid log). + /// Used to tag per-record eviction callbacks so applications can distinguish the source. + /// + internal bool IsReadCache; + /// The action to execute on flush completion; used only for internal Action flushCallback; diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AsyncIOContext.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AsyncIOContext.cs index 1794ebe84a1..9b685662821 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AsyncIOContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AsyncIOContext.cs @@ -4,14 +4,15 @@ using System; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; namespace Tsavorite.core { + using static LogAddress; + /// /// Async IO context for PMM /// - public unsafe struct AsyncIOContext + public struct AsyncIOContext { /// /// Id @@ -19,22 +20,15 @@ public unsafe struct AsyncIOContext public long id; /// - /// Key + /// Key; this is a shallow copy of the key in pendingContext, pointing to its requestKey. /// - public IHeapContainer request_key; + public ConditionallyHoistedKey requestKey; - /// - /// Retrieved key - /// - public TKey key; + /// The retrieved record, including deserialized ValueObject if RecordInfo.ValueIsObject, and key or value Overflows + public DiskLogRecord diskLogRecord; /// - /// Retrieved value - /// - public TValue value; - - /// - /// Logical address + /// Logical address that was requested /// public long logicalAddress; @@ -56,72 +50,88 @@ public unsafe struct AsyncIOContext /// /// Callback queue /// - public AsyncQueue> callbackQueue; - - /// - /// Async Operation ValueTask backer - /// - public TaskCompletionSource> asyncOperation; + public AsyncQueue callbackQueue; /// /// Synchronous completion event /// - internal AsyncIOContextCompletionEvent completionEvent; + internal AsyncIOContextCompletionEvent completionEvent; /// /// Indicates whether this is a default instance with no pending operation /// - public bool IsDefault() => callbackQueue is null && asyncOperation is null && completionEvent is null; + public readonly bool IsDefault() => callbackQueue is null && completionEvent is null; /// /// Dispose /// - public void Dispose() + public void DisposeRecord() { - // Do not dispose request_key as it is a shallow copy of the key in pendingContext + // Do not dispose requestKey as it is a shallow copy of the key in pendingContext + diskLogRecord.Dispose(); + diskLogRecord = default; record?.Return(); record = null; } + + /// + public override readonly string ToString() + => $"id {id}, key {requestKey}, LogAddr {AddressString(logicalAddress)}, MinAddr {minAddress}, LogRec [{diskLogRecord}]"; } // Wrapper class so we can communicate back the context.record even if it has to retry due to incomplete records. - internal sealed class AsyncIOContextCompletionEvent : IDisposable + internal sealed class AsyncIOContextCompletionEvent : IDisposable { internal SemaphoreSlim semaphore; internal Exception exception; - internal AsyncIOContext request; + internal AsyncIOContext request; internal AsyncIOContextCompletionEvent() { semaphore = new SemaphoreSlim(0); request.id = -1; - request.minAddress = Constants.kInvalidAddress; + request.minAddress = kInvalidAddress; request.completionEvent = this; } - internal void Prepare(IHeapContainer request_key, long logicalAddress) + /// + /// Prepares to issue an async IO. + /// + /// + /// SAFETY: The MUST be non-movable, such as on the stack, or pinned for the life of the IO operation. + /// + internal void Prepare(TKey requestKey, long logicalAddress, SectorAlignedBufferPool bufferPool) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - request.Dispose(); - request.request_key = request_key; + request.DisposeRecord(); + request.requestKey.Dispose(); + + request.requestKey = ConditionallyHoistedKey.Create(requestKey, bufferPool); request.logicalAddress = logicalAddress; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void Set(ref AsyncIOContext ctx) + internal void Set(ref AsyncIOContext ctx) { - request.Dispose(); + request.DisposeRecord(); + request = ctx; exception = null; - semaphore.Release(1); + _ = semaphore.Release(1); } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetException(Exception ex) { - request.Dispose(); + request.DisposeRecord(); + request.requestKey.Dispose(); + request = default; exception = ex; - semaphore.Release(1); + _ = semaphore.Release(1); } internal void Wait(CancellationToken token = default) => semaphore.Wait(token); @@ -129,7 +139,8 @@ internal void SetException(Exception ex) /// public void Dispose() { - request.Dispose(); + request.DisposeRecord(); + request.requestKey.Dispose(); semaphore?.Dispose(); } } diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocator.cs deleted file mode 100644 index 54f48093764..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocator.cs +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.CompilerServices; - -namespace Tsavorite.core -{ - /// - /// Struct wrapper (for inlining) around the fixed-length Blittable allocator. - /// - public struct BlittableAllocator : IAllocator - where TStoreFunctions : IStoreFunctions - { - /// The wrapped class containing all data and most actual functionality. This must be the ONLY field in this structure so its size is sizeof(IntPtr). - private readonly BlittableAllocatorImpl _this; - - public BlittableAllocator(AllocatorSettings settings, TStoreFunctions storeFunctions) - { - // Called by TsavoriteKV via allocatorCreator; must pass a wrapperCreator to AllocatorBase - _this = new(settings, storeFunctions, @this => new BlittableAllocator(@this)); - } - - public BlittableAllocator(object @this) - { - // Called by AllocatorBase via primary ctor wrapperCreator - _this = (BlittableAllocatorImpl)@this; - } - - /// - public readonly AllocatorBase GetBase() - where TAllocator : IAllocator - => (AllocatorBase)(object)_this; - - /// - public readonly bool IsFixedLength => true; - - /// - public readonly bool HasObjectLog => false; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetStartLogicalAddress(long page) => _this.GetStartLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetFirstValidLogicalAddress(long page) => _this.GetFirstValidLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetPhysicalAddress(long logicalAddress) => _this.GetPhysicalAddress(logicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref RecordInfo GetInfo(long physicalAddress) - => ref BlittableAllocatorImpl.GetInfo(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe ref RecordInfo GetInfoFromBytePointer(byte* ptr) - => ref BlittableAllocatorImpl.GetInfoFromBytePointer(ptr); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TKey GetKey(long physicalAddress) - => ref BlittableAllocatorImpl.GetKey(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TValue GetValue(long physicalAddress) - => ref BlittableAllocatorImpl.GetValue(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TValue GetAndInitializeValue(long physicalAddress, long endPhysicalAddress) => ref GetValue(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) - => BlittableAllocatorImpl.GetRecordSize(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref TKey key, ref TInput input, ref TValue value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - where TVariableLengthInput : IVariableLengthInput - => BlittableAllocatorImpl.GetRMWCopyDestinationRecordSize(ref key, ref input, ref value, ref recordInfo, varlenInput); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref TKey key) - => BlittableAllocatorImpl.GetTombstoneRecordSize(ref key); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetRequiredRecordSize(long physicalAddress, int availableBytes) => GetAverageRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetAverageRecordSize() - => BlittableAllocatorImpl.GetAverageRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetFixedRecordSize() - => BlittableAllocatorImpl.GetFixedRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref TKey key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => BlittableAllocatorImpl.GetRMWInitialRecordSize(ref key, ref input, sessionFunctions); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref TKey key, ref TValue value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => BlittableAllocatorImpl.GetUpsertRecordSize(ref key, ref value, ref input, sessionFunctions); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref TKey key, ref TValue value) - => BlittableAllocatorImpl.GetRecordSize(ref key, ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetValueLength(ref TValue value) - => BlittableAllocatorImpl.GetValueLength(ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) - => BlittableAllocatorImpl.RetrievedFullRecord(record, ref ctx); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void AllocatePage(int pageIndex) => _this.AllocatePage(pageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly bool IsAllocated(int pageIndex) => _this.IsAllocated(pageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe void PopulatePage(byte* src, int required_bytes, long destinationPageIndex) - => BlittableAllocatorImpl.PopulatePage(src, required_bytes, destinationPageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPage(long logicalAddress, long version) => _this.MarkPage(logicalAddress, version); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPageAtomic(long logicalAddress, long version) => _this.MarkPageAtomic(logicalAddress, version); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void ClearPage(long page, int offset = 0) => _this.ClearPage(page, offset); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); - - /// - public readonly ref TKey GetContextRecordKey(ref AsyncIOContext ctx) => ref ctx.key; - - /// - public readonly ref TValue GetContextRecordValue(ref AsyncIOContext ctx) => ref ctx.value; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetKeyContainer(ref TKey key) => new StandardHeapContainer(ref key); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetValueContainer(ref TValue value) => new StandardHeapContainer(ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long[] GetSegmentOffsets() - => BlittableAllocatorImpl.GetSegmentOffsets(); - - /// - public readonly int OverflowPageCount => _this.OverflowPageCount; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void SerializeKey(ref TKey key, long physicalAddress) - => BlittableAllocatorImpl.SerializeKey(ref key, physicalAddress); - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocatorImpl.cs deleted file mode 100644 index 83561273847..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableAllocatorImpl.cs +++ /dev/null @@ -1,356 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Threading; - -namespace Tsavorite.core -{ - internal sealed unsafe class BlittableAllocatorImpl : AllocatorBase> - where TStoreFunctions : IStoreFunctions - { - // Circular buffer definition - private readonly byte[][] values; - private readonly long[] pointers; - private readonly long* nativePointers; - - private static int KeySize => Unsafe.SizeOf(); - private static int ValueSize => Unsafe.SizeOf(); - internal static int RecordSize => Unsafe.SizeOf>(); - - private readonly OverflowPool overflowPagePool; - - public BlittableAllocatorImpl(AllocatorSettings settings, TStoreFunctions storeFunctions, Func> wrapperCreator) - : base(settings.LogSettings, storeFunctions, wrapperCreator, settings.evictCallback, settings.epoch, settings.flushCallback, settings.logger) - { - if (!Utility.IsBlittable() || !Utility.IsBlittable()) - throw new TsavoriteException($"BlittableAllocator requires blittlable Key ({typeof(TKey)}) and Value ({typeof(TValue)})"); - - overflowPagePool = new OverflowPool(4, p => { }); - - if (BufferSize > 0) - { - values = new byte[BufferSize][]; - pointers = GC.AllocateArray(BufferSize, true); - nativePointers = (long*)Unsafe.AsPointer(ref pointers[0]); - } - } - - public override void Reset() - { - base.Reset(); - for (int index = 0; index < BufferSize; index++) - { - if (IsAllocated(index)) - FreePage(index); - } - Initialize(); - } - - void ReturnPage(int index) - { - Debug.Assert(index < BufferSize); - if (values[index] != null) - { - _ = overflowPagePool.TryAdd(new PageUnit - { - pointer = pointers[index], - value = values[index] - }); - values[index] = null; - pointers[index] = 0; - _ = Interlocked.Decrement(ref AllocatedPageCount); - } - } - - public override void Initialize() => Initialize(Constants.kFirstValidAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref RecordInfo GetInfo(long physicalAddress) => ref Unsafe.AsRef((void*)physicalAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref RecordInfo GetInfoFromBytePointer(byte* ptr) => ref Unsafe.AsRef(ptr); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref TKey GetKey(long physicalAddress) => ref Unsafe.AsRef((byte*)physicalAddress + RecordInfo.GetLength()); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref TValue GetValue(long physicalAddress) => ref Unsafe.AsRef((byte*)physicalAddress + RecordInfo.GetLength() + KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) => (RecordSize, RecordSize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref TKey key, ref TInput input, ref TValue value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - => (RecordSize, RecordSize, KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref TKey key) - => (RecordSize, RecordSize, KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref TKey key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - => (RecordSize, RecordSize, KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetRequiredRecordSize(long physicalAddress, int availableBytes) => GetAverageRecordSize(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetAverageRecordSize() => RecordSize; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static int GetFixedRecordSize() => RecordSize; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref TKey key, ref TValue value) => (RecordSize, RecordSize, KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref TKey key, ref TValue value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - => (RecordSize, RecordSize, KeySize); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetValueLength(ref TValue value) => ValueSize; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SerializeKey(ref TKey src, long physicalAddress) => GetKey(physicalAddress) = src; - - /// - /// Dispose memory allocator - /// - public override void Dispose() - { - base.Dispose(); - overflowPagePool.Dispose(); - } - - /// - /// Allocate memory page, pinned in memory, and in sector aligned form, if possible - /// - /// - internal void AllocatePage(int index) - { - IncrementAllocatedPageCount(); - - if (overflowPagePool.TryGet(out var item)) - { - pointers[index] = item.pointer; - values[index] = item.value; - return; - } - - var adjustedSize = PageSize + 2 * sectorSize; - - byte[] tmp = GC.AllocateArray(adjustedSize, true); - long p = (long)Unsafe.AsPointer(ref tmp[0]); - pointers[index] = (p + (sectorSize - 1)) & ~((long)sectorSize - 1); - values[index] = tmp; - } - - internal int OverflowPageCount => overflowPagePool.Count; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetPhysicalAddress(long logicalAddress) - { - // Offset within page - var offset = (int)(logicalAddress & ((1L << LogPageSizeBits) - 1)); - - // Index of page within the circular buffer - var pageIndex = (int)((logicalAddress >> LogPageSizeBits) & (BufferSize - 1)); - return *(nativePointers + pageIndex) + offset; - } - - internal bool IsAllocated(int pageIndex) => values[pageIndex] != null; - - protected override void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult) - { - WriteAsync((IntPtr)pointers[flushPage % BufferSize], - (ulong)(AlignedPageSizeBytes * flushPage), - (uint)AlignedPageSizeBytes, - callback, - asyncResult, device); - } - - protected override void WriteAsyncToDevice - (long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, - PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long[] localSegmentOffsets, long fuzzyStartLogicalAddress) - { - VerifyCompatibleSectorSize(device); - var alignedPageSize = (pageSize + (sectorSize - 1)) & ~(sectorSize - 1); - - WriteAsync((IntPtr)pointers[flushPage % BufferSize], - (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), - (uint)alignedPageSize, callback, asyncResult, - device); - } - - /// - /// Get start logical address - /// - public long GetStartLogicalAddress(long page) => page << LogPageSizeBits; - - /// - /// Get first valid logical address - /// - public long GetFirstValidLogicalAddress(long page) - { - if (page == 0) - return (page << LogPageSizeBits) + Constants.kFirstValidAddress; - return page << LogPageSizeBits; - } - - internal void ClearPage(long page, int offset) - { - if (offset == 0) - Array.Clear(values[page % BufferSize], offset, values[page % BufferSize].Length - offset); - else - { - // Adjust array offset for cache alignment - offset += (int)(pointers[page % BufferSize] - (long)Unsafe.AsPointer(ref values[page % BufferSize][0])); - Array.Clear(values[page % BufferSize], offset, values[page % BufferSize].Length - offset); - } - } - - internal void FreePage(long page) - { - ClearPage(page, 0); - if (EmptyPageCount > 0) - ReturnPage((int)(page % BufferSize)); - } - - /// - /// Delete in-memory portion of the log - /// - internal override void DeleteFromMemory() - { - for (int i = 0; i < values.Length; i++) - values[i] = null; - } - - protected override void ReadAsync(ulong alignedSourceAddress, int destinationPageIndex, uint aligned_read_length, - DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device, IDevice objlogDevice) - => device.ReadAsync(alignedSourceAddress, (IntPtr)pointers[destinationPageIndex], aligned_read_length, callback, asyncResult); - - /// - /// Invoked by users to obtain a record from disk. It uses sector aligned memory to read - /// the record efficiently into memory. - /// - /// - /// - /// - /// - /// - protected override void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default) - => throw new InvalidOperationException("AsyncReadRecordObjectsToMemory invalid for BlittableAllocator"); - - internal static bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) - { - ctx.key = GetKey((long)record); - ctx.value = GetValue((long)record); - return true; - } - - internal static long[] GetSegmentOffsets() => null; - - internal static void PopulatePage(byte* src, int required_bytes, long destinationPage) - => throw new TsavoriteException("BlittableAllocator memory pages are sector aligned - use direct copy"); - - /// - /// Iterator interface for pull-scanning Tsavorite log - /// - public override ITsavoriteScanIterator Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords) - => new BlittableScanIterator(store, this, beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch, logger: logger); - - /// - /// Implementation for push-scanning Tsavorite log, called from LogAccessor - /// - internal override bool Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ref TScanFunctions scanFunctions, ScanBufferingMode scanBufferingMode) - { - using BlittableScanIterator iter = new(store, this, beginAddress, endAddress, scanBufferingMode, false, epoch, logger: logger); - return PushScanImpl(beginAddress, endAddress, ref scanFunctions, iter); - } - - /// - /// Implementation for push-scanning Tsavorite log with a cursor, called from LogAccessor - /// - internal override bool ScanCursor(TsavoriteKV> store, - ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, bool resetCursor = false, bool includeTombstones = false) - { - using BlittableScanIterator iter = new(store, this, cursor, endAddress, ScanBufferingMode.SinglePageBuffering, includeClosedRecords: maxAddress < long.MaxValue, epoch, logger: logger); - return ScanLookup>(store, scanCursorState, ref cursor, count, scanFunctions, iter, validateCursor, maxAddress, includeTombstones: includeTombstones); - } - - /// - /// Implementation for push-iterating key versions, called from LogAccessor - /// - internal override bool IterateKeyVersions(TsavoriteKV> store, ref TKey key, long beginAddress, ref TScanFunctions scanFunctions) - { - using BlittableScanIterator iter = new(store, this, beginAddress, epoch, logger: logger); - return IterateKeyVersionsImpl(store, ref key, beginAddress, ref scanFunctions, iter); - } - - /// - internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver> observer) - { - using var iter = new BlittableScanIterator(store: null, this, beginAddress, endAddress, ScanBufferingMode.NoBuffering, false, epoch, true, logger: logger); - observer?.OnNext(iter); - } - - /// - /// Read pages from specified device - /// - internal void AsyncReadPagesFromDeviceToFrame( - long readPageStart, - int numPages, - long untilAddress, - DeviceIOCompletionCallback callback, - TContext context, - BlittableFrame frame, - out CountdownEvent completed, - long devicePageOffset = 0, - IDevice device = null, - IDevice objectLogDevice = null, - CancellationTokenSource cts = null) - { - var usedDevice = device ?? this.device; - - completed = new CountdownEvent(numPages); - for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++) - { - int pageIndex = (int)(readPage % frame.frameSize); - if (frame.frame[pageIndex] == null) - frame.Allocate(pageIndex); - else - frame.Clear(pageIndex); - - var asyncResult = new PageAsyncReadResult() - { - page = readPage, - context = context, - handle = completed, - frame = frame, - cts = cts - }; - - ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); - - uint readLength = (uint)AlignedPageSizeBytes; - long adjustedUntilAddress = (AlignedPageSizeBytes * (untilAddress >> LogPageSizeBits) + (untilAddress & PageSizeMask)); - - if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) - { - readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); - readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); - } - - if (device != null) - offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); - - usedDevice.ReadAsync(offsetInFile, (IntPtr)frame.pointers[pageIndex], readLength, callback, asyncResult); - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableFrame.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableFrame.cs index 38e72056ae6..60d78eacc55 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableFrame.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableFrame.cs @@ -6,6 +6,8 @@ namespace Tsavorite.core { + using static Utility; + /// /// A frame is an in-memory circular buffer of log pages /// @@ -29,9 +31,9 @@ public unsafe void Allocate(int index) { var adjustedSize = pageSize + 2 * sectorSize; - byte[] tmp = GC.AllocateArray(adjustedSize, true); - long p = (long)Unsafe.AsPointer(ref tmp[0]); - pointers[index] = (p + (sectorSize - 1)) & ~((long)sectorSize - 1); + var tmp = GC.AllocateArray(adjustedSize, pinned: true); + var p = (long)Unsafe.AsPointer(ref tmp[0]); + pointers[index] = RoundUp(p, sectorSize); frame[index] = tmp; } @@ -40,11 +42,19 @@ public void Clear(int pageIndex) Array.Clear(frame[pageIndex], 0, frame[pageIndex].Length); } - public long GetPhysicalAddress(long frameNumber, long offset) + public long GetPhysicalAddress(long frameNumber, long offset = 0) { return pointers[frameNumber % frameSize] + offset; } + public unsafe (byte[] array, long offset) GetArrayAndUnalignedOffset(long frameNumber, long alignedOffset) + { + var frameIndex = frameNumber % frameSize; + + long ptr = (long)Unsafe.AsPointer(ref frame[frameIndex]); + return (frame[frameIndex], alignedOffset + ptr - pointers[frameIndex]); + } + public void Dispose() { } diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableScanIterator.cs deleted file mode 100644 index 57a92bf3999..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/BlittableScanIterator.cs +++ /dev/null @@ -1,284 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Threading; -using Microsoft.Extensions.Logging; - -namespace Tsavorite.core -{ - /// - /// Scan iterator for hybrid log - /// - public sealed class BlittableScanIterator : ScanIteratorBase, ITsavoriteScanIterator, IPushScanIterator - where TStoreFunctions : IStoreFunctions - { - private readonly TsavoriteKV> store; - private readonly BlittableAllocatorImpl hlog; - private readonly BlittableFrame frame; - private readonly bool forceInMemory; - - private TKey currentKey; - private TValue currentValue; - private long framePhysicalAddress; - - /// - /// Constructor for use with head-to-tail scan - /// - /// - /// The fully derived log implementation - /// - /// - /// - /// - /// Epoch to use for protection; may be null if is true. - /// Provided address range is known by caller to be in memory, even if less than HeadAddress - /// - internal BlittableScanIterator(TsavoriteKV> store, BlittableAllocatorImpl hlog, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords, LightEpoch epoch, bool forceInMemory = false, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch, hlog.LogPageSizeBits, logger: logger) - { - this.store = store; - this.hlog = hlog; - this.forceInMemory = forceInMemory; - if (frameSize > 0) - frame = new BlittableFrame(frameSize, hlog.PageSize, hlog.GetDeviceSectorSize()); - } - - /// - /// Constructor for use with tail-to-head push iteration of the passed key's record versions - /// - internal BlittableScanIterator(TsavoriteKV> store, BlittableAllocatorImpl hlog, - long beginAddress, LightEpoch epoch, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, hlog.GetTailAddress(), ScanBufferingMode.SinglePageBuffering, false, epoch, hlog.LogPageSizeBits, logger: logger) - { - this.store = store; - this.hlog = hlog; - forceInMemory = false; - if (frameSize > 0) - frame = new BlittableFrame(frameSize, hlog.PageSize, hlog.GetDeviceSectorSize()); - } - - /// - /// Get a reference to the current key - /// - public ref TKey GetKey() => ref framePhysicalAddress != 0 ? ref hlog._wrapper.GetKey(framePhysicalAddress) : ref currentKey; - - /// - /// Get a reference to the current value - /// - public ref TValue GetValue() => ref framePhysicalAddress != 0 ? ref hlog._wrapper.GetValue(framePhysicalAddress) : ref currentValue; - - /// - public bool SnapCursorToLogicalAddress(ref long cursor) - { - Debug.Assert(currentAddress == -1, "SnapCursorToLogicalAddress must be called before GetNext()"); - beginAddress = nextAddress = hlog.SnapToFixedLengthLogicalAddressBoundary(ref cursor, BlittableAllocatorImpl.RecordSize); - return true; - } - - /// - /// Get next record in iterator - /// - /// True if record found, false if end of scan - public unsafe bool GetNext(out RecordInfo recordInfo) - { - recordInfo = default; - - while (true) - { - currentAddress = nextAddress; - var stopAddress = endAddress < hlog.GetTailAddress() ? endAddress : hlog.GetTailAddress(); - if (currentAddress >= stopAddress) - return false; - - epoch?.Resume(); - var headAddress = hlog.HeadAddress; - - if (currentAddress < hlog.BeginAddress && !forceInMemory) - currentAddress = hlog.BeginAddress; - - // If currentAddress < headAddress and we're not buffering and not guaranteeing the records are in memory, fail. - if (frameSize == 0 && currentAddress < headAddress && !forceInMemory) - { - epoch?.Suspend(); - throw new TsavoriteException("Iterator address is less than log HeadAddress in memory-scan mode"); - } - - var currentPage = currentAddress >> hlog.LogPageSizeBits; - var offset = currentAddress & hlog.PageSizeMask; - - if (currentAddress < headAddress && !forceInMemory) - BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, stopAddress); - - long physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); - var recordSize = hlog._wrapper.GetRecordSize(physicalAddress).Item2; - - // If record does not fit on page, skip to the next page. - if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize) - { - nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits; - epoch?.Suspend(); - continue; - } - - nextAddress = currentAddress + recordSize; - - recordInfo = hlog._wrapper.GetInfo(physicalAddress); - bool skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (skipOnScan || recordInfo.IsNull()) - { - epoch?.Suspend(); - continue; - } - - OperationStackContext> stackCtx = default; - try - { - // Lock to ensure no value tearing while copying to temp storage. We cannot use GetKey() because it has not yet been set. - if (currentAddress >= headAddress && store is not null) - store.LockForScan(ref stackCtx, ref hlog._wrapper.GetKey(physicalAddress)); - _ = CopyDataMembers(physicalAddress); - } - finally - { - if (stackCtx.recSrc.HasLock) - store.UnlockForScan(ref stackCtx); - } - - // Success - epoch?.Suspend(); - return true; - } - } - - /// - /// Get previous record and keep the epoch held while we call the user's scan functions - /// - /// True if record found, false if end of scan - bool IPushScanIterator.BeginGetPrevInMemory(ref TKey key, out RecordInfo recordInfo, out bool continueOnDisk) - { - recordInfo = default; - continueOnDisk = false; - - while (true) - { - // "nextAddress" is reused as "previous address" for this operation. - currentAddress = nextAddress; - if (currentAddress < hlog.HeadAddress) - { - continueOnDisk = currentAddress >= hlog.BeginAddress; - return false; - } - - epoch?.Resume(); - var headAddress = hlog.HeadAddress; - - var currentPage = currentAddress >> hlog.LogPageSizeBits; - var offset = currentAddress & hlog.PageSizeMask; - - long physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); - - recordInfo = hlog._wrapper.GetInfo(physicalAddress); - nextAddress = recordInfo.PreviousAddress; - - // Do not SkipOnScan here; we Seal previous versions. - if (recordInfo.IsNull() || !hlog._storeFunctions.KeysEqual(ref hlog._wrapper.GetKey(physicalAddress), ref key)) - { - epoch?.Suspend(); - continue; - } - - // Success; defer epoch?.Suspend(); to EndGetPrevInMemory - return CopyDataMembers(physicalAddress); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IPushScanIterator.EndGetPrevInMemory() - { - epoch?.Suspend(); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - long GetPhysicalAddress(long currentAddress, long headAddress, long currentPage, long offset) - { - if (currentAddress >= headAddress) - { - // physicalAddress is in memory; set framePhysicalAddress to 0 so we'll set currentKey and currentValue from physicalAddress below - framePhysicalAddress = 0; - return hlog.GetPhysicalAddress(currentAddress); - } - - // physicalAddress is not in memory, so we'll GetKey and GetValue will use framePhysicalAddress - return framePhysicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool CopyDataMembers(long physicalAddress) - { - if (framePhysicalAddress == 0) - { - // Copy the values from the log to data members so we have no ref into the log after the epoch.Suspend(). - currentKey = hlog._wrapper.GetKey(physicalAddress); - currentValue = hlog._wrapper.GetValue(physicalAddress); - } - return true; - } - - /// - /// Get next record in iterator - /// - public bool GetNext(out RecordInfo recordInfo, out TKey key, out TValue value) - { - if (GetNext(out recordInfo)) - { - key = GetKey(); - value = GetValue(); - return true; - } - - key = default; - value = default; - return false; - } - - /// - /// Dispose iterator - /// - public override void Dispose() - { - base.Dispose(); - frame?.Dispose(); - } - - internal override void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, - long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) - => hlog.AsyncReadPagesFromDeviceToFrame(readPageStart, numPages, untilAddress, AsyncReadPagesCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice); - - private unsafe void AsyncReadPagesCallback(uint errorCode, uint numBytes, object context) - { - var result = (PageAsyncReadResult)context; - - if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncReadPagesCallback)} error: {{errorCode}}", errorCode); - result.cts?.Cancel(); - } - - if (result.freeBuffer1 != null) - { - BlittableAllocatorImpl.PopulatePage(result.freeBuffer1.GetValidPointer(), result.freeBuffer1.required_bytes, result.page); - result.freeBuffer1.Return(); - result.freeBuffer1 = null; - } - - if (errorCode == 0) - result.handle?.Signal(); - - Interlocked.MemoryBarrier(); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ConditionallyHositedKey.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ConditionallyHositedKey.cs new file mode 100644 index 00000000000..29b51c481a6 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ConditionallyHositedKey.cs @@ -0,0 +1,216 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Tsavorite.core +{ + /// + /// An that can be used in heap allocated contexts. + /// + public unsafe struct ConditionallyHoistedKey : IKey, IDisposable + { + private static ConditionallyHoistedKey Empty { get; } = new((byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference([])), 0); + + private readonly byte* keyPtr; + private readonly int keyLen; + private readonly byte* namespacePtr; + private readonly int namespaceLen; + private SectorAlignedMemory keyAndNamespaceMem; + + /// + public readonly bool IsEmpty => keyLen == 0; + + /// + public readonly bool IsPinned + => keyPtr != null; + + /// + public readonly ReadOnlySpan KeyBytes + { + get + { + if (keyPtr != null) + { + return new ReadOnlySpan(keyPtr, keyLen); + } + else if (keyAndNamespaceMem != null) + { + return keyAndNamespaceMem.TotalValidSpan[..keyLen]; + } + else + { + return default; + } + } + } + + /// + public readonly bool HasNamespace => namespaceLen != 0; + + /// + public readonly ReadOnlySpan NamespaceBytes + { + get + { + Debug.Assert(HasNamespace, "Should never be called if !HasNamespace"); + + if (namespacePtr != null) + { + return new ReadOnlySpan(namespacePtr, namespaceLen); + } + else if (keyAndNamespaceMem != null) + { + return keyAndNamespaceMem.TotalValidSpan.Slice(keyLen, namespaceLen); + } + else + { + return default; + } + } + } + + private ConditionallyHoistedKey(byte* keyPtr, int keyLen) + { + keyAndNamespaceMem = null; + namespaceLen = 0; + + this.keyPtr = keyPtr; + this.keyLen = keyLen; + } + + private ConditionallyHoistedKey(byte* keyPtr, int keyLen, byte* namespacePtr, int namespaceLen) + { + Debug.Assert(namespaceLen > 0, "Shouldn't use this constructor if namespace isn't set"); + + keyAndNamespaceMem = null; + + this.keyPtr = keyPtr; + this.keyLen = keyLen; + + this.namespacePtr = namespacePtr; + this.namespaceLen = namespaceLen; + } + + private ConditionallyHoistedKey(SectorAlignedMemory keyArr, int keyLen) + { + this.keyPtr = null; + this.keyAndNamespaceMem = keyArr; + this.keyLen = keyLen; + this.namespaceLen = 0; + } + + private ConditionallyHoistedKey(SectorAlignedMemory keyArr, int keyLen, int namespaceLen) + { + Debug.Assert(namespaceLen > 0, "Shouldn't use this constructor if namespace isn't set"); + + this.keyPtr = null; + this.keyAndNamespaceMem = keyArr; + this.keyLen = keyLen; + this.namespaceLen = namespaceLen; + } + + /// + public bool KeysEqual(TOther other) where TOther : + IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + if (other.HasNamespace) + { + if (!HasNamespace) + { + return false; + } + + // Namespace must be considered alongside key + return KeyBytes.SequenceEqual(other.KeyBytes) && NamespaceBytes.SequenceEqual(other.NamespaceBytes); + } + else + { + if (HasNamespace) + { + return false; + } + + // Namespace is known not set, ignore + return KeyBytes.SequenceEqual(other.KeyBytes); + } + } + + /// + public void Dispose() + { + keyAndNamespaceMem?.Return(); + keyAndNamespaceMem = null; + } + + /// + /// Create a new , copying bytes if needed. + /// + internal static ConditionallyHoistedKey Create(TKey key, SectorAlignedBufferPool bufferPool) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + + if (key.IsEmpty) + { + Debug.Assert(!key.HasNamespace, "Empty key should never have a namespace"); + return Empty; + } + + var keyBytes = key.KeyBytes; + + if (key.IsPinned) + { + var keyPtr = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(keyBytes)); + + if (key.HasNamespace) + { + var namespaceBytes = key.NamespaceBytes; + var namespacePtr = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(namespaceBytes)); + + return new( + keyPtr, + keyBytes.Length, + namespacePtr, + namespaceBytes.Length + ); + } + + return new(keyPtr, keyBytes.Length); + } + else + { + // TODO: This matches existing use, but is this correct? Seems like we'd get a big record here + + var recordMinLen = keyBytes.Length; + if (key.HasNamespace) + { + var namespaceBytes = key.NamespaceBytes; + + recordMinLen += namespaceBytes.Length; + + var mem = bufferPool.Get(keyBytes.Length); + keyBytes.CopyTo(mem.TotalValidSpan); + namespaceBytes.CopyTo(mem.TotalValidSpan[keyBytes.Length..]); + + return new(mem, keyBytes.Length, namespaceBytes.Length); + } + else + { + var mem = bufferPool.Get(keyBytes.Length); + keyBytes.CopyTo(mem.TotalValidSpan); + + return new(mem, keyBytes.Length); + } + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/CountdownWrapper.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/CountdownWrapper.cs new file mode 100644 index 00000000000..413b7b3d9e7 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/CountdownWrapper.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#pragma warning disable 0162 + +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + internal sealed class CountdownWrapper + { + // Separate event for sync code and tcs for async code: Do not block on async code. + private readonly CountdownEvent syncEvent; + private readonly TaskCompletionSource asyncTcs; + int remaining; + + internal CountdownWrapper(int count, bool isAsync) + { + if (isAsync) + { + asyncTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + remaining = count; + return; + } + syncEvent = new CountdownEvent(count); + } + + internal bool IsCompleted => syncEvent is null ? remaining == 0 : syncEvent.IsSet; + + internal void Wait() => syncEvent.Wait(); + internal async ValueTask WaitAsync(CancellationToken cancellationToken) + { + using var reg = cancellationToken.Register(() => asyncTcs.TrySetCanceled()); + await asyncTcs.Task.ConfigureAwait(false); + } + + internal void Decrement() + { + if (asyncTcs is not null) + { + Debug.Assert(remaining > 0); + if (Interlocked.Decrement(ref remaining) == 0) + asyncTcs.TrySetResult(0); + return; + } + syncEvent.Signal(); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/DiskLogRecord.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/DiskLogRecord.cs new file mode 100644 index 00000000000..fa183053ac4 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/DiskLogRecord.cs @@ -0,0 +1,556 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + /// A wrapper around LogRecord for retrieval from disk or carrying through pending operations + public unsafe struct DiskLogRecord : ISourceLogRecord, IDisposable + { + /// The > around the record data. + internal LogRecord logRecord; + + /// The buffer containing the record data, from either disk IO or a copy from a LogRecord that is carried through pending operations + /// such as Compact or ConditionalCopyToTail. The contains its + /// as its . + /// We always own the record buffer; it is either transferred to us, or allocated as a copy of the record memory. However, it may be + /// null if we transferred it out. + SectorAlignedMemory recordBuffer; + + public override readonly string ToString() + { + return $"logRec [{logRecord}], recordBuffer [{recordBuffer?.ToString() ?? ""}]"; + } + + /// + /// Constructor taking the record buffer and out-of-line objects. Private; use either CopyFrom or TransferFrom. + /// + /// The record buffer, either from IO or a copy for pending operations such as Compact or ConditionalCopyToTail. + /// The to hold the objects for the for the lifetime of this . + /// The key overflow byte[] wrapper, if any + /// The value overflow byte[] wrapper, if any + /// The value object, if any + /// We always own the record buffer; it is either transferred to us by TransferFrom, or allocated as a copy of the record memory by CopyFrom + private DiskLogRecord(SectorAlignedMemory recordBuffer, ObjectIdMap transientObjectIdMap, OverflowByteArray keyOverflow, + OverflowByteArray valueOverflow, IHeapObject valueObject) + { + this.recordBuffer = recordBuffer; + logRecord = new((long)recordBuffer.GetValidPointer(), transientObjectIdMap); + + // Assign any out-of-line fields. This will put them into transientObjectIdMap. + if (!keyOverflow.IsEmpty) + logRecord.KeyOverflow = keyOverflow; + if (!valueOverflow.IsEmpty) + logRecord.ValueOverflow = valueOverflow; + else if (valueObject is not null) + logRecord.ValueObject = valueObject; + } + + /// + /// Constructs the from an already-constructed LogRecord (e.g. from which + /// has transient ObjectIds if it has objects). + /// + internal DiskLogRecord(in LogRecord memoryLogRecord) + { + logRecord = memoryLogRecord; + } + + /// + /// Transfers a transient inline record buffer and creates our contained from it. Private; use either CopyFrom or TransferFrom. + /// + /// The record buffer, either from IO or a copy for pending operations such as Compact or ConditionalCopyToTail. + /// The to hold the objects for the for the lifetime of this . + /// We always own the record buffer; it is either transferred to us, or allocated as a copy of the record memory + private DiskLogRecord(SectorAlignedMemory recordBuffer, ObjectIdMap transientObjectIdMap) + { + this.recordBuffer = recordBuffer; + logRecord = new((long)recordBuffer.GetValidPointer(), transientObjectIdMap); + } + + /// + /// Creates a from an already-constructed LogRecord (e.g. from which + /// has transient ObjectIds if it has objects). + /// + internal static DiskLogRecord CreateFromTransientLogRecord(in LogRecord memoryLogRecord) => new(memoryLogRecord); + + /// + /// Allocates and copies the LogRecord's record memory into it; any out-of-line objects are shallow-copied. + /// + /// The to copy + /// The buffer pool to allocate from + /// The to hold the objects for the for the lifetime of this . + internal static DiskLogRecord CopyFrom(in LogRecord logRecord, SectorAlignedBufferPool bufferPool, ObjectIdMap transientObjectIdMap) + { + // Allocate from ActualSize roundup here because the value may have been shrunk. + var allocatedSize = RoundUp(logRecord.ActualSize, Constants.kRecordAlignment); + var recordBuffer = bufferPool.Get(allocatedSize); + + // Copy the inline portion of the logRecord. + logRecord.RecordSpan.CopyTo(recordBuffer.RequiredValidSpan); + + return new DiskLogRecord(recordBuffer, transientObjectIdMap, + logRecord.Info.KeyIsOverflow ? logRecord.KeyOverflow : default, + logRecord.Info.ValueIsOverflow ? logRecord.ValueOverflow : default, + logRecord.Info.ValueIsObject ? logRecord.ValueObject : default); + } + + /// + /// Copies a LogRecord with no out-of-line objects into our contained . + /// + /// The record buffer, either from IO or a copy for pending operations such as Compact or ConditionalCopyToTail. + /// The to hold the objects for the for the lifetime of this . + /// The key overflow byte[] wrapper, if any + /// The value overflow byte[] wrapper, if any + /// The value object, if any + internal static DiskLogRecord TransferFrom(ref SectorAlignedMemory recordBuffer, ObjectIdMap transientObjectIdMap, OverflowByteArray keyOverflow, + OverflowByteArray valueOverflow, IHeapObject valueObject) + { + var diskLogRecord = new DiskLogRecord(recordBuffer, transientObjectIdMap, keyOverflow, valueOverflow, valueObject); + recordBuffer = default; // Transfer ownership to us + return diskLogRecord; + } + + internal static DiskLogRecord TransferFrom(ref DiskLogRecord srcDiskLogRecord, SectorAlignedBufferPool bufferPool) + { + DiskLogRecord diskLogRecord; + if (srcDiskLogRecord.recordBuffer is not null) + diskLogRecord = new DiskLogRecord(in srcDiskLogRecord.logRecord) { recordBuffer = srcDiskLogRecord.recordBuffer }; + else + { + // Deep copy. This is necessary when srcDiskLogRecord does not own its recordBuffer, because the underlying memory + // may be freed or reused--e.g. if it is from an iterator frame. + diskLogRecord = CopyFrom(in srcDiskLogRecord.logRecord, bufferPool, srcDiskLogRecord.logRecord.objectIdMap); + } + + srcDiskLogRecord = default; // Transfer ownership to us, and make sure we don't try to clear the logRecord + return diskLogRecord; + } + + /// + /// Transfers a transient inline record buffer and creates our contained from it. + /// + /// The record buffer, either from IO or a copy for pending operations such as Compact or ConditionalCopyToTail. + /// The to hold the objects for the for the lifetime of this . + internal static DiskLogRecord TransferFrom(ref SectorAlignedMemory recordBuffer, ObjectIdMap transientObjectIdMap) + { + var diskLogRecord = new DiskLogRecord(recordBuffer, transientObjectIdMap); + recordBuffer = default; // Transfer ownership to us + return diskLogRecord; + } + + public void Dispose() + { + if (logRecord.IsSet) + { + // Pure cleanup: clear the inner LogRecord's heap-field slots and release the record buffer. + // The IHeapObject owned by this DiskLogRecord (if any) is disposed via the store-level + // IRecordTriggers.OnDisposeDiskRecord trigger, which callers must invoke before this + // Dispose(). The allocator's OnDisposeDiskRecord forwards to that trigger. + logRecord.Dispose(); + } + logRecord = default; + + recordBuffer?.Return(); + recordBuffer = default; + } + + #region ISourceLogRecord + /// + public readonly bool IsPinnedKey => logRecord.Info.KeyIsInline; + + /// + public readonly byte* PinnedKeyPointer => logRecord.PinnedKeyPointer; + + /// + public OverflowByteArray KeyOverflow + { + readonly get => logRecord.KeyOverflow; + set => logRecord.KeyOverflow = value; + } + + /// + public readonly bool IsPinnedValue => logRecord.Info.ValueIsInline; + + /// + public readonly byte* PinnedValuePointer => logRecord.PinnedValuePointer; + + /// + public OverflowByteArray ValueOverflow + { + readonly get => logRecord.ValueOverflow; + set => logRecord.ValueOverflow = value; + } + + /// + public readonly SpanByteAndMemory ValueSpanByteAndMemory + { + get + { + // For an inline value, the underlying SpanByte points into this DiskLogRecord's + // recordBuffer (a SectorAlignedMemory rented from a pool). That buffer is returned + // to the pool when this DiskLogRecord is disposed -- typically as part of pending- + // completion cleanup, immediately after the read callback returns, or when a scan + // iterator advances. To keep the contract uniform with in-memory LogRecord (where + // SpanByte is stable for the unsafe context), copy the bytes into a pooled + // IMemoryOwner so the returned SpanByteAndMemory remains valid past disposal. + if (logRecord.IsPinnedValue) + { + var span = logRecord.ValueSpan; + var owner = MemoryPool.Shared.Rent(span.Length); + span.CopyTo(owner.Memory.Span); + return new SpanByteAndMemory(owner, span.Length); + } + + // Overflow values come back as a no-copy BorrowedMemoryOwner around the underlying + // GC-managed byte[]. The byte[] stays rooted via the Memory reference inside + // the owner, so it survives DiskLogRecord disposal without an extra copy. + return logRecord.ValueSpanByteAndMemory; + } + } + + /// + public readonly byte RecordType => logRecord.IsSet ? logRecord.RecordType : default; + + /// + public readonly ReadOnlySpan Namespace => logRecord.IsSet ? logRecord.Namespace : default; + + /// + public readonly ObjectIdMap ObjectIdMap => logRecord.objectIdMap; + + /// + public readonly bool IsSet => logRecord.IsSet; + + /// + public ref RecordInfo InfoRef => ref logRecord.InfoRef; + /// + public readonly RecordInfo Info => logRecord.Info; + + /// + public readonly ReadOnlySpan Key => logRecord.Key; + + /// + public readonly Span ValueSpan => logRecord.ValueSpan; + + /// + public readonly IHeapObject ValueObject => logRecord.ValueObject; + + /// + public readonly long ETag => logRecord.IsSet ? logRecord.ETag : LogRecord.NoETag; + + /// + public readonly long Expiration => logRecord.Expiration; + + /// + public readonly void ClearValueIfHeap() { } // Nothing to do here; we dispose the object in the pending operation or iteration completion + + /// + public readonly bool IsMemoryLogRecord => false; + + /// + public readonly unsafe ref LogRecord AsMemoryLogRecordRef() => throw new TsavoriteException("DiskLogRecord cannot be returned as MemoryLogRecord"); + + /// + public readonly bool IsDiskLogRecord => true; + + /// + public readonly unsafe ref DiskLogRecord AsDiskLogRecordRef() => ref Unsafe.AsRef(in this); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordFieldInfo GetRecordFieldInfo() => logRecord.GetRecordFieldInfo(); + + /// + public readonly int AllocatedSize => logRecord.AllocatedSize; + + /// + public readonly int ActualSize => logRecord.ActualSize; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly long CalculateHeapMemorySize() => logRecord.CalculateHeapMemorySize(); + #endregion //ISourceLogRecord + + #region IKey + /// + public readonly bool IsPinned => IsPinnedKey; + + /// + public readonly ReadOnlySpan KeyBytes => Key; + + /// + public readonly bool HasNamespace => logRecord.HasNamespace; + + /// + public readonly ReadOnlySpan NamespaceBytes => logRecord.NamespaceBytes; + #endregion + + + #region Serialization to and from expanded record format + /// + /// Serialize a log record (which may be in-memory or IO'd ) to the + /// in inline-expanded format, with the Overflow Keys and Values and Object Values serialized inline to the Key and Value spans. + /// The serialized layout is: + /// + /// Inline portion of the LogRecord: RecordInfo, IndicatorWord, Key and Value data (each of 4 byte length, which restores to object Id), optionals (ETag, Expiration, ObjectLogPosition, ...) + /// Key data, if key is Overflow + /// Value data, if value is Overflow or Object + /// + /// + /// The Source log record to be serialized to inline form + /// The size of heap allocation to make in . + /// if the record is larger than the inline .. This can happen because: + /// + /// The known record size (due to inline data and Overflow (whose length is known in advance)) is larger than the + /// . size + /// The Object Value serialization (if any) surpasses the . size + /// (we don't know how big an object is until we serialize it) + /// + /// The serializer for the value object, if there is one (ignored if not) + /// The memory pool to use to allocate bytes into + /// . if we overflow the .. + /// The output to receive the serialized logRecord data, either in . + /// or . + /// + /// This is used for migration and replication, and output.SpanByteAndMemory is a span of the remaining space in the network buffer. + /// + /// If ., it points directly to the network buffer so we include the length prefix in the output. + /// is allocated if needed; in that case the caller will flush the network buffer and retry with the full length. + /// The record stream is prefixed with the int length of the stream. RespReadUtils.GetSerializedRecordSpan sets up for deserialization from the network buffer. + /// + /// + /// The total number of bytes in the output. + public static int Serialize(in TSourceLogRecord srcLogRecord, int maxHeapAllocationSize, IObjectSerializer valueObjectSerializer, + MemoryPool memoryPool, ref SpanByteAndMemory output) + where TSourceLogRecord : ISourceLogRecord + { + if (srcLogRecord.IsMemoryLogRecord) + return SerializeLogRecord(in srcLogRecord.AsMemoryLogRecordRef(), maxHeapAllocationSize, valueObjectSerializer, memoryPool, ref output); + + if (!srcLogRecord.IsDiskLogRecord) + throw new TsavoriteException("Unknown TSourceLogRecord type"); + return SerializeLogRecord(in srcLogRecord.AsDiskLogRecordRef().logRecord, maxHeapAllocationSize, valueObjectSerializer, memoryPool, ref output); + } + + // TODO: long value sizes (larger than the network buffer) are currently not supported; need to create a chunked protocol that will write incrementally to a PinnedMemoryStream + // specialized on a new ReplicaStreamBuffer that wraps the network buffer's PinnedSpanByte and the FlushAndReset callback does the network buffer flush and reset (with + // extra work to make this send to multiple replicas) and updates the output available length. Currently using maxHeapAllocationSize to get around not yet having this. + + static int SerializeLogRecord(in LogRecord logRecord, int maxHeapAllocationSize, IObjectSerializer valueObjectSerializer, MemoryPool memoryPool, ref SpanByteAndMemory output) + { + // TotalSize includes the length prefix, which is included in the output stream if we can write directly to the SpanByte, which is a span in the + // network buffer. In case of significant shrinkage, calculate this AllocatedSize separately rather than logRecord.GetInlineRecordSizes().allocatedSize. + var alignedInlineRecordSize = RoundUp(logRecord.ActualSize, Constants.kRecordAlignment); + var estimatedTotalSize = alignedInlineRecordSize + sizeof(int); // Include the record-size prefix in case we can use the SpanByte directly (see DirectCopyInlinePortionOfRecord) + + var heapSize = 0; + if (logRecord.Info.RecordIsInline) + { + // estimatedTotalSize is accurate here. + DirectCopyInlinePortionOfRecord(in logRecord, alignedInlineRecordSize, estimatedTotalSize, maxHeapAllocationSize, memoryPool, ref output); + } + else + { + var estimatedRecordHeapSize = logRecord.Info.KeyIsOverflow ? logRecord.KeyOverflow.Length : 0; + if (logRecord.Info.ValueIsOverflow) + estimatedRecordHeapSize += logRecord.ValueOverflow.Length; + else if (logRecord.Info.ValueIsObject) + { + // We don't know this size exactly so use a small value. Either we'll fit in inline and then have to adjust if we go beyond that, or we + // will just allocate max here and then if we go beyond that, we'll throw the capacity-exceeded exception. This is where "estimated" comes in. + estimatedRecordHeapSize += 1 << 17; // Currently 128KB, matching the processing layer's BufferSizeUtils.MaxBatchSize + } + + estimatedTotalSize += estimatedRecordHeapSize; + + // DirectCopyInlinePortionOfRecord will allocate Memory in output if needed. + DirectCopyInlinePortionOfRecord(in logRecord, alignedInlineRecordSize, estimatedTotalSize, maxHeapAllocationSize, memoryPool, ref output); + heapSize = SerializeHeapObjects(in logRecord, alignedInlineRecordSize, estimatedRecordHeapSize, valueObjectSerializer, ref output); + } + return alignedInlineRecordSize + heapSize; + } + + /// + /// Directly copies a record in inline format to the SpanByteAndMemory. Allocates if needed. + /// + /// If ., it points directly to the network buffer so we include the length prefix in the output. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void DirectCopyInlinePortionOfRecord(in TSourceLogRecord logRecord, int alignedInlineRecordSize, int estimatedTotalSize, int maxHeapAllocationSize, + MemoryPool memoryPool, ref SpanByteAndMemory output) + where TSourceLogRecord : ISourceLogRecord + { + // See if we have enough space in the SpanByte and, if not, if we would fit in maxHeapAllocationSize. + // For SpanByte the recordSize must include the length prefix, which is included in the output stream + // if we can write directly to the SpanByte, which is a span in the network buffer. + if (!output.IsSpanByte || estimatedTotalSize + sizeof(int) > output.SpanByte.Length || logRecord.Info.ValueIsObject) + { + var allocationSizeToUse = logRecord.Info.ValueIsObject ? maxHeapAllocationSize : estimatedTotalSize + sizeof(int); + if (estimatedTotalSize > allocationSizeToUse) + throw new TsavoriteException($"estimatedRecordSize ({estimatedTotalSize}) exceeds max allocated heap size (to use: {allocationSizeToUse}; max: {maxHeapAllocationSize})"); + output.EnsureHeapMemorySize(allocationSizeToUse, memoryPool); + } + + // We must reset the LogRecord's filler size, because we truncated the record down to the (rounded-up) ActualSize if it had been shrunken. + var newFillerLength = alignedInlineRecordSize - logRecord.ActualSize; + if (output.IsSpanByte) + { + // TotalSize includes the length prefix. If there is a SpanByte it is a span in the network buffer, so we include the prefix length in the output stream. + var outPtr = output.SpanByte.ToPointer(); + *(int*)outPtr = alignedInlineRecordSize; + outPtr += sizeof(int); + Buffer.MemoryCopy((byte*)logRecord.PhysicalAddress, outPtr, alignedInlineRecordSize, alignedInlineRecordSize); + new LogRecord((long)outPtr).SetRecordAndFillerLength(alignedInlineRecordSize, newFillerLength); + } + else + { + // Do not include the length prefix in the output stream; this is done by the caller before writing the stream to the network buffer. + fixed (byte* outPtr = output.MemorySpan) + { + Buffer.MemoryCopy((byte*)logRecord.PhysicalAddress, outPtr, alignedInlineRecordSize, alignedInlineRecordSize); + new LogRecord((long)outPtr).SetRecordAndFillerLength(alignedInlineRecordSize, newFillerLength); + } + } + } + + /// + /// Serialize any Overflow Key and Overflow or Object Value into the output after the inline record portion. + /// + /// The log record to serialize + /// The size of the inline portion of the log record (which includes the space for ObjectIds, but not the object/overflow data) + /// + /// + /// + /// + private static int SerializeHeapObjects(in LogRecord logRecord, int inlineRecordSize, int heapSize, IObjectSerializer valueObjectSerializer, ref SpanByteAndMemory output) + { + if (logRecord.Info.RecordIsInline) + return inlineRecordSize; + + // Serialize Key then Value, just like the Object log file. And this will be easy to modify for future chunking of multi-networkBuffer Keys and Values. + var outputOffset = (ulong)inlineRecordSize; + if (logRecord.Info.KeyIsOverflow) + { + var overflow = logRecord.KeyOverflow; + overflow.ReadOnlySpan.CopyTo(output.Span.Slice(inlineRecordSize)); + outputOffset += (ulong)overflow.Length; + } + + var valueObjectLength = 0UL; + if (logRecord.Info.ValueIsOverflow) + { + var overflow = logRecord.ValueOverflow; + overflow.ReadOnlySpan.CopyTo(output.Span.Slice((int)outputOffset)); + valueObjectLength = (ulong)overflow.Length; + } + else + { + Debug.Assert(logRecord.Info.ValueIsObject, "Expected ValueIsObject to be true"); + if (output.IsSpanByte) + valueObjectLength = DoSerialize(logRecord.ValueObject, valueObjectSerializer, output.SpanByte.ToPointer() + outputOffset, output.Length); + else + { + fixed (byte* ptr = output.MemorySpan.Slice((int)outputOffset)) + valueObjectLength = DoSerialize(logRecord.ValueObject, valueObjectSerializer, ptr, output.Length); + } + } + outputOffset += valueObjectLength; + + // Create a temp LogRecord over the output data so we can store the lengths in serialized format, using the offset to the serialized + // part of the buffer as a fake file offset (implicitly for segment 0). + var fakeFilePos = new ObjectLogFilePositionInfo((ulong)inlineRecordSize, segSizeBits: 0); + if (output.IsSpanByte) + { + var serializedLogRecord = new LogRecord((long)output.SpanByte.ToPointer()); + serializedLogRecord.SetObjectLogRecordStartPositionAndLength(fakeFilePos, valueObjectLength); + } + else + { + fixed (byte* ptr = output.MemorySpan.Slice(0, inlineRecordSize)) + { + var serializedLogRecord = new LogRecord((long)ptr, logRecord.objectIdMap); + serializedLogRecord.SetObjectLogRecordStartPositionAndLength(fakeFilePos, valueObjectLength); + serializedLogRecord = new LogRecord((long)ptr); // Reset to clear objectIdMap because it may be the one in the main log and we pass in a transient one when deserializing + } + } + + return (int)outputOffset - inlineRecordSize; + + static ulong DoSerialize(IHeapObject valueObject, IObjectSerializer valueObjectSerializer, byte* destPtr, int destLength) + { + var stream = new UnmanagedMemoryStream(destPtr, destLength, destLength, FileAccess.ReadWrite); + valueObjectSerializer.BeginSerialize(stream); + valueObjectSerializer.Serialize(valueObject); + valueObjectSerializer.EndSerialize(); + var valueLength = (ulong)stream.Position; + return valueLength; + } + } + + /// + /// Deserialize from a over a stream of bytes created by . + /// + public static DiskLogRecord Deserialize(PinnedSpanByte recordSpan, IObjectSerializer valueObjectSerializer, ObjectIdMap transientObjectIdMap, + TStoreFunctions storeFunctions) + where TStoreFunctions : IStoreFunctions + { + // Serialize() did not change the state of the KeyIsInline/ValueIsInline/ValueIsObject bits, but it did change the value at the ObjectId + // location to be serialized length. Create a transient logRecord to decode these and restore the objectId values. + var ptr = recordSpan.ToPointer(); + var serializedLogRecord = new LogRecord((long)ptr, transientObjectIdMap); + if (serializedLogRecord.Info.RecordIsInline) + return new(serializedLogRecord); + var offset = serializedLogRecord.GetObjectLogRecordStartPositionAndLengths(out var keyLength, out var valueLength); + + // Note: Similar logic to this is in ObjectLogReader.ReadObjects. + var keyWasSet = false; + try + { + if (serializedLogRecord.Info.KeyIsOverflow) + { + // This assignment also allocates the slot in ObjectIdMap. The RecordDataHeader length info should be unchanged from ObjectIdSize. + serializedLogRecord.KeyOverflow = new OverflowByteArray(keyLength, startOffset: 0, endOffset: 0, zeroInit: false); + recordSpan.ReadOnlySpan.Slice((int)offset, keyLength).CopyTo(serializedLogRecord.KeyOverflow.Span); + offset += (uint)keyLength; + keyWasSet = true; + } + + if (serializedLogRecord.Info.ValueIsOverflow) + { + // This assignment also allocates the slot in ObjectIdMap. The RecordDataHeader length info should be unchanged from ObjectIdSize. + serializedLogRecord.ValueOverflow = new OverflowByteArray((int)valueLength, startOffset: 0, endOffset: 0, zeroInit: false); + recordSpan.ReadOnlySpan.Slice((int)offset, (int)valueLength).CopyTo(serializedLogRecord.ValueOverflow.Span); + } + else + { + var stream = new UnmanagedMemoryStream(ptr + offset, (int)valueLength); + valueObjectSerializer.BeginDeserialize(stream); + valueObjectSerializer.Deserialize(out var valueObject); + serializedLogRecord.ValueObject = valueObject; + valueObjectSerializer.EndDeserialize(); + } + return new(serializedLogRecord); + } + catch + { + serializedLogRecord.OnDeserializationError(keyWasSet); + throw; + } + } + + /// + /// Return the serialized size of the contained logRecord. + /// + public readonly int GetSerializedSize() => logRecord.GetSerializedSize(); + + #endregion Serialization to and from expanded record format + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocator.cs deleted file mode 100644 index bc14667d390..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocator.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Runtime.CompilerServices; - -namespace Tsavorite.core -{ - /// - /// Struct wrapper (for inlining) around the fixed-length Blittable allocator. - /// - public struct GenericAllocator : IAllocator - where TStoreFunctions : IStoreFunctions - { - /// The wrapped class containing all data and most actual functionality. This must be the ONLY field in this structure so its size is sizeof(IntPtr). - private readonly GenericAllocatorImpl _this; - - public GenericAllocator(AllocatorSettings settings, TStoreFunctions storeFunctions) - { - // Called by TsavoriteKV via allocatorCreator; must pass a wrapperCreator to AllocatorBase - _this = new(settings, storeFunctions, @this => new GenericAllocator(@this)); - } - - public GenericAllocator(object @this) - { - // Called by AllocatorBase via primary ctor wrapperCreator - _this = (GenericAllocatorImpl)@this; - } - - /// - public readonly AllocatorBase GetBase() - where TAllocator : IAllocator - => (AllocatorBase)(object)_this; - - /// - public readonly bool IsFixedLength => true; - - /// - public readonly bool HasObjectLog => true; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetStartLogicalAddress(long page) => _this.GetStartLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetFirstValidLogicalAddress(long page) => _this.GetFirstValidLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetPhysicalAddress(long logicalAddress) => _this.GetPhysicalAddress(logicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref RecordInfo GetInfo(long physicalAddress) => ref _this.GetInfo(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe ref RecordInfo GetInfoFromBytePointer(byte* ptr) => ref _this.GetInfoFromBytePointer(ptr); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TKey GetKey(long physicalAddress) => ref _this.GetKey(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TValue GetValue(long physicalAddress) => ref _this.GetValue(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref TValue GetAndInitializeValue(long physicalAddress, long endPhysicalAddress) => ref GetValue(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) => _this.GetRecordSize(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref TKey key, ref TInput input, ref TValue value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - where TVariableLengthInput : IVariableLengthInput - => _this.GetRMWCopyDestinationRecordSize(ref key, ref input, ref value, ref recordInfo, varlenInput); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref TKey key) - => _this.GetTombstoneRecordSize(ref key); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetRequiredRecordSize(long physicalAddress, int availableBytes) => GetAverageRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetAverageRecordSize() => _this.GetAverageRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetFixedRecordSize() => _this.GetFixedRecordSize(); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref TKey key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => _this.GetRMWInitialRecordSize(ref key, ref input, sessionFunctions); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref TKey key, ref TValue value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => _this.GetUpsertRecordSize(ref key, ref value, ref input, sessionFunctions); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref TKey key, ref TValue value) => _this.GetRecordSize(ref key, ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetValueLength(ref TValue value) => _this.GetValueLength(ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) => _this.RetrievedFullRecord(record, ref ctx); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void AllocatePage(int pageIndex) => _this.AllocatePage(pageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly bool IsAllocated(int pageIndex) => _this.IsAllocated(pageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe void PopulatePage(byte* src, int required_bytes, long destinationPageIndex) => _this.PopulatePage(src, required_bytes, destinationPageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPage(long logicalAddress, long version) => _this.MarkPage(logicalAddress, version); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPageAtomic(long logicalAddress, long version) => _this.MarkPageAtomic(logicalAddress, version); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void ClearPage(long page, int offset = 0) => _this.ClearPage(page, offset); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); - - /// - public readonly ref TKey GetContextRecordKey(ref AsyncIOContext ctx) => ref ctx.key; - - /// - public readonly ref TValue GetContextRecordValue(ref AsyncIOContext ctx) => ref ctx.value; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetKeyContainer(ref TKey key) => new StandardHeapContainer(ref key); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetValueContainer(ref TValue value) => new StandardHeapContainer(ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long[] GetSegmentOffsets() => _this.GetSegmentOffsets(); - - /// - public readonly int OverflowPageCount => _this.OverflowPageCount; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void SerializeKey(ref TKey key, long physicalAddress) => _this.SerializeKey(ref key, physicalAddress); - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocatorImpl.cs deleted file mode 100644 index 453f2be0818..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericAllocatorImpl.cs +++ /dev/null @@ -1,1093 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Runtime.CompilerServices; -using System.Threading; -using Microsoft.Extensions.Logging; - -namespace Tsavorite.core -{ - internal sealed unsafe class GenericAllocatorImpl : AllocatorBase> - where TStoreFunctions : IStoreFunctions - { - // Circular buffer definition - internal AllocatorRecord[][] values; - - // Object log related variables - private readonly IDevice objectLogDevice; - // Size of object chunks being written to storage - private readonly int ObjectBlockSize = 100 * (1 << 20); - // Tail offsets per segment, in object log - public readonly long[] segmentOffsets; - - // Record sizes. We do not support variable-length keys in GenericAllocator - internal static int KeySize => Unsafe.SizeOf(); - internal static int ValueSize => Unsafe.SizeOf(); - internal static int RecordSize => Unsafe.SizeOf>(); - - private readonly OverflowPool[]> overflowPagePool; - - public GenericAllocatorImpl(AllocatorSettings settings, TStoreFunctions storeFunctions, Func> wrapperCreator) - : base(settings.LogSettings, storeFunctions, wrapperCreator, settings.evictCallback, settings.epoch, settings.flushCallback, settings.logger) - { - overflowPagePool = new OverflowPool[]>(4); - - if (settings.LogSettings.ObjectLogDevice == null) - throw new TsavoriteException("LogSettings.ObjectLogDevice needs to be specified (e.g., use Devices.CreateLogDevice, AzureStorageDevice, or NullDevice)"); - - if (typeof(TKey) == typeof(SpanByte)) - throw new TsavoriteException("SpanByte Keys cannot be mixed with object Values"); - if (typeof(TValue) == typeof(SpanByte)) - throw new TsavoriteException("SpanByte Values cannot be mixed with object Keys"); - - values = new AllocatorRecord[BufferSize][]; - segmentOffsets = new long[SegmentBufferSize]; - - objectLogDevice = settings.LogSettings.ObjectLogDevice; - - if ((settings.LogSettings.LogDevice as NullDevice) == null && (KeyHasObjects() || ValueHasObjects())) - { - if (objectLogDevice == null) - throw new TsavoriteException("Objects in key/value, but object log not provided during creation of Tsavorite instance"); - if (objectLogDevice.SegmentSize != -1) - throw new TsavoriteException("Object log device should not have fixed segment size. Set preallocateFile to false when calling CreateLogDevice for object log"); - } - } - - internal int OverflowPageCount => overflowPagePool.Count; - - public override void Reset() - { - base.Reset(); - objectLogDevice.Reset(); - for (int index = 0; index < BufferSize; index++) - { - if (IsAllocated(index)) - FreePage(index); - } - Array.Clear(segmentOffsets, 0, segmentOffsets.Length); - Initialize(); - } - - void ReturnPage(int index) - { - Debug.Assert(index < BufferSize); - if (values[index] != default) - { - _ = overflowPagePool.TryAdd(values[index]); - values[index] = default; - _ = Interlocked.Decrement(ref AllocatedPageCount); - } - } - - public override void Initialize() => Initialize(RecordSize); - - /// Get start logical address - internal long GetStartLogicalAddress(long page) => page << LogPageSizeBits; - - /// Get first valid logical address - internal long GetFirstValidLogicalAddress(long page) - { - if (page == 0) - return (page << LogPageSizeBits) + RecordSize; - return page << LogPageSizeBits; - } - - internal ref RecordInfo GetInfo(long physicalAddress) - { - // Offset within page - int offset = (int)(physicalAddress & PageSizeMask); - - // Index of page within the circular buffer - int pageIndex = (int)((physicalAddress >> LogPageSizeBits) & BufferSizeMask); - - return ref values[pageIndex][offset / RecordSize].info; - } - - internal ref RecordInfo GetInfoFromBytePointer(byte* ptr) => ref Unsafe.AsRef>(ptr).info; - - internal ref TKey GetKey(long physicalAddress) - { - // Offset within page - var offset = (int)(physicalAddress & PageSizeMask); - - // Index of page within the circular buffer - var pageIndex = (int)((physicalAddress >> LogPageSizeBits) & BufferSizeMask); - - return ref values[pageIndex][offset / RecordSize].key; - } - - internal ref TValue GetValue(long physicalAddress) - { - // Offset within page - var offset = (int)(physicalAddress & PageSizeMask); - - // Index of page within the circular buffer - var pageIndex = (int)((physicalAddress >> LogPageSizeBits) & BufferSizeMask); - - return ref values[pageIndex][offset / RecordSize].value; - } - - internal (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) => (RecordSize, RecordSize); - - public int GetValueLength(ref TValue value) => ValueSize; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SerializeKey(ref TKey src, long physicalAddress) => GetKey(physicalAddress) = src; - - internal (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref TKey key, ref TInput input, ref TValue value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - => (RecordSize, RecordSize, KeySize); - - internal (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref TKey key) - => (RecordSize, RecordSize, KeySize); - - internal int GetAverageRecordSize() => RecordSize; - - internal int GetFixedRecordSize() => RecordSize; - - internal (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref TKey key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - => (RecordSize, RecordSize, KeySize); - - internal (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref TKey key, ref TValue value) => (RecordSize, RecordSize, KeySize); - - internal (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref TKey key, ref TValue value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - => (RecordSize, RecordSize, KeySize); - - internal override bool TryComplete() - { - var b1 = objectLogDevice.TryComplete(); - var b2 = base.TryComplete(); - return b1 || b2; - } - - /// - /// Dispose memory allocator - /// - public override void Dispose() - { - if (values != null) - { - for (int i = 0; i < values.Length; i++) - values[i] = null; - values = null; - } - overflowPagePool.Dispose(); - base.Dispose(); - } - - /// Delete in-memory portion of the log - internal override void DeleteFromMemory() - { - for (int i = 0; i < values.Length; i++) - values[i] = null; - values = null; - } - - internal AddressInfo* GetKeyAddressInfo(long physicalAddress) - => (AddressInfo*)Unsafe.AsPointer(ref Unsafe.AsRef>((byte*)physicalAddress).key); - - internal AddressInfo* GetValueAddressInfo(long physicalAddress) - => (AddressInfo*)Unsafe.AsPointer(ref Unsafe.AsRef>((byte*)physicalAddress).value); - - /// Allocate memory page, pinned in memory, and in sector aligned form, if possible - internal void AllocatePage(int index) => values[index] = AllocatePage(); - - internal AllocatorRecord[] AllocatePage() - { - IncrementAllocatedPageCount(); - - if (overflowPagePool.TryGet(out var item)) - return item; - - return new AllocatorRecord[(PageSize + RecordSize - 1) / RecordSize]; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static long SnapToLogicalAddressBoundary(ref long logicalAddress) - => logicalAddress = ((logicalAddress - Constants.kFirstValidAddress) / RecordSize) * RecordSize + Constants.kFirstValidAddress; - - public long GetPhysicalAddress(long logicalAddress) => logicalAddress; - - internal bool IsAllocated(int pageIndex) => values[pageIndex] != null; - - protected override void TruncateUntilAddress(long toAddress) - { - base.TruncateUntilAddress(toAddress); - objectLogDevice.TruncateUntilSegment((int)(toAddress >> LogSegmentSizeBits)); - } - - protected override void TruncateUntilAddressBlocking(long toAddress) - { - base.TruncateUntilAddressBlocking(toAddress); - objectLogDevice.TruncateUntilSegment((int)(toAddress >> LogSegmentSizeBits)); - } - - protected override void RemoveSegment(int segment) - { - base.RemoveSegment(segment); - objectLogDevice.RemoveSegment(segment); - } - - protected override void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult) - { - WriteAsync(flushPage, - (ulong)(AlignedPageSizeBytes * flushPage), - (uint)PageSize, - callback, - asyncResult, device, objectLogDevice); - } - - protected override void WriteAsyncToDevice - (long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, - PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long[] localSegmentOffsets, long fuzzyStartLogicalAddress) - { - VerifyCompatibleSectorSize(device); - VerifyCompatibleSectorSize(objectLogDevice); - - var epochTaken = false; - if (!epoch.ThisInstanceProtected()) - { - epochTaken = true; - epoch.Resume(); - } - try - { - if (HeadAddress >= (flushPage << LogPageSizeBits) + pageSize) - { - // Requested page is unavailable in memory, ignore - callback(0, 0, asyncResult); - } - else - { - // We are writing to separate device, so use fresh segment offsets - WriteAsync(flushPage, - (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), - (uint)pageSize, callback, asyncResult, - device, objectLogDevice, flushPage, localSegmentOffsets, fuzzyStartLogicalAddress); - } - } - finally - { - if (epochTaken) - epoch.Suspend(); - } - } - - internal void ClearPage(long page, int offset) - => Array.Clear(values[page % BufferSize], offset / RecordSize, values[page % BufferSize].Length - offset / RecordSize); - - internal void FreePage(long page) - { - ClearPage(page, 0); - - // Close segments - var thisCloseSegment = page >> (LogSegmentSizeBits - LogPageSizeBits); - var nextCloseSegment = (page + 1) >> (LogSegmentSizeBits - LogPageSizeBits); - - if (thisCloseSegment != nextCloseSegment) - { - // We are clearing the last page in current segment - segmentOffsets[thisCloseSegment % SegmentBufferSize] = 0; - } - - // If all pages are being used (i.e. EmptyPageCount == 0), nothing to re-utilize by adding - // to overflow pool. - if (EmptyPageCount > 0) - ReturnPage((int)(page % BufferSize)); - } - - private void WriteAsync(long flushPage, ulong alignedDestinationAddress, uint numBytesToWrite, - DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult, - IDevice device, IDevice objlogDevice, long intendedDestinationPage = -1, long[] localSegmentOffsets = null, long fuzzyStartLogicalAddress = long.MaxValue) - { - // Short circuit if we are using a null device - if ((device as NullDevice) != null) - { - device.WriteAsync(IntPtr.Zero, 0, 0, numBytesToWrite, callback, asyncResult); - return; - } - - int start = 0, aligned_start = 0, end = (int)numBytesToWrite; - if (asyncResult.partial) - { - // We're writing only a subset of the page - start = (int)(asyncResult.fromAddress - (asyncResult.page << LogPageSizeBits)); - aligned_start = (start / sectorSize) * sectorSize; - end = (int)(asyncResult.untilAddress - (asyncResult.page << LogPageSizeBits)); - } - - // Check if user did not override with special segment offsets - localSegmentOffsets ??= segmentOffsets; - - // This is the in-memory buffer page to be written - var src = values[flushPage % BufferSize]; - - // We create a shadow copy of the page if we are under epoch protection. - // This copy ensures that object references are kept valid even if the original page is reclaimed. - // We suspend epoch during the actual flush as that can take a long time. - var epochProtected = false; - if (epoch.ThisInstanceProtected()) - { - epochProtected = true; - src = new AllocatorRecord[values[flushPage % BufferSize].Length]; - Array.Copy(values[flushPage % BufferSize], src, values[flushPage % BufferSize].Length); - epoch.Suspend(); - } - try - { - // Temporary storage to hold the image "template" we'll write to disk: It will have RecordInfos and object pointers that will be overwritten by addresses - // when writing to the main log (both object pointers and addresses are 8 bytes). - var buffer = bufferPool.Get((int)numBytesToWrite); - - if (aligned_start < start && (KeyHasObjects() || ValueHasObjects())) - { - // Do not read back the invalid header of page 0 - if ((flushPage > 0) || (start > GetFirstValidLogicalAddress(flushPage))) - { - // Get the overlapping HLOG from disk as we wrote it with object pointers previously. This avoids object reserialization - PageAsyncReadResult result = new() - { - handle = new CountdownEvent(1) - }; - device.ReadAsync(alignedDestinationAddress + (ulong)aligned_start, (IntPtr)buffer.aligned_pointer + aligned_start, - (uint)sectorSize, AsyncReadPageCallback, result); - result.handle.Wait(); - } - fixed (RecordInfo* pin = &src[0].info) - { - // Write all the RecordInfos on one operation. This also includes object pointers, but for valid records we will overwrite those below. - Debug.Assert(buffer.aligned_pointer + numBytesToWrite <= (byte*)Unsafe.AsPointer(ref buffer.buffer[0]) + buffer.buffer.Length); - - Buffer.MemoryCopy((void*)((long)Unsafe.AsPointer(ref src[0]) + start), buffer.aligned_pointer + start, - numBytesToWrite - start, numBytesToWrite - start); - } - } - else - { - fixed (RecordInfo* pin = &src[0].info) - { - // Write all the RecordInfos on one operation. This also includes object pointers, but for valid records we will overwrite those below. - Debug.Assert(buffer.aligned_pointer + numBytesToWrite <= (byte*)Unsafe.AsPointer(ref buffer.buffer[0]) + buffer.buffer.Length); - - Buffer.MemoryCopy((void*)((long)Unsafe.AsPointer(ref src[0]) + aligned_start), buffer.aligned_pointer + aligned_start, - numBytesToWrite - aligned_start, numBytesToWrite - aligned_start); - } - } - - // In the main log, we write addresses to pages in the object log. This array saves the addresses of the key and/or value fields in 'buffer', - // which again is the image we're building from the 'values' "page" for this write. The "addresses into 'buffer'" are cast below to AddressInfo - // structures and stored in the sequence we'll write them: alternating series of key then value if both are object types, else keys or values only. - var addr = new List(); - asyncResult.freeBuffer1 = buffer; - - // Object keys and values are serialized into this MemoryStream. - MemoryStream ms = new(); - var keySerializer = KeyHasObjects() ? _storeFunctions.BeginSerializeKey(ms) : null; - var valueSerializer = ValueHasObjects() ? _storeFunctions.BeginSerializeValue(ms) : null; - - // Track the size to be written to the object log. - long endPosition = 0; - - for (int i = start / RecordSize; i < end / RecordSize; i++) - { - byte* recordPtr = buffer.aligned_pointer + i * RecordSize; - - // Retrieve reference to record struct - ref var record = ref Unsafe.AsRef>(recordPtr); - AddressInfo* key_address = null, value_address = null; - - // Zero out object reference addresses (AddressInfo) in the planned disk image - if (KeyHasObjects()) - { - key_address = GetKeyAddressInfo((long)recordPtr); - *key_address = default; - } - if (ValueHasObjects()) - { - value_address = GetValueAddressInfo((long)recordPtr); - *value_address = default; - } - - // Now fill in AddressInfo data for the valid records - if (!record.info.Invalid) - { - // Calculate the logical address of the 'values' page currently being written. - var address = (flushPage << LogPageSizeBits) + i * RecordSize; - - // Do not write v+1 records (e.g. during a checkpoint) - if (address < fuzzyStartLogicalAddress || !record.info.IsInNewVersion) - { - if (KeyHasObjects()) - { - long pos = ms.Position; - keySerializer.Serialize(ref src[i].key); - - // Store the key address into the 'buffer' AddressInfo image as an offset into 'ms'. - key_address->Address = pos; - key_address->Size = (int)(ms.Position - pos); - addr.Add((long)key_address); - endPosition = pos + key_address->Size; - } - - if (ValueHasObjects() && !record.info.Tombstone) - { - long pos = ms.Position; - try - { - valueSerializer.Serialize(ref src[i].value); - } - catch (Exception ex) - { - logger?.LogError(ex, "Failed to serialize value"); - ms.Position = pos; - TValue defaultValue = default; - valueSerializer.Serialize(ref defaultValue); - } - - // Store the value address into the 'buffer' AddressInfo image as an offset into 'ms'. - value_address->Address = pos; - value_address->Size = (int)(ms.Position - pos); - addr.Add((long)value_address); - endPosition = pos + value_address->Size; - } - } - else - { - // Mark v+1 records as invalid to avoid deserializing them on recovery - record.info.SetInvalid(); - } - } - - // If this record's serialized size surpassed ObjectBlockSize or it's the last record to be written, write to the object log. - if (endPosition > ObjectBlockSize || i == (end / RecordSize) - 1) - { - var memoryStreamActualLength = ms.Position; - var memoryStreamTotalLength = (int)endPosition; - endPosition = 0; - - if (KeyHasObjects()) - keySerializer.EndSerialize(); - if (ValueHasObjects()) - valueSerializer.EndSerialize(); - ms.Close(); - - // Get the total serialized length rounded up to sectorSize - var _alignedLength = (memoryStreamTotalLength + (sectorSize - 1)) & ~(sectorSize - 1); - - // Reserve the current address in the object log segment offsets for this chunk's write operation. - var _objAddr = Interlocked.Add(ref localSegmentOffsets[(long)(alignedDestinationAddress >> LogSegmentSizeBits) % SegmentBufferSize], _alignedLength) - _alignedLength; - - // Allocate the object-log buffer to build the image we'll write to disk, then copy to it from the memory stream. - SectorAlignedMemory _objBuffer = null; - if (memoryStreamTotalLength > 0) - { - _objBuffer = bufferPool.Get(memoryStreamTotalLength); - - fixed (void* src_ = ms.GetBuffer()) - Buffer.MemoryCopy(src_, _objBuffer.aligned_pointer, memoryStreamTotalLength, memoryStreamActualLength); - } - - // Each address we calculated above is now an offset to objAddr; convert to the actual address. - foreach (var address in addr) - ((AddressInfo*)address)->Address += _objAddr; - - // If we have not written all records, prepare for the next chunk of records to be written. - if (i < (end / RecordSize) - 1) - { - // Create a new MemoryStream for the next chunk of records to be written. - ms = new MemoryStream(); - if (KeyHasObjects()) - keySerializer.BeginSerialize(ms); - if (ValueHasObjects()) - valueSerializer.BeginSerialize(ms); - - // Reset address list for the next chunk of records to be written. - addr = new List(); - - // Write this chunk of records to the object log device. - asyncResult.done = new AutoResetEvent(false); - Debug.Assert(memoryStreamTotalLength > 0); - objlogDevice.WriteAsync( - (IntPtr)_objBuffer.aligned_pointer, - (int)(alignedDestinationAddress >> LogSegmentSizeBits), - (ulong)_objAddr, (uint)_alignedLength, AsyncFlushPartialObjectLogCallback, asyncResult); - - // Wait for write to complete before resuming next write - _ = asyncResult.done.WaitOne(); - _objBuffer.Return(); - } - else - { - // We have written all records in this 'values' "page". - if (memoryStreamTotalLength > 0) - { - // Increment the count because we need to write both page and object cache. - _ = Interlocked.Increment(ref asyncResult.count); - - asyncResult.freeBuffer2 = _objBuffer; - objlogDevice.WriteAsync( - (IntPtr)_objBuffer.aligned_pointer, - (int)(alignedDestinationAddress >> LogSegmentSizeBits), - (ulong)_objAddr, (uint)_alignedLength, callback, asyncResult); - } - } - } - } - - if (asyncResult.partial) - { - // We're writing only a subset of the page, so update our count of bytes to write. - var aligned_end = (int)(asyncResult.untilAddress - (asyncResult.page << LogPageSizeBits)); - aligned_end = (aligned_end + (sectorSize - 1)) & ~(sectorSize - 1); - numBytesToWrite = (uint)(aligned_end - aligned_start); - } - - // Round up the number of byte to write to sector alignment. - var alignedNumBytesToWrite = (uint)((numBytesToWrite + (sectorSize - 1)) & ~(sectorSize - 1)); - - // Finally write the hlog page - device.WriteAsync((IntPtr)buffer.aligned_pointer + aligned_start, alignedDestinationAddress + (ulong)aligned_start, - alignedNumBytesToWrite, callback, asyncResult); - } - finally - { - if (epochProtected) - epoch.Resume(); - } - } - - private void AsyncReadPageCallback(uint errorCode, uint numBytes, object context) - { - if (errorCode != 0) - logger?.LogError($"{nameof(AsyncReadPageCallback)} error: {{errorCode}}", errorCode); - - // Set the page status to flushed - var result = (PageAsyncReadResult)context; - _ = result.handle.Signal(); - } - - protected override void ReadAsync( - ulong alignedSourceAddress, int destinationPageIndex, uint aligned_read_length, - DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device, IDevice objlogDevice) - { - asyncResult.freeBuffer1 = bufferPool.Get((int)aligned_read_length); - asyncResult.freeBuffer1.required_bytes = (int)aligned_read_length; - - if (!(KeyHasObjects() || ValueHasObjects())) - { - device.ReadAsync(alignedSourceAddress, (IntPtr)asyncResult.freeBuffer1.aligned_pointer, - aligned_read_length, callback, asyncResult); - return; - } - - asyncResult.callback = callback; - - if (objlogDevice == null) - { - Debug.Assert(objectLogDevice != null); - objlogDevice = objectLogDevice; - } - asyncResult.objlogDevice = objlogDevice; - - device.ReadAsync(alignedSourceAddress, (IntPtr)asyncResult.freeBuffer1.aligned_pointer, - aligned_read_length, AsyncReadPageWithObjectsCallback, asyncResult); - } - - - /// - /// IOCompletion callback for page flush - /// - /// - /// - /// - private void AsyncFlushPartialObjectLogCallback(uint errorCode, uint numBytes, object context) - { - if (errorCode != 0) - logger?.LogError($"{nameof(AsyncFlushPartialObjectLogCallback)} error: {{errorCode}}", errorCode); - - // Set the page status to flushed - var result = (PageAsyncFlushResult)context; - _ = result.done.Set(); - } - - private void AsyncReadPageWithObjectsCallback(uint errorCode, uint numBytes, object context) - { - if (errorCode != 0) - logger?.LogError($"{nameof(AsyncReadPageWithObjectsCallback)} error: {{errorCode}}", errorCode); - - var result = (PageAsyncReadResult)context; - - AllocatorRecord[] src; - - // We are reading into a frame - if (result.frame != null) - { - var frame = (GenericFrame)result.frame; - src = frame.GetPage(result.page % frame.frameSize); - } - else - src = values[result.page % BufferSize]; - - - // Deserialize all objects until untilptr - if (result.resumePtr < result.untilPtr) - { - MemoryStream ms = new(result.freeBuffer2.buffer); - ms.Seek(result.freeBuffer2.offset, SeekOrigin.Begin); - // We do not track deserialization size changes if we are deserializing to a frame - Deserialize(result.freeBuffer1.GetValidPointer(), result.resumePtr, result.untilPtr, src, ms, result.frame != null); - ms.Dispose(); - - result.freeBuffer2.Return(); - result.freeBuffer2 = null; - result.resumePtr = result.untilPtr; - } - - // If we have processed entire page, return - if (result.untilPtr >= result.maxPtr) - { - result.Free(); - - // Call the "real" page read callback - result.callback(errorCode, numBytes, context); - return; - } - - // We will now be able to process all records until (but not including) untilPtr - GetObjectInfo(result.freeBuffer1.GetValidPointer(), ref result.untilPtr, result.maxPtr, ObjectBlockSize, out long startptr, out long alignedLength); - - // Object log fragment should be aligned by construction - Debug.Assert(startptr % sectorSize == 0); - Debug.Assert(alignedLength % sectorSize == 0); - - if (alignedLength > int.MaxValue) - throw new TsavoriteException("Unable to read object page, total size greater than 2GB: " + alignedLength); - - var objBuffer = bufferPool.Get((int)alignedLength); - result.freeBuffer2 = objBuffer; - - // Request objects from objlog - result.objlogDevice.ReadAsync( - (int)((result.page - result.offset) >> (LogSegmentSizeBits - LogPageSizeBits)), - (ulong)startptr, - (IntPtr)objBuffer.aligned_pointer, (uint)alignedLength, AsyncReadPageWithObjectsCallback, result); - } - - /// - /// Invoked by users to obtain a record from disk. It uses sector aligned memory to read - /// the record efficiently into memory. - /// - /// - /// - /// - /// - /// - protected override void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default) - { - var fileOffset = (ulong)(AlignedPageSizeBytes * (fromLogical >> LogPageSizeBits) + (fromLogical & PageSizeMask)); - var alignedFileOffset = (ulong)(((long)fileOffset / sectorSize) * sectorSize); - - var alignedReadLength = (uint)((long)fileOffset + numBytes - (long)alignedFileOffset); - alignedReadLength = (uint)((alignedReadLength + (sectorSize - 1)) & ~(sectorSize - 1)); - - var record = bufferPool.Get((int)alignedReadLength); - record.valid_offset = (int)(fileOffset - alignedFileOffset); - record.available_bytes = (int)(alignedReadLength - (fileOffset - alignedFileOffset)); - record.required_bytes = numBytes; - - var asyncResult = default(AsyncGetFromDiskResult>); - asyncResult.context = context; - asyncResult.context.record = result; - asyncResult.context.objBuffer = record; - objectLogDevice.ReadAsync( - (int)(context.logicalAddress >> LogSegmentSizeBits), - alignedFileOffset, - (IntPtr)asyncResult.context.objBuffer.aligned_pointer, - alignedReadLength, - callback, - asyncResult); - } - - /// - /// Read pages from specified device - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - internal void AsyncReadPagesFromDeviceToFrame( - long readPageStart, - int numPages, - long untilAddress, - DeviceIOCompletionCallback callback, - TContext context, - GenericFrame frame, - out CountdownEvent completed, - long devicePageOffset = 0, - IDevice device = null, IDevice objectLogDevice = null) - { - var usedDevice = device ?? this.device; - IDevice usedObjlogDevice = objectLogDevice; - - completed = new CountdownEvent(numPages); - for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++) - { - int pageIndex = (int)(readPage % frame.frameSize); - if (frame.GetPage(pageIndex) == null) - frame.Allocate(pageIndex); - else - frame.Clear(pageIndex); - - var asyncResult = new PageAsyncReadResult() - { - page = readPage, - context = context, - handle = completed, - maxPtr = PageSize, - frame = frame, - }; - - var offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); - var readLength = (uint)AlignedPageSizeBytes; - long adjustedUntilAddress = (AlignedPageSizeBytes * (untilAddress >> LogPageSizeBits) + (untilAddress & PageSizeMask)); - - if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) - { - readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); - asyncResult.maxPtr = readLength; - readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); - } - - if (device != null) - offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); - - ReadAsync(offsetInFile, pageIndex, readLength, callback, asyncResult, usedDevice, usedObjlogDevice); - } - } - - - #region Page handlers for objects - /// - /// Deseialize part of page from stream - /// - /// - /// From pointer - /// Until pointer - /// - /// Stream - /// Whenther we lets observers see this deserialization - public void Deserialize(byte* raw, long ptr, long untilptr, AllocatorRecord[] src, Stream stream, bool doNotObserve) - { - long streamStartPos = stream.Position; - long start_addr = -1; - int start_offset = -1, end_offset = -1; - - var keySerializer = KeyHasObjects() ? _storeFunctions.BeginDeserializeKey(stream) : null; - var valueSerializer = ValueHasObjects() ? _storeFunctions.BeginDeserializeValue(stream) : null; - - while (ptr < untilptr) - { - ref var record = ref Unsafe.AsRef>(raw + ptr); - src[ptr / RecordSize].info = record.info; - if (start_offset == -1) - start_offset = (int)(ptr / RecordSize); - - end_offset = (int)(ptr / RecordSize) + 1; - - if (!record.info.Invalid) - { - if (KeyHasObjects()) - { - var key_addr = GetKeyAddressInfo((long)raw + ptr); - if (start_addr == -1) start_addr = key_addr->Address & ~((long)sectorSize - 1); - if (stream.Position != streamStartPos + key_addr->Address - start_addr) - _ = stream.Seek(streamStartPos + key_addr->Address - start_addr, SeekOrigin.Begin); - - keySerializer.Deserialize(out src[ptr / RecordSize].key); - } - else - src[ptr / RecordSize].key = record.key; - - if (!record.info.Tombstone) - { - if (ValueHasObjects()) - { - var value_addr = GetValueAddressInfo((long)raw + ptr); - if (start_addr == -1) start_addr = value_addr->Address & ~((long)sectorSize - 1); - if (stream.Position != streamStartPos + value_addr->Address - start_addr) - stream.Seek(streamStartPos + value_addr->Address - start_addr, SeekOrigin.Begin); - - valueSerializer.Deserialize(out src[ptr / RecordSize].value); - } - else - src[ptr / RecordSize].value = record.value; - } - } - ptr += GetRecordSize(ptr).Item2; - } - if (KeyHasObjects()) - keySerializer.EndDeserialize(); - if (ValueHasObjects()) - valueSerializer.EndDeserialize(); - - if (OnDeserializationObserver != null && start_offset != -1 && end_offset != -1 && !doNotObserve) - { - using var iter = new MemoryPageScanIterator(src, start_offset, end_offset, -1, RecordSize); - OnDeserializationObserver.OnNext(iter); - } - } - - /// - /// Get location and range of object log addresses for specified log page - /// - /// - /// - /// - /// - /// - /// - public void GetObjectInfo(byte* raw, ref long ptr, long untilptr, int objectBlockSize, out long startptr, out long size) - { - var minObjAddress = long.MaxValue; - var maxObjAddress = long.MinValue; - var done = false; - - while (!done && (ptr < untilptr)) - { - ref var record = ref Unsafe.AsRef>(raw + ptr); - - if (!record.info.Invalid) - { - if (KeyHasObjects()) - { - var key_addr = GetKeyAddressInfo((long)raw + ptr); - var addr = key_addr->Address; - - if (addr < minObjAddress) minObjAddress = addr; - addr += key_addr->Size; - if (addr > maxObjAddress) maxObjAddress = addr; - - // If object pointer is greater than kObjectSize from starting object pointer - if (minObjAddress != long.MaxValue && (addr - minObjAddress > objectBlockSize)) - done = true; - } - - - if (ValueHasObjects() && !record.info.Tombstone) - { - var value_addr = GetValueAddressInfo((long)raw + ptr); - var addr = value_addr->Address; - - if (addr < minObjAddress) minObjAddress = addr; - addr += value_addr->Size; - if (addr > maxObjAddress) maxObjAddress = addr; - - // If object pointer is greater than kObjectSize from starting object pointer - if (minObjAddress != long.MaxValue && (addr - minObjAddress > objectBlockSize)) - done = true; - } - } - ptr += GetRecordSize(ptr).allocatedSize; - } - - // Handle the case where no objects are to be written - if (minObjAddress == long.MaxValue && maxObjAddress == long.MinValue) - { - minObjAddress = 0; - maxObjAddress = 0; - } - - // Align start pointer for retrieval - minObjAddress &= ~((long)sectorSize - 1); - - // Align max address as well - maxObjAddress = (maxObjAddress + (sectorSize - 1)) & ~((long)sectorSize - 1); - - startptr = minObjAddress; - size = maxObjAddress - minObjAddress; - } - - /// Retrieve objects from object log - internal bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) - { - if (!KeyHasObjects()) - ctx.key = Unsafe.AsRef>(record).key; - if (!ValueHasObjects()) - ctx.value = Unsafe.AsRef>(record).value; - - if (!(KeyHasObjects() || ValueHasObjects())) - return true; - - if (ctx.objBuffer == null) - { - // Issue IO for objects - long startAddress = -1; - long endAddress = -1; - if (KeyHasObjects()) - { - var x = GetKeyAddressInfo((long)record); - startAddress = x->Address; - endAddress = x->Address + x->Size; - } - - if (ValueHasObjects() && !GetInfoFromBytePointer(record).Tombstone) - { - var x = GetValueAddressInfo((long)record); - if (startAddress == -1) - startAddress = x->Address; - endAddress = x->Address + x->Size; - } - - // We are limited to a 2GB size per key-value - if (endAddress - startAddress > int.MaxValue) - throw new TsavoriteException("Size of key-value exceeds max of 2GB: " + (endAddress - startAddress)); - - if (startAddress < 0) - startAddress = 0; - - AsyncGetFromDisk(startAddress, (int)(endAddress - startAddress), ctx, ctx.record); - return false; - } - - // Parse the key and value objects - var ms = new MemoryStream(ctx.objBuffer.buffer); - _ = ms.Seek(ctx.objBuffer.offset + ctx.objBuffer.valid_offset, SeekOrigin.Begin); - - if (KeyHasObjects()) - { - var keySerializer = _storeFunctions.BeginDeserializeKey(ms); - keySerializer.Deserialize(out ctx.key); - keySerializer.EndDeserialize(); - } - - if (ValueHasObjects() && !GetInfoFromBytePointer(record).Tombstone) - { - var valueSerializer = _storeFunctions.BeginDeserializeValue(ms); - valueSerializer.Deserialize(out ctx.value); - valueSerializer.EndDeserialize(); - } - - ctx.objBuffer.Return(); - return true; - } - - /// Whether KVS has keys to serialize/deserialize - internal bool KeyHasObjects() => _storeFunctions.HasKeySerializer; - - /// Whether KVS has values to serialize/deserialize - internal bool ValueHasObjects() => _storeFunctions.HasValueSerializer; - #endregion - - public long[] GetSegmentOffsets() => segmentOffsets; - - internal void PopulatePage(byte* src, int required_bytes, long destinationPage) - => PopulatePage(src, required_bytes, ref values[destinationPage % BufferSize]); - - internal void PopulatePageFrame(byte* src, int required_bytes, AllocatorRecord[] frame) - => PopulatePage(src, required_bytes, ref frame); - - internal void PopulatePage(byte* src, int required_bytes, ref AllocatorRecord[] destinationPage) - { - fixed (RecordInfo* pin = &destinationPage[0].info) - { - Debug.Assert(required_bytes <= RecordSize * destinationPage.Length); - Buffer.MemoryCopy(src, Unsafe.AsPointer(ref destinationPage[0]), required_bytes, required_bytes); - } - } - - /// - /// Iterator interface for scanning Tsavorite log - /// - /// - public override ITsavoriteScanIterator Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords) - => new GenericScanIterator(store, this, beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch); - - /// - /// Implementation for push-scanning Tsavorite log, called from LogAccessor - /// - internal override bool Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ref TScanFunctions scanFunctions, ScanBufferingMode scanBufferingMode) - { - using GenericScanIterator iter = new(store, this, beginAddress, endAddress, scanBufferingMode, false, epoch, logger: logger); - return PushScanImpl(beginAddress, endAddress, ref scanFunctions, iter); - } - - /// - /// Implementation for push-scanning Tsavorite log with a cursor, called from LogAccessor - /// - internal override bool ScanCursor(TsavoriteKV> store, - ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) - { - using GenericScanIterator iter = new(store, this, cursor, endAddress, ScanBufferingMode.SinglePageBuffering, includeClosedRecords: maxAddress < long.MaxValue, epoch, logger: logger); - return ScanLookup>(store, scanCursorState, ref cursor, count, scanFunctions, iter, validateCursor, maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); - } - - /// - /// Implementation for push-iterating key versions, called from LogAccessor - /// - internal override bool IterateKeyVersions(TsavoriteKV> store, - ref TKey key, long beginAddress, ref TScanFunctions scanFunctions) - { - using GenericScanIterator iter = new(store, this, beginAddress, epoch, logger: logger); - return IterateKeyVersionsImpl(store, ref key, beginAddress, ref scanFunctions, iter); - } - - private void ComputeScanBoundaries(long beginAddress, long endAddress, out long pageStartAddress, out int start, out int end) - { - pageStartAddress = beginAddress & ~PageSizeMask; - start = (int)(beginAddress & PageSizeMask) / RecordSize; - var count = (int)(endAddress - beginAddress) / RecordSize; - end = start + count; - } - - /// - internal override void EvictPage(long page) - { - if (OnEvictionObserver is not null) - { - var beginAddress = page << LogPageSizeBits; - var endAddress = (page + 1) << LogPageSizeBits; - ComputeScanBoundaries(beginAddress, endAddress, out var pageStartAddress, out var start, out var end); - using var iter = new MemoryPageScanIterator(values[(int)(page % BufferSize)], start, end, pageStartAddress, RecordSize); - OnEvictionObserver?.OnNext(iter); - } - - FreePage(page); - } - - /// - internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver> observer) - { - var page = (beginAddress >> LogPageSizeBits) % BufferSize; - ComputeScanBoundaries(beginAddress, endAddress, out var pageStartAddress, out var start, out var end); - using var iter = new MemoryPageScanIterator(values[page], start, end, pageStartAddress, RecordSize); - Debug.Assert(epoch.ThisInstanceProtected()); - try - { - epoch.Suspend(); - observer?.OnNext(iter); - } - finally - { - epoch.Resume(); - } - } - - internal override void AsyncFlushDeltaToDevice(long startAddress, long endAddress, long prevEndAddress, long version, DeltaLog deltaLog, out SemaphoreSlim completedSemaphore, int throttleCheckpointFlushDelayMs) - { - throw new TsavoriteException("Incremental snapshots not supported with generic allocator"); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericFrame.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/GenericFrame.cs deleted file mode 100644 index cb3e78abef4..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericFrame.cs +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Runtime.CompilerServices; - -namespace Tsavorite.core -{ - /// - /// A frame is an in-memory circular buffer of log pages - /// - internal sealed class GenericFrame : IDisposable - { - private readonly AllocatorRecord[][] frame; - public readonly int frameSize, pageSize; - private static int RecordSize => Unsafe.SizeOf>(); - - public GenericFrame(int frameSize, int pageSize) - { - this.frameSize = frameSize; - this.pageSize = pageSize; - frame = new AllocatorRecord[frameSize][]; - } - - public void Allocate(int index) - { - frame[index] = new AllocatorRecord[(pageSize + RecordSize - 1) / RecordSize]; - } - - public void Clear(int pageIndex) - { - Array.Clear(frame[pageIndex], 0, frame[pageIndex].Length); - } - - public ref TKey GetKey(long frameNumber, long offset) - { - return ref frame[frameNumber][offset].key; - } - - public ref TValue GetValue(long frameNumber, long offset) - { - return ref frame[frameNumber][offset].value; - } - - public ref RecordInfo GetInfo(long frameNumber, long offset) - { - return ref frame[frameNumber][offset].info; - } - - public ref AllocatorRecord[] GetPage(long frameNumber) - { - return ref frame[frameNumber]; - } - - public void Dispose() - { - Array.Clear(frame, 0, frame.Length); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/GenericScanIterator.cs deleted file mode 100644 index 395580541a5..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/GenericScanIterator.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Diagnostics; -using System.Threading; -using Microsoft.Extensions.Logging; - -namespace Tsavorite.core -{ - /// - /// Scan iterator for hybrid log - /// - internal sealed class GenericScanIterator : ScanIteratorBase, ITsavoriteScanIterator, IPushScanIterator - where TStoreFunctions : IStoreFunctions - { - private readonly TsavoriteKV> store; - private readonly GenericAllocatorImpl hlog; - private readonly GenericFrame frame; - private readonly int recordSize; - - private TKey currentKey; - private TValue currentValue; - - private long currentPage = -1, currentOffset = -1, currentFrame = -1; - - /// - /// Constructor - /// - public GenericScanIterator(TsavoriteKV> store, GenericAllocatorImpl hlog, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords, LightEpoch epoch, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch, hlog.LogPageSizeBits, logger: logger) - { - this.store = store; - this.hlog = hlog; - recordSize = hlog.GetRecordSize(0).allocatedSize; - if (frameSize > 0) - frame = new GenericFrame(frameSize, hlog.PageSize); - } - - /// - /// Constructor for use with tail-to-head push iteration of the passed key's record versions - /// - public GenericScanIterator(TsavoriteKV> store, GenericAllocatorImpl hlog, - long beginAddress, LightEpoch epoch, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, hlog.GetTailAddress(), ScanBufferingMode.SinglePageBuffering, false, epoch, hlog.LogPageSizeBits, logger: logger) - { - this.store = store; - this.hlog = hlog; - recordSize = hlog.GetRecordSize(0).allocatedSize; - if (frameSize > 0) - frame = new GenericFrame(frameSize, hlog.PageSize); - } - - /// - /// Gets reference to current key - /// - /// - public ref TKey GetKey() => ref currentKey; - - /// - /// Gets reference to current value - /// - /// - public ref TValue GetValue() => ref currentValue; - - /// - public bool SnapCursorToLogicalAddress(ref long cursor) - { - Debug.Assert(currentAddress == -1, "SnapCursorToLogicalAddress must be called before GetNext()"); - beginAddress = nextAddress = hlog.SnapToFixedLengthLogicalAddressBoundary(ref cursor, GenericAllocatorImpl.RecordSize); - return true; - } - - /// - /// Get next record in iterator - /// - /// True if record found, false if end of scan - public unsafe bool GetNext(out RecordInfo recordInfo) - { - recordInfo = default; - currentKey = default; - currentValue = default; - currentPage = currentOffset = currentFrame = -1; - - while (true) - { - currentAddress = nextAddress; - var stopAddress = endAddress < hlog.GetTailAddress() ? endAddress : hlog.GetTailAddress(); - if (currentAddress >= stopAddress) - return false; - - epoch?.Resume(); - try - { - var headAddress = hlog.HeadAddress; - - if (currentAddress < hlog.BeginAddress) - currentAddress = hlog.BeginAddress; - - // If currentAddress < headAddress and we're not buffering, fail. - if (frameSize == 0 && currentAddress < headAddress) - { - throw new TsavoriteException("Iterator address is less than log HeadAddress in memory-scan mode"); - } - - currentPage = currentAddress >> hlog.LogPageSizeBits; - currentOffset = (currentAddress & hlog.PageSizeMask) / recordSize; - - if (currentAddress < headAddress) - _ = BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, stopAddress); - - // Check if record fits on page, if not skip to next page - if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize) - { - nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits; - continue; - } - - nextAddress = currentAddress + recordSize; - - if (currentAddress >= headAddress) - { - // Read record from cached page memory - currentPage %= hlog.BufferSize; - currentFrame = -1; // Frame is not used in this case. - - recordInfo = hlog.values[currentPage][currentOffset].info; - bool _skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (_skipOnScan || recordInfo.IsNull()) - { - continue; - } - - // Copy the object values from cached page memory to data members; we have no ref into the log after the epoch.Suspend(). - // These are pointer-sized shallow copies but we need to lock to ensure no value tearing inside the object while copying to temp storage. - OperationStackContext> stackCtx = default; - try - { - // We cannot use GetKey() because it has not yet been set. - if (currentAddress >= headAddress && store is not null) - store.LockForScan(ref stackCtx, ref hlog.values[currentPage][currentOffset].key); - - recordInfo = hlog.values[currentPage][currentOffset].info; - currentKey = hlog.values[currentPage][currentOffset].key; - currentValue = hlog.values[currentPage][currentOffset].value; - } - finally - { - if (stackCtx.recSrc.HasLock) - store.UnlockForScan(ref stackCtx); - } - - // Success - return true; - } - - currentFrame = currentPage % frameSize; - recordInfo = frame.GetInfo(currentFrame, currentOffset); - bool skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (skipOnScan || recordInfo.IsNull()) - { - continue; - } - - // Copy the object values from the frame to data members. - currentKey = frame.GetKey(currentFrame, currentOffset); - currentValue = frame.GetValue(currentFrame, currentOffset); - currentPage = currentOffset = -1; - } - finally - { - // Success - epoch?.Suspend(); - } - return true; - } - } - - /// - /// Get previous record and keep the epoch held while we call the user's scan functions - /// - /// True if record found, false if end of scan - bool IPushScanIterator.BeginGetPrevInMemory(ref TKey key, out RecordInfo recordInfo, out bool continueOnDisk) - { - recordInfo = default; - currentKey = default; - currentValue = default; - currentPage = currentOffset = currentFrame = -1; - continueOnDisk = false; - - while (true) - { - // "nextAddress" is reused as "previous address" for this operation. - currentAddress = nextAddress; - if (currentAddress < hlog.HeadAddress) - { - continueOnDisk = currentAddress >= hlog.BeginAddress; - return false; - } - - epoch?.Resume(); - - currentPage = currentAddress >> hlog.LogPageSizeBits; - currentOffset = (currentAddress & hlog.PageSizeMask) / recordSize; - - // Read record from cached page memory - currentPage %= hlog.BufferSize; - currentFrame = -1; // Frame is not used in this case. - - recordInfo = hlog.values[currentPage][currentOffset].info; - nextAddress = currentAddress + recordSize; - - bool skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (skipOnScan || recordInfo.IsNull() || !hlog._storeFunctions.KeysEqual(ref hlog.values[currentPage][currentOffset].key, ref key)) - { - epoch?.Suspend(); - continue; - } - - // Copy the object values from cached page memory to data members; we have no ref into the log after the epoch.Suspend(). - // These are pointer-sized shallow copies. - recordInfo = hlog.values[currentPage][currentOffset].info; - currentKey = hlog.values[currentPage][currentOffset].key; - currentValue = hlog.values[currentPage][currentOffset].value; - - // Success; defer epoch?.Suspend(); to EndGet - return true; - } - } - - bool IPushScanIterator.EndGetPrevInMemory() - { - epoch?.Suspend(); - return true; - } - - /// - /// Get next record using iterator - /// - /// - /// - /// - /// - public bool GetNext(out RecordInfo recordInfo, out TKey key, out TValue value) - { - if (GetNext(out recordInfo)) - { - key = currentKey; - value = currentValue; - return true; - } - - key = default; - value = default; - return false; - } - - /// - /// Dispose iterator - /// - public override void Dispose() - { - base.Dispose(); - frame?.Dispose(); - } - - internal override void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, - long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) - => hlog.AsyncReadPagesFromDeviceToFrame(readPageStart, numPages, untilAddress, AsyncReadPagesCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice); - - private unsafe void AsyncReadPagesCallback(uint errorCode, uint numBytes, object context) - { - var result = (PageAsyncReadResult)context; - - if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncReadPagesCallback)} error: {{errorCode}}", errorCode); - result.cts?.Cancel(); - } - - if (result.freeBuffer1 != null) - { - hlog.PopulatePage(result.freeBuffer1.GetValidPointer(), result.freeBuffer1.required_bytes, ref frame.GetPage(result.page % frame.frameSize)); - result.freeBuffer1.Return(); - result.freeBuffer1 = null; - } - - if (errorCode == 0) - _ = result.handle?.Signal(); - - Interlocked.MemoryBarrier(); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/HeapObjectBase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/HeapObjectBase.cs new file mode 100644 index 00000000000..b024f009e49 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/HeapObjectBase.cs @@ -0,0 +1,175 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.IO; +using System.Text; +using System.Threading; +using static Tsavorite.core.Utility; + +namespace Tsavorite.core +{ + /// + /// The base class for heap Value Objects in Tsavorite. + /// + public abstract class HeapObjectBase : IHeapObject + { + /// + public long HeapMemorySize { get; protected set; } + + /// The current internal serialization phase of the object. + SerializationPhase SerializationPhase + { + get => (SerializationPhase)serializationPhaseInt; + set => serializationPhaseInt = (int)value; + } + + int serializationPhaseInt; + + /// The internal serialized bytes of the object. + byte[] serializedBytes; + + /// + /// Create a cloned (shallow copy) of this object + /// + /// The implementation of this method should NOT copy . + public abstract IHeapObject Clone(); + + /// + /// Serialize to the binary writer. + /// + public abstract void DoSerialize(BinaryWriter writer); + + /// + /// Transition the serialization phase of the object. + /// + public bool MakeTransition(SerializationPhase expectedPhase, SerializationPhase nextPhase) + => Interlocked.CompareExchange(ref serializationPhaseInt, (int)nextPhase, (int)expectedPhase) == (int)expectedPhase; + + /// + public abstract void Dispose(); + + /// + public abstract void WriteType(BinaryWriter writer, bool isNull); + + /// + public void Serialize(BinaryWriter writer) + { + while (true) + { + // This is probably called from Flush, including for checkpoints. If CopyUpdate() has already serialized the object, we will use that + // serialized state. Otherwise, we will serialize the object directly to the writer, and not create the serialized byte[]; only + // CopyUpdater does that, as it must ensure the object's (v1) data is not changed during the checkpoint. + if (SerializationPhase == SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZING)) + { + // Directly serialize to wire, do not cache serialized state + WriteType(writer, isNull: false); + DoSerialize(writer); + SerializationPhase = SerializationPhase.REST; + return; + } + + // If we are here, SerializationPhase is one of the .SERIALIZ* states. This means that one of the following is true: + // - Another thread is currently serializing this object (e.g. checkpoint and eviction) + // - CopyUpdate() is serializing this object + // - Serialization is complete. If the serializedBytes array is null, it means the checkpoint has completed and cleared it + // and the object has been superseded in the database so is no longer reachable, so we can write a null indicator. + + if (SerializationPhase == SerializationPhase.SERIALIZED) + { + // If serialized state is cached, use that + var _serialized = serializedBytes; + if (_serialized != null) + { + WriteType(writer, isNull: false); + writer.Write(_serialized); + } + else + { + // Write null object to stream + WriteType(writer, isNull: true); + } + return; + } + + Thread.Yield(); + } + } + + /// + public void CacheSerializedObjectData(ref LogRecord dstLogRecord, ref RMWInfo rmwInfo, bool srcIsOnMemoryLog) + { + // We'll want to clone the source object to the destination log record so PostCopyUpdater can modify it. + // Note that this does a shallow copy of the object's internal structures (e.g. List<>), which means subsequent modifications of newValue + // in the (v+1) version of the record will modify data seen from the 'this' in the (v) record. Normally this is OK because the (v) version + // of the record is not reachable once the (v+1) version is inserted, but if a checkpoint is ongoing, the (v) version is part of that. + // (If this was an Overflow instead of an Object, then PostCopyUpdater will follow the normal RCU logic, creating a new ValueSpan which will + // probably (but not necessarily) be another Overflow.) + Debug.Assert(dstLogRecord.Info.ValueIsObject, $"{GetCurrentMethodName()} must be called for non-object {nameof(dstLogRecord)}."); + + // CopyUpdater may have already set the ValueObject so only set if not already set. + if (!dstLogRecord.ValueObjectIsSet) + _ = dstLogRecord.TrySetValueObject(Clone()); + + // For pending-IO (DiskLogRecord) sources, skip the checkpoint-serialization state machine: + // - the (v) data is already persisted on disk from a prior flush, so there's nothing to capture; + // - 'this' is an ephemeral deserialized object that will be disposed up the pending chain, so + // driving its SerializationPhase and allocating serializedBytes would be pure waste; + // - ClearSourceValueObject is ignored by InternalRMW for non-memory sources anyway. + if (!srcIsOnMemoryLog) + return; + + // If we are not currently taking a checkpoint, we can delete the old version + // since the new version of the object is already created. + if (!dstLogRecord.Info.IsInNewVersion) + { + // Wait for any concurrent ongoing serialization of oldValue to complete + while (true) + { + if (SerializationPhase == (int)SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZED)) + break; + + if ((int)SerializationPhase >= (int)SerializationPhase.SERIALIZED) + break; + + _ = Thread.Yield(); + } + rmwInfo.ClearSourceValueObject = true; + return; + } + + // Create a serialized version for checkpoint version (v). This is only done for CopyUpdate during a checkpoint, to preserve the (v) data + // of the object during a checkpoint while the (v+1) version of the record may modify the shallow-copied internal structures. + while (true) + { + if (SerializationPhase == (int)SerializationPhase.REST && MakeTransition(SerializationPhase.REST, SerializationPhase.SERIALIZING)) + { + using var ms = new MemoryStream(); + using var writer = new BinaryWriter(ms, Encoding.UTF8); + DoSerialize(writer); + serializedBytes = ms.ToArray(); + + SerializationPhase = SerializationPhase.SERIALIZED; // This is the only place .SERIALIZED is set + break; + } + + // If we're here, serializationState is one of the .SERIALIZ* states. CopyUpdate has a lock on the tag chain, so no other thread will + // be running CopyUpdate. Therefore there are two possibilities: + // 1. CopyUpdate has been called before and the state is .SERIALIZED and '_serialized' is created. We're done. + // 2. Serialize() is running (likely in a Flush()) and the state is .SERIALIZING. We will Yield and loop to wait for it to finish. + if (SerializationPhase >= SerializationPhase.SERIALIZED) + break; + + _ = Thread.Yield(); + } + } + + /// + public void ClearSerializedObjectData() + { + // Clear the serialized data, so it can be GC'd + serializedBytes = null; + SerializationPhase = SerializationPhase.REST; // Reset to initial state + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocator.cs index 967e115ac6c..6c1fe626382 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocator.cs @@ -1,59 +1,121 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Tsavorite.core { /// - /// Interface for hybrid log memory allocator struct wrapper for inlining. This contains the performance-critical methods that must be inlined; - /// abstract/virtual methods may be called via . + /// Non-generic interface for hybrid log memory allocator struct wrapper for inlining. This contains the performance-critical methods that must be inlined; + /// abstract/virtual methods may be called via . /// - public interface IAllocator : IAllocatorCallbacks - where TStoreFunctions : IStoreFunctions + public interface IAllocator { - /// The base class instance of the allocator implementation - AllocatorBase GetBase() - where TAllocator : IAllocator; + /// Get record size required to allocate a new record. Includes allocator-specific information such as key and value overflow. + /// Requires to be populated already. + void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo); + } - /// Whether this allocator uses fixed-length records - bool IsFixedLength { get; } + /// + /// Genric interface for hybrid log memory allocator struct wrapper for inlining. This contains the performance-critical methods that must be inlined; + /// abstract/virtual methods may be called via . + /// + public interface IAllocator : IAllocator, IAllocatorCallbacks + where TStoreFunctions : IStoreFunctions + { + /// The base class instance of the allocator implementation + AllocatorBase GetBase() + where TAllocator : IAllocator; /// Whether this allocator uses a separate object log bool HasObjectLog { get; } - /// Cast address range to . For this will also initialize the value to span the address range. - ref TValue GetAndInitializeValue(long physicalAddress, long endPhysicalAddress); + /// Initialize the RecordDataHeader lengths to key length and a value that spans the address range, and the serialize the key into the record. + /// The key to be copied into the record + /// The logical address of the new record + /// The record size info, which tells us the value size and whether that is overflow. + /// The new log record being initialized + void InitializeRecord(TKey key, long logicalAddress, in RecordSizeInfo sizeInfo, ref LogRecord logRecord) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; /// Get copy destination size for RMW, taking Input into account - (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref TKey key, ref TInput input, ref TValue value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - where TVariableLengthInput : IVariableLengthInput; + RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput; /// Get initial record size for RMW, given the and - (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref TKey key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput; + RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput; + + /// Get record size required for the given , , and + RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput; /// Get record size required for the given , , and - (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref TKey key, ref TValue value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput; + RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput; + + /// Get record size required for the given , , and + RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput; + + /// Get record size required for a new tombstone record + RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; - /// Get record size required for the given and - (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref TKey key, ref TValue value); + /// Return the for the allocator page at + LogRecord CreateLogRecord(long logicalAddress); - /// Get the record size for a tombstoned record - (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref TKey key); + /// Return the for the allocator page at + LogRecord CreateLogRecord(long logicalAddress, long physicalAddress); - /// Get the size of the given - int GetValueLength(ref TValue value); + /// Return the for a transient (e.g. iterator or pending IO) page at + LogRecord CreateRemappedLogRecordOverPinnedTransientMemory(long logicalAddress, long physicalAddress); - /// Mark the page that contains as dirty - void MarkPage(long logicalAddress, long version); + /// Return the for transient log records (e.g. iterator) + ObjectIdMap TransientObjectIdMap { get; } - /// Mark the page that contains as dirty atomically - void MarkPageAtomic(long logicalAddress, long version); + /// Dispose an in-memory log record + void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason); - /// Get segment offsets - long[] GetSegmentOffsets(); + /// Dispose an on-disk / transient log record. Invokes the store's + /// trigger; the caller should then call + /// to release the record buffer. + void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason); - /// Serialize key to log - void SerializeKey(ref TKey key, long physicalAddress); + /// + /// Iterate records in the given logical address range and invoke the application-level + /// hook for each valid, non-tombstoned record. + /// Used during page eviction to allow cleanup of external resources. + /// + /// Start logical address of the range. + /// End logical address of the range (exclusive). + /// Identifies whether this eviction is from the main log or the read cache. + void EvictRecordsInRange(long startAddress, long endAddress, EvictionSource source); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocatorCallbacks.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocatorCallbacks.cs index 6c74dea3dcc..63e0506c09a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocatorCallbacks.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/IAllocatorCallbacks.cs @@ -4,79 +4,20 @@ namespace Tsavorite.core { /// - /// Interface for hybrid log memory allocator struct wrapper callbacks for inlining performance-path callbacks from - /// - /// to the fully derived allocator, including both record accessors and Scan calls. + /// Interface for hybrid log memory allocator struct wrapper callbacks for inlining performance-path callbacks from within + /// to the fully derived allocator, including both record accessors and Scan calls. /// /// This interface does not currently appear in type constraints, but the organization may prove useful. - public interface IAllocatorCallbacks - where TStoreFunctions : IStoreFunctions + public interface IAllocatorCallbacks + where TStoreFunctions : IStoreFunctions { - /// Get start logical address on - long GetStartLogicalAddress(long page); - - /// Get first valid logical address on - long GetFirstValidLogicalAddress(long page); - - /// Get physical address from - long GetPhysicalAddress(long logicalAddress); - - /// Get from - ref RecordInfo GetInfo(long physicalAddress); - - /// Get from pinned memory - unsafe ref RecordInfo GetInfoFromBytePointer(byte* ptr); - - /// Get from - ref TKey GetKey(long physicalAddress); - - /// Get from - ref TValue GetValue(long physicalAddress); - - /// Get the actual (used) and allocated record sizes at - (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress); - - /// Get number of bytes required to read the full record that starts at for . - int GetRequiredRecordSize(long physicalAddress, int availableBytes); - - /// Get average record size - int GetAverageRecordSize(); - /// Allocate the page in the circular buffer slot at void AllocatePage(int pageIndex); - /// Whether the page at is allocated - bool IsAllocated(int pageIndex); - - /// - /// Populate the page at from the pointer, which has bytes. - /// - unsafe void PopulatePage(byte* src, int required_bytes, long destinationPageIndex); - - /// Free the page at , starting at - void ClearPage(long pageIndex, int offset = 0); - /// Free the page at void FreePage(long pageIndex); /// Number of extra overflow pages allocated int OverflowPageCount { get; } - - int GetFixedRecordSize(); - - /// Retrieve key from IO context record - ref TKey GetContextRecordKey(ref AsyncIOContext ctx); - - /// Retrieve value from IO context record - ref TValue GetContextRecordValue(ref AsyncIOContext ctx); - - /// Determine whether we IO has returned the full record - unsafe bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx); - - /// Get heap container for pending key - IHeapContainer GetKeyContainer(ref TKey key); - - /// Get heap container for pending value - IHeapContainer GetValueContainer(ref TValue value); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/IHeapObject.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/IHeapObject.cs new file mode 100644 index 00000000000..72acc349faf --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/IHeapObject.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; + +namespace Tsavorite.core +{ + /// + /// This is the base interface from which any value of object type must derive for . + /// + public interface IHeapObject : IDisposable + { + /// + /// The maximum length of a serialized Value object. + /// + public const long MaxSerializedObjectSize = 1L << 40; + + /// + /// Total estimated size of the object in heap memory, including .NET object overheads, for Overflow allocations and Objects. + /// + long HeapMemorySize { get; } + + /// + /// Create a cloned (shallow copy) of this object + /// + IHeapObject Clone(); + + /// + /// Top-level routine to Serialize to the binary writer; checks for cached checkpoint data and calls if needed. + /// + void Serialize(BinaryWriter binaryWriter); + + /// + /// Write the type of the object to the binary writer. + /// + void WriteType(BinaryWriter binaryWriter, bool isNull); + + /// + /// Internal routine to Serialize to the binary writer. + /// + void DoSerialize(BinaryWriter writer); + + /// + /// Clone this object into 's value slot (if not already set) and, when the + /// source record is on the in-memory log, run the checkpoint-serialization state machine on this so + /// the (v) snapshot is preserved for an in-flight checkpoint. + /// + /// Destination record; its ValueObject is set to a clone of this if not already set by the caller. + /// RMW info; is set when the caller may safely null the source's value slot. + /// + /// True when the source record resides on the in-memory log (state machine runs). False for pending-IO + /// sources, where the (v) data is already persisted on disk, the source + /// this is ephemeral and about to be disposed up the pending chain, and clone is all that's needed. + /// + void CacheSerializedObjectData(ref LogRecord dstLogRecord, ref RMWInfo rmwInfo, bool srcIsOnMemoryLog); + + /// + /// Clear any serialized data from + /// + void ClearSerializedObjectData(); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/IScanIteratorFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/IScanIteratorFunctions.cs index e163c6b72dd..c531f472e8c 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/IScanIteratorFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/IScanIteratorFunctions.cs @@ -5,6 +5,8 @@ namespace Tsavorite.core { + using static LogAddress; + /// /// Result for records sent to the callback during ScanCursor. /// @@ -35,49 +37,44 @@ public enum CursorRecordResult /// /// Callback functions for log scan or key-version iteration /// - public interface IScanIteratorFunctions + public interface IScanIteratorFunctions { /// Iteration is starting. /// Start address of the scan - /// End address of the scan; if iterating key versions, this is + /// End address of the scan; if iterating key versions, this is /// True to continue iteration, else false bool OnStart(long beginAddress, long endAddress); - /// Next record in iteration for a record not in mutable log memory. - /// Reference to the current record's key - /// Reference to the current record's Value - /// Record metadata, including and the current record's logical address - /// The number of records accepted so far, not including the current one. - /// Indicates whether the current record was accepted, or whether to end the current ScanCursor call. - /// Ignored for non-cursor Scans; set to . - /// True to continue iteration, else false - bool SingleReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult); - - /// Next record in iteration for a record in mutable log memory. - /// Reference to the current record's key - /// Reference to the current record's Value + /// Read the next record in the iteration. + /// Reference to the current log record's info /// Record metadata, including and the current record's logical address /// The number of records accepted so far, not including the current one. /// Indicates whether the current record was accepted, or whether to end the current ScanCursor call. /// Ignored for non-cursor Scans; set to . /// True to continue iteration, else false - bool ConcurrentReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult); + bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord; /// Iteration is complete. /// If true, the iteration completed; else scanFunctions.*Reader() returned false to stop the iteration. /// The number of records returned before the iteration stopped. void OnStop(bool completed, long numberOfRecords); - /// An exception was thrown on iteration (likely during or . + /// An exception was thrown on iteration (likely during . /// The exception that was thrown. /// The number of records returned, including the current one, before the exception. void OnException(Exception exception, long numberOfRecords); } - internal interface IPushScanIterator + internal interface IPushScanIterator { - bool BeginGetPrevInMemory(ref TKey key, out RecordInfo recordInfo, out bool continueOnDisk); - bool EndGetPrevInMemory(); + bool BeginGetPrevInMemory(TKey key, out LogRecord logRecord, out bool continueOnDisk) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; + void EndGetPrevInMemory(); /// /// When beginning a cursor scan, if it is not the last cursor returned, snap it to the preceding logical address boundary. diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ISourceLogRecord.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ISourceLogRecord.cs new file mode 100644 index 00000000000..e2f4996dd19 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ISourceLogRecord.cs @@ -0,0 +1,136 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// An interface to cover either an in-memory or on-disk log record for RCU + public unsafe interface ISourceLogRecord : IKey + { + /// The physical address of the record data + unsafe long PhysicalAddress => (long)Unsafe.AsPointer(ref InfoRef); + + /// A ref to the record header + ref RecordInfo InfoRef { get; } + + /// Fast access returning a copy of the record header + RecordInfo Info { get; } + + /// Type of the record. Should be set on creation of the and then immutable. + byte RecordType { get; } + + /// Namespace of the record. Should be set on creation of the and then immutable. + ReadOnlySpan Namespace { get; } + + /// The for this instance. May be the allocator's or transient (for ). + ObjectIdMap ObjectIdMap { get; } + + /// Whether there is actually a record here + bool IsSet { get; } + + /// The key, which may be inline in this record or an overflow byte[] + /// Not a ref return as it cannot be changed + ReadOnlySpan Key { get; } + + /// Whether the record's key is pinned in memory, e.g. inline in the log vs an overflow byte[]. If this is true, is non-null. + bool IsPinnedKey { get; } + + /// The pointer to the pinned memory if is true, else null. + byte* PinnedKeyPointer { get; } + + /// Get and set the if this Key is Overflow; an exception is thrown if it is a pinned pointer (e.g. to a . + OverflowByteArray KeyOverflow { get; set; } + + /// The value , if this is a String LogRecord; an assertion is raised if it is an Object LogRecord. + /// Not a ref return as it cannot be changed directly; use instead. + Span ValueSpan { get; } + + /// The value object, if the value in this record is an IHeapObject; an exception is thrown if it is a Span, either inline or overflow byte[]. + IHeapObject ValueObject { get; } + + /// Whether the record's value is pinned in memory, e.g. inline in the log vs an overflow byte[]. If this is true, is non-null. + bool IsPinnedValue { get; } + + /// The pointer to the pinned memory if is true, else null. + byte* PinnedValuePointer { get; } + + /// Get and set the if this Value is not Overflow; an exception is thrown if it is a pinned pointer (e.g. to a . + OverflowByteArray ValueOverflow { get; set; } + + /// + /// Expose the value bytes through the abstraction. + /// + /// + /// The shape and lifetime of the returned depend on + /// the source record: + /// + /// In-memory , inline value — returns a + /// pointing directly at the allocator's main log + /// memory (no copy). The pointer is valid only while the enclosing epoch / unsafe + /// context is held; once the epoch is released the page may be evicted and the + /// pointer becomes invalid. + /// , inline value — the bytes are + /// copied into a pooled returned in + /// . The underlying + /// recordBuffer is returned to its pool when the + /// is disposed (e.g. at the end of pending completion or when + /// a scan iterator advances), so the inline pointer would otherwise dangle; the copy + /// makes the returned safe beyond the callback / + /// iterator scope. + /// Overflow value (either record type) — returns a no-copy + /// wrapping the underlying GC-managed byte[]. The array + /// stays rooted via the reference inside the owner, so the + /// contents survive disposal of the source record (and do not require epoch protection). + /// + /// + /// + /// Consumers that need a stable native pointer (e.g. for SIMD operations) into the + /// path MUST call + /// on it and hold the resulting for the duration + /// of the operation, otherwise GC compaction may relocate the underlying array. + /// The caller owns the returned (when set) + /// and is responsible for disposing it. 's + /// is a no-op; pooled owners returned for the + /// inline path return their buffer to the pool on dispose. + /// Throws if the value is an object. + /// + SpanByteAndMemory ValueSpanByteAndMemory { get; } + + /// The ETag of the record, if any (see ; 0 by default. + long ETag { get; } + + /// The Expiration of the record, if any (see ; 0 by default. + long Expiration { get; } + + /// If requested by CopyUpdater or InPlaceDeleter, the source ValueObject or ValueOverflow will be cleared immediately (to manage object size tracking most effectively). + /// This is called after we have either ensured there is a newer record inserted at tail, or after we have tombstoned the record; either way, we won't be accessing its value. + /// The cleared , if any, has its invoked. + void ClearValueIfHeap(); + + /// Whether this is an instance of + bool IsMemoryLogRecord { get; } + + /// Return this as a ref , or throw if not + ref LogRecord AsMemoryLogRecordRef(); + + /// Whether this is an instance of + bool IsDiskLogRecord { get; } + + /// Return this as a ref , or throw if not + ref DiskLogRecord AsDiskLogRecordRef(); + + /// Get the record's field info, for use in calculating required record size + RecordFieldInfo GetRecordFieldInfo(); + + /// The total allocated inline size of the main-log record; includes filler length. + int AllocatedSize { get; } + + /// The total used inline portion of the size of the main-log portion of the record; does not include filler length. + int ActualSize { get; } + + /// Calculate the heap memory size of this log record + public long CalculateHeapMemorySize(); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/IStreamingSnapshotIteratorFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/IStreamingSnapshotIteratorFunctions.cs index 422353aa745..71deef642d0 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/IStreamingSnapshotIteratorFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/IStreamingSnapshotIteratorFunctions.cs @@ -8,7 +8,7 @@ namespace Tsavorite.core /// /// Callback functions for streaming snapshot iteration /// - public interface IStreamingSnapshotIteratorFunctions + public interface IStreamingSnapshotIteratorFunctions { /// Iteration is starting. /// Checkpoint token @@ -18,12 +18,11 @@ public interface IStreamingSnapshotIteratorFunctions bool OnStart(Guid checkpointToken, long currentVersion, long nextVersion); /// Next record in the streaming snapshot. - /// Reference to the current record's key - /// Reference to the current record's Value + /// Reference to the current record /// Record metadata, including and the current record's logical address /// The number of records returned so far, not including the current one. /// True to continue iteration, else false - bool Reader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords); + bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords) where TSourceLogRecord : ISourceLogRecord; /// Iteration is complete. /// If true, the iteration completed; else OnStart() or Reader() returned false to stop the iteration. diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ITsavoriteScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ITsavoriteScanIterator.cs index 3ecb9286e3a..7add4ffb8ea 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/ITsavoriteScanIterator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ITsavoriteScanIterator.cs @@ -6,9 +6,9 @@ namespace Tsavorite.core { /// - /// Scan buffering mode + /// Scan buffering mode when reading from disk /// - public enum ScanBufferingMode + public enum DiskScanBufferingMode { /// /// Buffer only current page being scanned @@ -27,39 +27,31 @@ public enum ScanBufferingMode } /// - /// Scan iterator interface for Tsavorite log + /// Scan buffering mode for in-memory records, e.g. for copying and holding a record for Pull iterators /// - /// - /// - public interface ITsavoriteScanIterator : IDisposable + public enum InMemoryScanBufferingMode { /// - /// Gets reference to current key - /// - /// - ref TKey GetKey(); - - /// - /// Gets reference to current value + /// Buffer the current record being scanned. Automatic for Pull iteration. /// - /// - ref TValue GetValue(); + CurrentRecordBuffering, /// - /// Get next record + /// Do not buffer - with this mode, Push iteration will hold the epoch during each record's push to the client /// - /// - /// True if record found, false if end of scan - bool GetNext(out RecordInfo recordInfo); + NoBuffering + } + /// + /// Scan iterator interface for Tsavorite log + /// + public interface ITsavoriteScanIterator : ISourceLogRecord, IDisposable + { /// /// Get next record /// - /// - /// - /// /// True if record found, false if end of scan - bool GetNext(out RecordInfo recordInfo, out TKey key, out TValue value); + bool GetNext(); /// /// Current address diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/LogField.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/LogField.cs new file mode 100644 index 00000000000..772c98ea53f --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/LogField.cs @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Static class providing functions to operate on a Log field (Key Span, or Value Span or Object) at a certain address. Since (small) Objects can be represented + /// as inline spans, this applies to those forms as well as the inline component of the Object, which is the ObjectId. The layout is: + /// + /// RecordDataHeader indicator byte and lengths; see header comments for details + /// Key data: either the inline data or an int ObjectId for a byte[] that is held in + /// Value data: either the inline data or an int ObjectId for a byte[] that is held in + /// + /// + /// Considerations regarding variable field sizes: + /// + /// Keys are immutable (unless revivification is happening), so the inline size of a Key field does not change + /// When Values change size the Filler length and offsets to optional ETag and Extension are adjusted. Converting between inline and out-of-line + /// due to size changes altering whether the Value overflows is handled as part of normal Value-sizechange operations + /// + /// + internal static unsafe class LogField + { + /// + /// Convert a Span field from inline to overflow. + /// + /// + /// Applies to Value-only during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field does not currently contain an overflow allocation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span ConvertInlineToOverflow(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, long oldValueLength, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + Debug.Assert(recordInfo.ValueIsInline); + + // First copy the data. We are converting to overflow so the length is limited to int. + var newLength = sizeInfo.FieldInfo.ValueSize; + var overflow = new OverflowByteArray(newLength, startOffset: 0, endOffset: 0, zeroInit: false); + var copyLength = oldValueLength < newLength ? oldValueLength : newLength; + + if (copyLength > 0) + { + var oldSpan = new ReadOnlySpan((byte*)valueAddress, (int)copyLength); + oldSpan.CopyTo(overflow.Span); + } + + var objectId = objectIdMap.Allocate(); + *(int*)valueAddress = objectId; + objectIdMap.Set(objectId, overflow); + recordInfo.SetValueIsOverflow(); + return overflow.Span; + } + + /// + /// Convert a Span field from inline to overflow. + /// + /// + /// Applies to Value-only during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field does not currently contain an overflow allocation. Here we do not copy the data; we assume the caller will have already + /// prepared to convert from Object format to inline format. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span ConvertValueObjectToOverflow(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + Debug.Assert(recordInfo.ValueIsObject); + var overflow = new OverflowByteArray(sizeInfo.FieldInfo.ValueSize, startOffset: 0, endOffset: 0, zeroInit: false); + + var objectId = *(int*)valueAddress; + if (objectId == ObjectIdMap.InvalidObjectId) + *(int*)valueAddress = objectId = objectIdMap.Allocate(); + objectIdMap.Set(objectId, overflow); + recordInfo.SetValueIsOverflow(); + return overflow.Span; + } + + /// + /// Convert a Span field from inline to ObjectId. + /// + /// + /// Applies to Value during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field does not currently contain a valid ObjectId. Here we do not copy the data; we assume the caller will have already + /// created an object that has converted from inline format to object format. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int ConvertInlineToValueObject(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + Debug.Assert(recordInfo.ValueIsInline); + var objectId = objectIdMap.Allocate(); + recordInfo.SetValueIsObject(); + + *(int*)valueAddress = objectId; + return objectId; + } + + /// + /// Convert a Span field from an out-of-line overflow allocation to ObjectId. + /// + /// + /// Applies to Value during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field does not currently contain a valid ObjectId. Here we do not copy the data; we assume the caller will have already + /// created an object that has converted from inline format to object format. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int ConvertOverflowToValueObject(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + Debug.Assert(recordInfo.ValueIsOverflow); + + var objectId = *(int*)valueAddress; + if (objectId != ObjectIdMap.InvalidObjectId) + objectIdMap.Set(objectId, null); // Clear the byte[] from the existing slot but do not free the slot; caller will put the HeapObject into the slot. + else + *(int*)valueAddress = objectId = objectIdMap.Allocate(); + + recordInfo.SetValueIsObject(); + return objectId; + } + + /// + /// Convert a Span field from overflow to inline. + /// + /// + /// Applies to Value-only during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field currently contains an overflow allocation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span ConvertOverflowToInline(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + Debug.Assert(recordInfo.ValueIsOverflow); + + // First copy the data + var objectId = *(int*)valueAddress; + + var newLength = sizeInfo.FieldInfo.ValueSize; + var newSpan = new Span((byte*)valueAddress, newLength); + + if (objectId != ObjectIdMap.InvalidObjectId) + { + var overflow = objectIdMap.GetOverflowByteArray(objectId); + var oldSpan = overflow.Span; + + var copyLength = oldSpan.Length < newLength ? oldSpan.Length : newLength; + recordInfo.SetValueIsInline(); + oldSpan.Slice(0, copyLength).CopyTo(newSpan); + objectIdMap.Free(objectId); + } + return newSpan; + } + + /// + /// Called when disposing a record, to free an Object or Overflow allocation and convert to inline so the lengths are set for record scanning or revivification. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void ClearObjectIdAndConvertToInline(ref RecordInfo recordInfo, long fieldAddress, ObjectIdMap objectIdMap, bool isKey) + { + Debug.Assert(isKey ? !recordInfo.KeyIsInline : !recordInfo.ValueIsInline); + + // We don't have to adjust the filler length, since the field size here isn't changing; we'll just have int-sized "data". This method is called by record disposal, which + // also clears the optionals, which may adjust filler length. Consistency Note: LogRecord.InitializeForReuse also sets field lengths to zero and sets the filler length. + // However, here we may be called after setting the IgnoreOptionals word, so we don't want to decode the indicator. + var objectId = *(int*)fieldAddress; + if (objectId != ObjectIdMap.InvalidObjectId) + { + objectIdMap.Free(objectId); + *(int*)fieldAddress = ObjectIdMap.InvalidObjectId; + } + + // We don't need to change the length; we'll keep the current length and just convert to inline. + if (isKey) + recordInfo.SetKeyIsInline(); + else + recordInfo.SetValueIsInline(); + } + + /// + /// Convert a Value field from ObjectId to inline. + /// + /// + /// Applies to Value during normal ops, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field currently contains an ObjectId (which may be ObjectIdMap.InvalidObjectId). Here we do not copy the data; we assume + /// the caller will have already prepared to convert from Object format to inline format. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span ConvertValueObjectToInline(ref RecordInfo recordInfo, long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + var objIdPtr = (int*)valueAddress; + var objectId = *objIdPtr; + if (objectId != ObjectIdMap.InvalidObjectId) + objectIdMap.Free(objectId); + *objIdPtr = 0; + + recordInfo.SetValueIsInline(); + return new((byte*)valueAddress, sizeInfo.FieldInfo.ValueSize); + } + + /// + /// Reallocate a Span field that is overflow, e.g. to make the overflow allocation larger. Shrinkage is done in-place (the caller decides if the + /// shrinkage is sufficient (given available space in the record) to convert the field in-place to inline. + /// + /// + /// Applies to Value only, and assumes any record size adjustment due to Value growth/shrinkage has already been handled + /// and that the field currently contains an overflow allocation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static Span ReallocateValueOverflow(long physicalAddress, long valueAddress, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + { + OverflowByteArray newOverflow; + var newLength = sizeInfo.FieldInfo.ValueSize; + + var objectId = *(int*)valueAddress; + if (objectId != ObjectIdMap.InvalidObjectId) + { + var oldOverflow = objectIdMap.GetOverflowByteArray(objectId); + var oldSpan = oldOverflow.Span; + if (oldSpan.Length == newLength) + return oldSpan; + + // AllocateUninitialized and copy, and zeroinit any remainder + newOverflow = new(newLength, startOffset: 0, endOffset: 0, zeroInit: false); + var copyLength = oldSpan.Length < newLength ? oldSpan.Length : newLength; + + oldOverflow.AsReadOnlySpan(0, copyLength).CopyTo(newOverflow.Span); + if (copyLength < newLength) + newOverflow.AsSpan(copyLength, newLength - copyLength).Clear(); + } + else + { + // Allocate; nothing to copy, so allocate with zero initialization + newOverflow = new(newLength, startOffset: 0, endOffset: 0, zeroInit: false); + objectId = objectIdMap.Allocate(); + *(int*)valueAddress = objectId; + } + objectIdMap.Set(objectId, newOverflow); + return newOverflow.Span; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/LogRecord.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/LogRecord.cs new file mode 100644 index 00000000000..35aa7e308c7 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/LogRecord.cs @@ -0,0 +1,1537 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + /// The in-memory record on the log. The space is laid out as: + /// + /// header + /// RecordDataHeader bytes (including RecordType and Namespace) and lengths; see header comments for details + /// Key data: either the inline data or an int ObjectId for a byte[] that is held in + /// Value data: either the inline data or an int ObjectId for a byte[] that is held in + /// Optional data (may or may not be present): ETag, Expiration + /// Pseudo-optional ObjectLogPosition indicating the position in the object log file, if the record is not fully inline. + /// Optional filler length: Extra space in the record, due to record-alignment round-up or Value shrinkage + /// + /// This lets us get to the key without intermediate computations having to account for the optional fields. + /// Some methods have both member and static versions for ease of access and possibly performance gains. + /// + public unsafe partial struct LogRecord : ISourceLogRecord + { + /// The physicalAddress in the log. + internal readonly long physicalAddress; + + /// The ObjectIdMap if this is a record in the object log. + internal readonly ObjectIdMap objectIdMap; + + /// Number of bytes required to store an ETag + public const int ETagSize = sizeof(long); + /// Invalid ETag, and also the pre-incremented value + public const long NoETag = 0; + /// Number of bytes required to store an Expiration + public const int ExpirationSize = sizeof(long); + /// Invalid Expiration + public const long NoExpiration = 0; + /// Number of bytes required to the object log position + public const int ObjectLogPositionSize = sizeof(long); + /// Number of bytes required to store the FillerLen + internal const int FillerLengthSize = sizeof(int); + + /// Address-only ctor. Must only be used for simple record parsing, including inline size calculations. + /// In particular, if knowledge of whether this is a string or object record is required, or an overflow allocator is needed, this method cannot be used. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public LogRecord(long physicalAddress) => this.physicalAddress = physicalAddress; + + /// Address-only ctor. Must only be used for simple record parsing, including inline size calculations. + /// In particular, if knowledge of whether this is a string or object record is required, or an overflow allocator is needed, this method cannot be used. + public LogRecord(byte* recordPtr) => physicalAddress = (long)recordPtr; + + /// Address of the + internal readonly long DataHeaderAddress => physicalAddress + RecordInfo.Size; + /// Address of the namespace indicator byte. If the is not set, then the bits + /// contain the full namespace as a single byte; otherwise those bits are the length of the extended namespace data preceding the key data. + private readonly long NamespaceAddress => physicalAddress + RecordInfo.Size + RecordDataHeader.NamespaceOffsetInHeader; + /// Address of the Record type indicator byte + private readonly long RecordTypeAddress => physicalAddress + RecordInfo.Size + RecordDataHeader.RecordTypeOffsetInHeader; + + public readonly byte IndicatorByte => *(byte*)DataHeaderAddress; + + public readonly RecordDataHeader RecordDataHeader => new((byte*)DataHeaderAddress); + + /// This ctor is primarily used for internal record-creation operations for the ObjectAllocator, and is passed to IObjectSessionFunctions callbacks. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public LogRecord(long physicalAddress, ObjectIdMap objectIdMap) + : this(physicalAddress) + { + this.objectIdMap = objectIdMap; + } + + /// This ctor is primarily used for internal record-creation operations for the ObjectAllocator, and is passed to IObjectSessionFunctions callbacks. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public LogRecord(byte* recordPtr, ObjectIdMap objectIdMap) + : this((long)recordPtr, objectIdMap) + { + } + + /// This ctor is used construct a transient copy of an in-memory LogRecord that remaps the object Ids in to the transient map. + /// is a pointer to transient memory that contains a copy of the in-memory allocator page's record span, including the objectIds + /// in Key and Value data. This is used for iteration. Note that the objects are not removed from the allocator-page map, so for iteration they may temporarily be in both. + /// + /// This is ONLY to be done for transient log records, not records on the main log. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static LogRecord CreateRemappedOverPinnedTransientMemory(long physicalAddress, ObjectIdMap allocatorMap, ObjectIdMap transientMap) + { + var logRecord = new LogRecord(physicalAddress, transientMap); + logRecord.RemapOverPinnedTransientMemory(allocatorMap, transientMap); + return logRecord; + } + + /// Remaps the object Ids to the transient map. + /// This is ONLY to be done for transient log records, not records on the main log. + public readonly void RemapOverPinnedTransientMemory(ObjectIdMap allocatorMap, ObjectIdMap transientMap) + { + if (ReferenceEquals(allocatorMap, transientMap)) + return; + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + + if (Info.KeyIsOverflow) + { + var (length, dataAddress) = dataHeader.GetKeyFieldInfo(); + var overflow = allocatorMap.GetOverflowByteArray(*(int*)dataAddress); + *(int*)dataAddress = transientMap.AllocateAndSet(overflow); + } + + if (Info.ValueIsOverflow) + { + var (length, dataAddress) = dataHeader.GetValueFieldInfo(Info); + var overflow = allocatorMap.GetOverflowByteArray(*(int*)dataAddress); + *(int*)dataAddress = transientMap.AllocateAndSet(overflow); + } + else if (Info.ValueIsObject) + { + var (length, dataAddress) = dataHeader.GetValueFieldInfo(Info); + var heapObj = allocatorMap.GetHeapObject(*(int*)dataAddress); + *(int*)dataAddress = transientMap.AllocateAndSet(heapObj); + } + } + + #region ISourceLogRecord + /// + public readonly byte RecordType => *(byte*)RecordTypeAddress; + + /// + public readonly ReadOnlySpan Namespace + { + get + { + var indicator = *(byte*)NamespaceAddress; + if ((indicator & RecordDataHeader.ExtendedNamespaceIndicatorBit) == 0) + { + // Single-byte namespace + return new ReadOnlySpan(ref *(byte*)NamespaceAddress); + } + else + { + // Extended namespace + // var length = indicator & RecordDataHeader.NamespaceIndicatorMask; + // return new ReadOnlySpan((byte*)(ExtendedNamespaceAddress + 1), length); + ThrowTsavoriteException("Extended namespace not yet supported"); + return default; + } + } + } + + /// + public readonly ObjectIdMap ObjectIdMap => objectIdMap; + + /// + public readonly bool IsSet => physicalAddress != 0; + + /// + public readonly ref RecordInfo InfoRef => ref *(RecordInfo*)physicalAddress; + + /// + public readonly RecordInfo Info => *(RecordInfo*)physicalAddress; + + /// + public readonly ReadOnlySpan Key + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetKeyFieldInfo(); + return Info.KeyIsInline ? new((byte*)dataAddress, length) : objectIdMap.GetOverflowByteArray(*(int*)dataAddress).ReadOnlySpan; + } + } + + /// + public readonly bool IsPinnedKey => Info.KeyIsInline; + + /// + public readonly byte* PinnedKeyPointer + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (!IsPinnedKey) + ThrowTsavoriteException("PinnedKeyPointer is unavailable when Key is not pinned; use IsPinnedKey"); + (_ /*length*/, var dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetKeyFieldInfo(); + return (byte*)dataAddress; + } + } + + /// Get and set the if this Key is not pinned; an exception is thrown if it is a pinned pointer (e.g. to a . + public readonly OverflowByteArray KeyOverflow + { + get + { + if (Info.KeyIsInline) + ThrowTsavoriteException("get_Overflow is unavailable when Key is inline"); + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetKeyFieldInfo(); + return objectIdMap.GetOverflowByteArray(*(int*)dataAddress); + } + set + { + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetKeyFieldInfo(); + if (!Info.KeyIsOverflow || length != ObjectIdMap.ObjectIdSize) + ThrowTsavoriteException("set_KeyOverflow should only be called when transferring into a new record with KeyIsInline==false and key.Length==ObjectIdSize"); + *(int*)dataAddress = objectIdMap.AllocateAndSet(value); + } + } + + /// + public readonly Span ValueSpan + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (Info.ValueIsObject) + ThrowTsavoriteException("ValueSpan is not valid for Object values"); + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + return Info.ValueIsInline ? new((byte*)dataAddress, (int)length) : objectIdMap.GetOverflowByteArray(*(int*)dataAddress).Span; + } + } + + /// + public readonly IHeapObject ValueObject + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (Info.ValueIsObject) + { + var (_ /*valueLength*/, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + return objectIdMap.GetHeapObject(*(int*)valueAddress); + } + ThrowTsavoriteException("ValueObject is not valid for Span values"); + return default; + } + internal set + { + if (Info.ValueIsObject) + { + var (valueLength, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + Debug.Assert(valueLength == ObjectIdMap.ObjectIdSize, $"valueLength {valueLength} should be ObjectIdSize {ObjectIdMap.ObjectIdSize}"); + *(int*)valueAddress = objectIdMap.AllocateAndSet(value); + + // Clear the object log file position. + *(ulong*)GetObjectLogPositionAddress(GetOptionalStartAddress()) = ObjectLogFilePositionInfo.NotSet; + return; + } + ThrowTsavoriteException("SetValueObject should only be called by DiskLogRecord or Deserialization with ValueIsObject==true"); + } + } + + /// Whether the value in this record is a valid IHeapObject; an exception is thrown if it is a Span, either inline or overflow byte[]. + public readonly bool ValueObjectIsSet + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (Info.ValueIsObject) + { + return *(int*)new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info).valueAddress != ObjectIdMap.InvalidObjectId; + } + ThrowTsavoriteException("ValueObjectIsSet is not valid for Span values"); + return default; + } + } + + /// + /// We track the deserialized length of an object value in the ObjectLogPosition field after deserialization is complete. This allows + /// flushes during recovery to both avoid re-serializing the object and know how to reset the ObjectLogPosition. + /// + /// The deserialized object + /// The deserialized length of the object + internal readonly void SetDeserializedValueObject(IHeapObject heapObject, ulong deserializedLength) + { + var (valueLength, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + + if (!Info.ValueIsObject) + ThrowTsavoriteException("SetDeserializedValueObject should only be called by Deserialization with ValueIsObject==true"); + Debug.Assert(valueLength == ObjectIdMap.ObjectIdSize, $"valueLength {valueLength} should be ObjectIdSize {ObjectIdMap.ObjectIdSize}"); + + *(int*)valueAddress = objectIdMap.AllocateAndSet(heapObject); + + // Adding valueAddress and length is the same as GetOptionalStartAddress() but faster + var objectLogPositionPtr = (ulong*)GetObjectLogPositionAddress(valueAddress + valueLength); + *objectLogPositionPtr = deserializedLength; + } + + /// The span of the entire record, including the ObjectId space if the record has objects. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ReadOnlySpan AsReadOnlySpan() => new((byte*)physicalAddress, ActualSize); + + /// + public readonly bool IsPinnedValue => Info.ValueIsInline; + + /// + public readonly byte* PinnedValuePointer + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (!IsPinnedValue) + ThrowTsavoriteException("PinnedValuePointer is unavailable when Key is not pinned; use IsPinnedValue"); + return (byte*)new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info).valueAddress; + } + } + + /// + /// Return the pinned value address and length, or throw if the value is not pinned + /// + public readonly (long address, int length) PinnedValueAddressAndLength + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (!IsPinnedValue) + ThrowTsavoriteException("PinnedValuePointer is unavailable when Key is not pinned; use IsPinnedValue"); + var (length, address) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + return (address, (int)length); + } + } + + /// Get and set the if this Value is not pinned; an exception is thrown if it is a pinned pointer (e.g. to a . + public readonly OverflowByteArray ValueOverflow + { + get + { + if (!Info.ValueIsOverflow) + ThrowTsavoriteException("get_Overflow is unavailable when Value is not overflow"); + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + return objectIdMap.GetOverflowByteArray(*(int*)dataAddress); + } + set + { + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + if (!Info.ValueIsOverflow || length != ObjectIdMap.ObjectIdSize) + ThrowTsavoriteException("set_ValueOverflow should only be called when trnasferring into a new record with ValueIsOverflow == true and value.Length==ObjectIdSize"); + *(int*)dataAddress = objectIdMap.AllocateAndSet(value); + } + } + + /// + public readonly SpanByteAndMemory ValueSpanByteAndMemory + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (Info.ValueIsObject) + ThrowTsavoriteException("ValueSpanByteAndMemory is not valid for Object values"); + var (length, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + if (Info.ValueIsInline) + return SpanByteAndMemory.FromPinnedPointer((byte*)dataAddress, (int)length); + var overflow = objectIdMap.GetOverflowByteArray(*(int*)dataAddress); + return new SpanByteAndMemory(new BorrowedMemoryOwner(overflow.AsMemory()), overflow.Length); + } + } + + /// + public readonly long ETag => Info.HasETag ? *(long*)GetETagAddress(GetOptionalStartAddress()) : NoETag; + /// + public readonly long Expiration => Info.HasExpiration ? *(long*)GetExpirationAddress(GetETagAddress(GetOptionalStartAddress())) : 0; + + /// + public readonly bool IsMemoryLogRecord => true; + + /// + public readonly ref LogRecord AsMemoryLogRecordRef() => ref Unsafe.AsRef(in this); + + /// + public readonly bool IsDiskLogRecord => false; + + /// + public readonly ref DiskLogRecord AsDiskLogRecordRef() => throw new InvalidOperationException("Cannot cast a memory LogRecord to a DiskLogRecord."); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordFieldInfo GetRecordFieldInfo() + { + var (keyLength, valueLength) = new RecordDataHeader((byte*)DataHeaderAddress).GetKVLengths(Info); + return new() + { + KeySize = keyLength, + ValueSize = valueLength, + ValueIsObject = Info.ValueIsObject, + HasETag = Info.HasETag, + HasExpiration = Info.HasExpiration + }; + } + + /// + public readonly int AllocatedSize => Info.IsNull ? RecordInfo.Size : new RecordDataHeader((byte*)DataHeaderAddress).GetAllocatedRecordSize(); + + public readonly int ActualSize => Info.IsNull ? RecordInfo.Size : new RecordDataHeader((byte*)DataHeaderAddress).GetActualRecordSize(Info); + + public static int GetAllocatedSize(long physicalAddress) + { + // Ensure this isn't called accidentally on a null record; it is used by revivification so that should never happen. + Debug.Assert(!(*(RecordInfo*)physicalAddress).IsNull, "GetAllocatedSize should not be called on a null RecordInfo"); + return new RecordDataHeader((byte*)(physicalAddress + RecordInfo.Size)).GetAllocatedRecordSize(); + } + + #endregion // ISourceLogRecord + + internal readonly void SetRecordAndFillerLength(int recordLength, int newFillerLen) + { + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + dataHeader.SetRecordLength(recordLength); + dataHeader.SetFillerLength(ref InfoRef, recordLength, newFillerLen); + } + + /// + /// Initialize record for --includes Overflow option for Key and Overflow and Object option for Value + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void InitializeRecord(TKey key, in RecordSizeInfo sizeInfo, ObjectIdMap objectIdMap) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + var header = new RecordDataHeader((byte*)DataHeaderAddress); + _ = header.Initialize(ref InfoRef, in sizeInfo, out var keyAddress, out var namespaceAddress, out var valueAddress); // TODO: Pass in (possibly) namespace span + + // Note: We do not set ETag and Expiration here, as that may confuse ISessionFunctions into thinking those values have actually been set. + // This is deferred to TrySetContentLengths, which should be first in the chain of calls that includes TrySetETag and/or TrySetExpiration. + + // Serialize Key + if (sizeInfo.KeyIsInline) + { + InfoRef.SetKeyIsInline(); + key.KeyBytes.CopyTo(new Span((byte*)keyAddress, sizeInfo.InlineKeySize)); + } + else + { + InfoRef.SetKeyIsOverflow(); + var overflow = new OverflowByteArray(key.KeyBytes.Length, startOffset: 0, endOffset: 0, zeroInit: false); + key.KeyBytes.CopyTo(overflow.Span); + + // This is record initialization so no object has been allocated for this field yet. + var objectId = objectIdMap.Allocate(); + *(int*)keyAddress = objectId; + objectIdMap.Set(objectId, overflow); + } + + // Serialize namespace, if any + // + // Since TKey is generic, the hope is this whole branch gets elided when using a no-namespace key type + if (key.HasNamespace) + { + var namespaceBytes = key.NamespaceBytes; + Debug.Assert(namespaceBytes.Length == 1, "Should have exactly 1 namespace byte, variable length is not implemented"); + namespaceBytes.CopyTo(new Span((byte*)namespaceAddress, namespaceBytes.Length)); + } + + // Initialize Value metadata (but we don't have the value here to set yet; that's done in ISessionFunctions). + if (sizeInfo.ValueIsInline) + InfoRef.SetValueIsInline(); + else + { + if (!sizeInfo.ValueIsObject) + { + // We must have the space allocated for Overflow just like we do for inline, so we set the Overflow allocation and objectId here. + // We have no value data to copy yet. + InfoRef.SetValueIsOverflow(); + var overflow = new OverflowByteArray(sizeInfo.FieldInfo.ValueSize, startOffset: 0, endOffset: 0, zeroInit: false); + + // This is record initialization so no object has been allocated for this field yet. + var objectId = objectIdMap.Allocate(); + *(int*)valueAddress = objectId; + objectIdMap.Set(objectId, overflow); + } + else + { + Debug.Assert(sizeInfo.FieldInfo.ValueSize == ObjectIdMap.ObjectIdSize, $"Expected object size ({ObjectIdMap.ObjectIdSize}) for Object ValueSize but was {sizeInfo.FieldInfo.ValueSize}"); + + // Unlike for Keys and Overflow values, we do not set the objectId here; we wait for the UMD operation to do that. + *(int*)valueAddress = ObjectIdMap.InvalidObjectId; + InfoRef.SetValueIsObject(); + } + } + } + + /// + /// Initialize record for --does not include Overflow/Object options so is streamlined + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void InitializeRecord(TKey key, in RecordSizeInfo sizeInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + var header = new RecordDataHeader((byte*)DataHeaderAddress); + _ = header.Initialize(ref InfoRef, in sizeInfo, out var keyAddress, out var namespaceAddress, out _ /*valueAddress*/); + + InfoRef.SetKeyAndValueInline(); + + // Serialize Key. Do nothing for the value; we've set it inline and the actual value setting is done in ISessionFunctions). + key.KeyBytes.CopyTo(new Span((byte*)keyAddress, sizeInfo.InlineKeySize)); + + // Serialize namespace, if any + // + // Since TKey is generic, the hope is this whole branch gets elided when using a no-namespace key type + if (key.HasNamespace) + { + var namespaceBytes = key.NamespaceBytes; + Debug.Assert(namespaceBytes.Length == 1, "Should have exactly 1 namespace byte, variable length is not implemented"); + namespaceBytes.CopyTo(new Span((byte*)namespaceAddress, namespaceBytes.Length)); + } + } + + /// A ref to the record header + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ref RecordInfo GetInfoRef(long physicalAddress) => ref *(RecordInfo*)physicalAddress; + + /// Fast access returning a copy of the record header + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static RecordInfo GetInfo(long physicalAddress) => *(RecordInfo*)physicalAddress; + + internal static ReadOnlySpan GetInlineKey(long physicalAddress) + { + Debug.Assert((*(RecordInfo*)physicalAddress).KeyIsInline, "Key must be inline"); + var (length, dataAddress) = new RecordDataHeader((byte*)(physicalAddress + RecordInfo.Size)).GetKeyFieldInfo(); + return new((byte*)dataAddress, length); + } + + /// Get the span of the inline portion of the record. Following this, the caller should be sure the objectIds are remapped + /// to a transient ObjectIdMap if necessary. + public readonly Span RecordSpan => new((byte*)physicalAddress, ActualSize); + + /// + /// Tries to set the length of the value field, as well as verifying there is also space for the optionals (ETag, Expiration, ObjectLogPosition) as + /// specified by , shifting the optional positions if necessary, and setting or clearing the appropriate optional RecordInfo flags. + /// Asserts that is the same size as the value data size in the before setting the length. + /// + /// If successful, returns true and the caller can proceed to set the value data. + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetContentLengths(int newValueSize, in RecordSizeInfo sizeInfo, bool zeroInit = false) + { + Debug.Assert(newValueSize == sizeInfo.FieldInfo.ValueSize, $"Mismatched value size; expected {sizeInfo.FieldInfo.ValueSize}, actual {newValueSize}"); + return TrySetContentLengthsAndPrepareOptionals(in sizeInfo, zeroInit, out _ /*valueAddress*/); + } + + /// + /// Tries to set the length of the value field, as well as verifying there is also space for the optionals (ETag, Expiration, ObjectLogPosition) as + /// specified by , shifting the optional positions if necessary, and setting or clearing the appropriate optional RecordInfo flags. + /// + /// If successful, returns true and the caller can proceed to set the value data. + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetContentLengths(in RecordSizeInfo sizeInfo, bool zeroInit = false) => TrySetContentLengthsAndPrepareOptionals(in sizeInfo, zeroInit, out _ /*valueAddress*/); + + /// + /// Tries to set the length of the value field, as well as verifying there is also space for the optionals (ETag, Expiration, ObjectLogPosition) as + /// specified by , shifting the optional positions if necessary, and setting or clearing the appropriate optional RecordInfo flags. + /// + /// If successful, returns true and the caller can proceed to set the value data. + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + private readonly bool TrySetContentLengthsAndPrepareOptionals(in RecordSizeInfo sizeInfo, bool zeroInit, out long valueAddress) + { + // Get the number of bytes in existing key and value lengths. + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (_ /*keyLength*/, oldInlineValueSize) = dataHeader.GetKVLengths(Info, out var recordLength, out var oldETagLen, out var oldExpirationLen, out var oldObjectLogPositionLen, out var oldFillerLen, out valueAddress); + var oldOptionalSize = oldETagLen + oldExpirationLen + oldObjectLogPositionLen; + + // Key does not change, so its size and size byte count remain the same. valueAddress does not change either, as everything before it is immutable. + // optionalStartAddress will change if inline value size changes. + var optionalStartAddress = valueAddress + oldInlineValueSize; + + // It is OK if the record is shrinking but we cannot grow past the old RecordLength. (If we are converting from inline to overflow that will + // already be accounted for because sizeInfo will be set for the ObjectId length.) + if (sizeInfo.ActualInlineRecordSize > recordLength) + return false; + + // We don't need to change the size if values are both inline and their size hasn't changed and the optional specs are the same, + // we can exit early with success. + var newInlineValueSize = sizeInfo.InlineValueSize; + var inlineValueGrowth = newInlineValueSize - oldInlineValueSize; + if (Info.RecordIsInline && sizeInfo.RecordIsInline && inlineValueGrowth == 0 + && Info.HasETag == sizeInfo.FieldInfo.HasETag && Info.HasExpiration == sizeInfo.FieldInfo.HasExpiration) + return true; + + // inlineValueGrowth and fillerLen may be negative if shrinking value or converting to Overflow/Object. + // ETag and Expiration won't change, but optionalGrowth may be positive or negative if adding or removing ObjectLogPosition. + var optionalGrowth = sizeInfo.OptionalSize - oldOptionalSize; + + // See if we have enough room for the change in Value inline data. Note: This includes things like moving inline data that is less than + // overflow length into overflow, which frees up inline space > ObjectIdMap.ObjectIsSize. We calculate the inline size required for the + // new value (including whether it is overflow) and the existing optionals, and success is based on whether that can fit into the allocated + // record space. We also change the presence of optionals here, shift their positions, and adjust their RecordInfo flags as needed. + // Subsequent operations must assign new ETag or Expiration if the flag was set in sizeInfo. + if (oldFillerLen < inlineValueGrowth + optionalGrowth) // Optional growth here includes ObjLogPositionSize changes + return false; + + // Update record part 1: Save the optionals if shifting is needed. We can't just shift now because we may be e.g. converting from inline to + // overflow and they'd overwrite needed data. + var optionalFields = new OptionalFieldsShift(); + optionalFields.Save(optionalStartAddress, Info); + + // Update record part 2: Do any necessary conversions between Inline, Overflow, and Object. This may allocate or free Heap Objects. + // Evaluate in order of most common (i.e. most perf-critical) cases first. + if (Info.ValueIsInline && sizeInfo.ValueIsInline) + { + // Both are inline, so nothing to do here; we will adjust the lengths below. + } + else if (Info.ValueIsOverflow && sizeInfo.ValueIsOverflow) + { + // Both are out-of-line, so reallocate in place if needed; the caller will operate on that space after we return. + _ = LogField.ReallocateValueOverflow(physicalAddress, valueAddress, in sizeInfo, objectIdMap); + } + else if (Info.ValueIsObject && sizeInfo.ValueIsObject) + { + // Both are object records, so nothing to change; the caller will operate on the object after we return. + } + else + { + // Overflow/Object-ness differs and we've verified there is enough space for the change, so convert. The LogField.ConvertTo* functions copy + // existing data, as we are likely here for IPU or for the initial update going from inline to overflow with Value length == sizeof(IntPtr). + if (Info.ValueIsInline) + { + if (sizeInfo.ValueIsOverflow) + { + Debug.Assert(inlineValueGrowth == ObjectIdMap.ObjectIdSize - oldInlineValueSize, + $"ValueGrowth {inlineValueGrowth} does not equal expected {oldInlineValueSize - ObjectIdMap.ObjectIdSize}"); + _ = LogField.ConvertInlineToOverflow(ref InfoRef, physicalAddress, valueAddress, oldInlineValueSize, in sizeInfo, objectIdMap); + } + else + { + Debug.Assert(sizeInfo.ValueIsObject, "Expected ValueIsObject to be set, pt 1"); + _ = LogField.ConvertInlineToValueObject(ref InfoRef, physicalAddress, valueAddress, in sizeInfo, objectIdMap); + } + } + else if (Info.ValueIsOverflow) + { + if (sizeInfo.ValueIsInline) + _ = LogField.ConvertOverflowToInline(ref InfoRef, physicalAddress, valueAddress, in sizeInfo, objectIdMap); + else + { + Debug.Assert(sizeInfo.ValueIsObject, "Expected ValueIsObject to be set, pt 2"); + _ = LogField.ConvertOverflowToValueObject(ref InfoRef, physicalAddress, valueAddress, in sizeInfo, objectIdMap); + } + } + else + { + Debug.Assert(Info.ValueIsObject, "Expected ValueIsObject to be set, pt 3"); + + if (sizeInfo.ValueIsInline) + _ = LogField.ConvertValueObjectToInline(ref InfoRef, physicalAddress, valueAddress, in sizeInfo, objectIdMap); + else + { + Debug.Assert(sizeInfo.ValueIsOverflow, "Expected ValueIsOverflow to be true"); + _ = LogField.ConvertValueObjectToOverflow(ref InfoRef, physicalAddress, valueAddress, in sizeInfo, objectIdMap); + } + } + } + + // Update record part 3: Restore optionals to their new location. If we have some optionals in sizeInfo that weren't in the record previously, they'll get + // their default values; subsequently, the caller should set them to the actual values. We have to do this even if not sizeInfo.HasOptionalFields because + // this also sets or clears optional flags. + optionalStartAddress += inlineValueGrowth; + optionalFields.Restore(optionalStartAddress, in sizeInfo, ref InfoRef); + + // Update record part 4: Update Filler length in the record. Optional data size for ETag/Expiration is unchanged even if newOptionalSize != oldOptionalSize, + // because we are not updating those optionals here, so don't adjust fillerLen for that. However, a change in the presence or absence of the pseudo-optional + // ObjectLogPosition must be accounted for if we have changed whether the record is inline or has objects. Note that we don't have a valueLength to update; + // it is a calculated value, which depends (in part) upon FillerLength. + var newFillerLen = oldFillerLen - inlineValueGrowth - optionalGrowth; + if (newFillerLen != oldFillerLen) + dataHeader.SetFillerLength(ref InfoRef, recordLength, newFillerLen > 0 ? newFillerLen : 0); + if (zeroInit && inlineValueGrowth > 0) + { + // Zeroinit any extra space we grew the value by. For example, if we grew by one byte we might have a stale fillerLength in that byte. + new Span((byte*)(valueAddress + oldInlineValueSize), newInlineValueSize - oldInlineValueSize).Clear(); + } + + Debug.Assert(Info.ValueIsInline == sizeInfo.ValueIsInline, "Final ValueIsInline is inconsistent"); + Debug.Assert(!Info.ValueIsInline || ValueSpan.Length <= sizeInfo.MaxInlineValueSize, $"Inline ValueSpan.Length {ValueSpan.Length} is greater than sizeInfo.MaxInlineValueSpanSize {sizeInfo.MaxInlineValueSize}"); + return true; + } + + /// + /// Tries to set the length of the value field, including shifting optionals as needed. Does NOT change the presence of optionals, + /// and only works on Inline values. Used for in-place updates and preceded by calling + /// which is usually necessary to evaluate the current value data, e.g. for INCRBY. + /// + /// The new size of the value. + /// The address of the value, obtained from + /// The current length of the value; on input obtained from ; set on output to newValueSize + /// If true, set any value space "exposed" by increasing + /// If successful, returns true and the caller can proceed to set the value data. + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + public readonly bool TrySetPinnedValueLength(in int newValueSize, long valueAddress, ref int valueLength, bool zeroInit = false) + { + if (!Info.ValueIsInline) + { + Debug.Fail($"{nameof(TrySetPinnedValueLength)} should only be called when Value is known to be inline, such as INCRBY"); + return false; + } + + // If we're not changing value size, there's nothing to do. + var inlineValueGrowth = newValueSize - valueLength; + if (inlineValueGrowth == 0) + return true; + + // Get the number of bytes in existing key and value lengths. + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + int oldFillerLen, recordLength; + if (!Info.HasOptionalOrObjectFields) + { + oldFillerLen = dataHeader.GetFillerLength(Info, out recordLength); + if (oldFillerLen < inlineValueGrowth) + return false; + } + else + { + _ = dataHeader.GetKVLengths(Info, out recordLength, out var oldETagLen, out var oldExpirationLen, out var oldObjectLogPositionLen, out oldFillerLen, out _ /*valueAddress*/); + if (oldFillerLen < inlineValueGrowth) + return false; + + // Shift optionals if needed. We include space for the ObjectLogPosition here even though it's the last field and not used until we + // serialize the record, because otherwise we may not have enough bytes to write the full FillerLength. + var oldOptionalSize = oldETagLen + oldExpirationLen + oldObjectLogPositionLen; + var optionalStartAddress = valueAddress + valueLength; + if (oldOptionalSize != 0) + Buffer.MemoryCopy((void*)optionalStartAddress, (void*)(optionalStartAddress + inlineValueGrowth), oldOptionalSize, oldOptionalSize); + } + + // Zeroinit any extra space we grew the value by. For example, if we grew by one byte we might have a stale fillerLength in that byte. + if (zeroInit && inlineValueGrowth > 0) + new Span((byte*)(valueAddress + valueLength), inlineValueGrowth).Clear(); + + // Update FillerLength. Note that we don't have a valueLength to update; it is a calculated value, which depends (in part) upon FillerLength. + dataHeader.SetFillerLength(ref InfoRef, recordLength, oldFillerLen - inlineValueGrowth); + + // Key does not change, so its size and size byte count remain the same. valueAddress does not change either, as everything before it is immutable. + // So the only things that change are FillerLength and ValueLength. + valueLength += inlineValueGrowth; + return true; + } + + /// + /// Tries to set the length of the value field, including shifting optionals as needed. Does NOT change the presence of optionals, + /// and only works on Inline values. Used for in-place updates and preceded by calling + /// which is usually necessary to evaluate the current value data, e.g. for INCRBY. + /// + /// The new value to set into the record. + /// The address of the value, obtained from + /// The current length of the value; on input obtained from ; set on output to newValueSize + /// If true, set any value space "exposed" by increasing + /// If successful, returns true and the caller can proceed to set the value data. + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetPinnedValueSpan(ReadOnlySpan newValue, long valueAddress, ref int valueLength, bool zeroInit = false) + { + if (!TrySetPinnedValueLength(newValue.Length, valueAddress, ref valueLength, zeroInit)) + return false; + newValue.CopyTo(new Span((byte*)valueAddress, newValue.Length)); + return true; + } + + /// + /// Set the value span, checking for conversion to/from inline as well as verifying there is also space for the optionals (ETag, Expiration, ObjectLogPosition) as + /// specified by , shifting the optional positions if necessary, and setting or clearing the appropriate optional RecordInfo flags. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetValueSpanAndPrepareOptionals(ReadOnlySpan value, in RecordSizeInfo sizeInfo, bool zeroInit = false) + { + RecordSizeInfo.AssertValueDataLength(value.Length, in sizeInfo); + if (!TrySetContentLengthsAndPrepareOptionals(in sizeInfo, zeroInit, out var valueAddress)) + return false; + + var valueSpan = sizeInfo.ValueIsInline ? new((byte*)valueAddress, sizeInfo.FieldInfo.ValueSize) : objectIdMap.GetOverflowByteArray(*(int*)valueAddress).Span; + value.CopyTo(valueSpan); + return true; + } + + internal readonly bool TryReinitializeValueLength(in RecordSizeInfo sizeInfo) + { + // This is called when reinitializing a record for InitialUpdater or InitialWriter; we don't want to them to see initial state with optionals set. + // Because it is for (re)initialization, we don't zero-initialize; the caller should assume they have to do that if they only copy partial data in. + ClearOptionals(); + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (_ /*valueLength*/, valueAddress) = dataHeader.GetValueFieldInfo(Info); + var recordLength = dataHeader.GetRecordLength(); + var fillerLength = (int)(physicalAddress + recordLength - (valueAddress + sizeInfo.InlineValueSize)); + if (fillerLength < 0) + return false; + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLength); + return true; + } + + /// + /// Set the value span, checking for conversion to/from inline as well as verifying there is also space for the optionals (ETag, Expiration, ObjectLogPosition) as + /// specified by , shifting the optional positions if necessary, and setting or clearing the appropriate optional RecordInfo flags. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetValueObjectAndPrepareOptionals(IHeapObject value, in RecordSizeInfo sizeInfo) => TrySetContentLengths(in sizeInfo) && TrySetValueObject(value); + + /// + /// This overload must be called only when it is known the LogRecord's Value is not inline, and there is no need to check + /// optionals (ETag or Expiration). In that case it is faster to just set the object. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetValueObject(IHeapObject value) + { + Debug.Assert(Info.ValueIsObject, $"Cannot call this overload of {GetCurrentMethodName()} for non-object Value"); + + if (!Info.ValueIsObject) + { + Debug.Fail($"Cannot call {GetCurrentMethodName()} with no {nameof(RecordSizeInfo)} when !ValueIsObject"); + return false; + } + + var (valueLength, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + + // If there is no object there yet, allocate a slot + var objectId = *(int*)valueAddress; + if (objectId == ObjectIdMap.InvalidObjectId) + objectId = *(int*)valueAddress = objectIdMap.Allocate(); + + // Set the new object into the slot + objectIdMap.Set(objectId, value); + return true; + } + + public readonly int ETagLen => Info.HasETag ? ETagSize : 0; + public readonly int ExpirationLen => Info.HasExpiration ? ExpirationSize : 0; + internal readonly int ObjectLogPositionLen => Info.RecordHasObjects ? ObjectLogPositionSize : 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly long GetOptionalStartAddress() + { + var (valueLength, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info, out _ /*keyLength*/, out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + return valueAddress + valueLength; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly ReadOnlySpan GetOptionalFieldsSpan() => new((byte*)GetOptionalStartAddress(), OptionalLength); + + public readonly int OptionalLength => ETagLen + ExpirationLen + ObjectLogPositionLen; + + #region IKey + /// + public readonly bool IsPinned => IsPinnedKey; + + /// + public readonly ReadOnlySpan KeyBytes => Key; + + /// + public readonly bool HasNamespace + { + get + { + // A 1-byte 0 values namespace is the "default" and should be ignored. + // Any non-zero value (including the ExtendedNamespaceIndicatorBit being set) means we have a namespace. + var indicator = *(byte*)NamespaceAddress; + return indicator != 0; + } + } + + /// + public readonly ReadOnlySpan NamespaceBytes + { + get + { + Debug.Assert(HasNamespace, "Shouldn't call if !HasNamespace"); + return Namespace; + } + } + #endregion + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long GetETagAddress(long optionalStartAddress) => optionalStartAddress; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly long GetExpirationAddress(long optionalStartAddress) => optionalStartAddress + ETagLen; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly long GetObjectLogPositionAddress(long optionalStartAddress) => optionalStartAddress + ETagLen + ExpirationLen; + + /// + /// Called during cleanup of a record allocation, before the key was copied. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly void InitializeForReuse(in RecordSizeInfo sizeInfo) + { + Debug.Assert(!Info.HasETag && !Info.HasExpiration, "Record should not have ETag or Expiration here"); + + // This is called after the record has been allocated, so it's at the tail (or very close to it), and before it is returned to the TryBlockAllocate + // caller. So all we need to do is initialize it to a consistent RecordLength state. We could make this a little leaner for this case but this is + // called only on recovery from a failed TryAllocate (e.g. HeadAddress moved up so we couldn't complete the allocation), so it's not perf-critical. + InfoRef = RecordInfo.InitialValid; + _ = new RecordDataHeader((byte*)DataHeaderAddress).Initialize(ref InfoRef, in sizeInfo, out _ /*keyAddress*/, out _ /*namespaceAddress*/, out _ /*valueAddress*/); + } + + /// + /// Set the ETag, checking for space for optionals. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetETag(long eTag) + { + var optionalStartAddress = GetOptionalStartAddress(); + if (Info.HasETag) + { + *(long*)GetETagAddress(optionalStartAddress) = eTag; + return true; + } + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var recordLength = dataHeader.GetRecordLength(); + + // We're adding an ETag where there wasn't one before. + var fillerLen = dataHeader.GetFillerLength(Info, recordLength); + // We'll keep the original FillerLen address and back up, for speed. + var address = physicalAddress + recordLength - fillerLen; + fillerLen -= ETagSize; + if (fillerLen < 0) + return false; + + // We don't preserve the ObjectLogPosition field; that's only for serialization. + if (Info.RecordHasObjects) + address -= ObjectLogPositionSize; + + // Preserve Expiration if present; set ETag; re-enter Expiration if present + var expiration = 0L; + if (Info.HasExpiration) + { + address -= ExpirationSize; + expiration = *(long*)address; + } + + // Set the eTag + *(long*)address = eTag; + InfoRef.SetHasETag(); + address += ETagSize; + + // Restore expiration, if any + if (Info.HasExpiration) + { + *(long*)address = expiration; // will be 0 or a valid expiration + address += ExpirationSize; // repositions to ObjectLogPosition address + } + + // ObjectLogPosition is not preserved (it's only for serialization) so set it to NotSet. + if (Info.RecordHasObjects) + *(ulong*)address = ObjectLogFilePositionInfo.NotSet; + + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLen); + return true; + } + + /// + /// Remove the ETag. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool RemoveETag() + { + if (!Info.HasETag) + return true; + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var recordLength = dataHeader.GetRecordLength(); + + // We're adding an ETag where there wasn't one before. + var fillerLen = dataHeader.GetFillerLength(Info, recordLength); + // We'll keep the original FillerLen address and back up, for speed. + var address = physicalAddress + recordLength - fillerLen; + fillerLen += ETagSize; + + // We don't preserve the ObjectLogPosition field; that's only for serialization. Just set it to 0 here. + if (Info.RecordHasObjects) + { + address -= ObjectLogPositionSize; + *(ulong*)address = 0; + } + + // Move Expiration, if present, up to cover ETag; then clear the ETag bit + var expiration = 0L; + var expirationSize = 0; + if (Info.HasExpiration) + { + expirationSize = ExpirationSize; + address -= expirationSize; + expiration = *(long*)address; + *(long*)address = 0L; // To ensure zero-init + } + + // Expiration will be either zero or a valid expiration, and we have not changed the info.HasExpiration flag + address -= ETagSize; + *(long*)address = expiration; // will be 0 or a valid expiration + address += expirationSize; // repositions to fillerAddress if expirationSize is nonzero + InfoRef.ClearHasETag(); + + // ObjectLogPosition is not preserved (it's only for serialization) but we set it to NotSet. + if (Info.RecordHasObjects) + *(ulong*)address = ObjectLogFilePositionInfo.NotSet; + + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLen); + return true; + } + + /// + /// Set the Expiration, checking for space for optionals. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TrySetExpiration(long expiration) + { + if (expiration == NoExpiration) + return RemoveExpiration(); + + var optionalStartAddress = GetOptionalStartAddress(); + if (Info.HasExpiration) + { + *(long*)GetExpirationAddress(optionalStartAddress) = expiration; + return true; + } + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var recordLength = dataHeader.GetRecordLength(); + + // We're adding an Expiration where there wasn't one before. + var fillerLen = dataHeader.GetFillerLength(Info, recordLength); + // We'll keep the original FillerLen address and back up, for speed. + var address = physicalAddress + recordLength - fillerLen; + fillerLen -= ExpirationSize; + if (fillerLen < 0) + return false; + + // We don't preserve the ObjectLogPosition field; that's only for serialization. + if (Info.RecordHasObjects) + address -= ObjectLogPositionSize; + + // Set the Expiration + InfoRef.SetHasExpiration(); + *(long*)address = expiration; + address += ExpirationSize; + + // ObjectLogPosition is not preserved (it's only for serialization) but we set it to NotSet. + if (Info.RecordHasObjects) + *(ulong*)address = ObjectLogFilePositionInfo.NotSet; + + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLen); + return true; + } + + /// + /// Remove the expiration + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool RemoveExpiration() + { + if (!Info.HasExpiration) + return true; + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var recordLength = dataHeader.GetRecordLength(); + + // We're adding an ETag where there wasn't one before. + var fillerLen = dataHeader.GetFillerLength(Info, recordLength); + // We'll keep the original FillerLen address and back up, for speed. + var address = physicalAddress + recordLength - fillerLen; + fillerLen += ExpirationSize; + + // We don't preserve the ObjectLogPosition field; that's only for serialization. Just set it to 0 here. + if (Info.RecordHasObjects) + { + address -= ObjectLogPositionSize; + *(ulong*)address = 0; + } + + // Remove Expiration and clear the Expiration bit; this will be the new fillerLenAddress + address -= ExpirationSize; + *(long*)address = 0; + InfoRef.ClearHasExpiration(); + + // ObjectLogPosition is not preserved (it's only for serialization) but we set it to NotSet. + if (Info.RecordHasObjects) + *(ulong*)address = ObjectLogFilePositionInfo.NotSet; + + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLen); + return true; + } + + /// + /// Copy the entire record values: Value and optionals (ETag, Expiration). Key is not copied as it has already been set into 'this'. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TryCopyFrom(in TSourceLogRecord srcLogRecord, in RecordSizeInfo sizeInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (srcLogRecord.Info.ValueIsInline) + { + if (!TrySetContentLengths(in sizeInfo)) + return false; + srcLogRecord.ValueSpan.CopyTo(ValueSpan); + } + else + { + if (srcLogRecord.Info.ValueIsOverflow) + { + Debug.Assert(Info.ValueIsOverflow, "Expected this.Info.ValueIsOverflow to be set already"); + ValueOverflow = srcLogRecord.ValueOverflow; + } + else + { + // TODO: Clone the value object here so source and destination have independent + // HeapMemorySize fields. Currently both records share the same IHeapObject instance, + // which means mutations on the destination affect the source's reported heap size + // at eviction time, causing accounting drift in logSizeTracker. A naive Clone() + // here causes CanDoHashExpireLTM to crash — needs investigation in a follow-up. + Debug.Assert(srcLogRecord.ValueObject is not null, "Expected srcLogRecord.ValueObject to be set (or deserialized) already"); + if (!TrySetValueObjectAndPrepareOptionals(srcLogRecord.ValueObject, in sizeInfo)) + return false; + } + } + return TryCopyOptionals(in srcLogRecord, in sizeInfo); + } + + /// + /// Check if there is sufficient space to store an ETag in the log record + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void PopulateRecordSizeInfoForIPU(ref RecordSizeInfo sizeInfo) + { + Debug.Assert(sizeInfo.word == 0, "RecordSizeInfo should not be resused"); + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (keyLength, existingValueLength) = dataHeader.GetKVLengths(Info, out _ /*recordLength*/, out var eTagLen, out var expirationLen, out var objectLogPositionLen, out var fillerLen, out _ /*valueAddress*/); + + // The sizeInfo's FieldInfo has already been populated. Key size won't change in IPU. + var keyOverflowInlineSize = 0; + if (Info.KeyIsInline) + sizeInfo.SetKeyIsInline(); + else + keyOverflowInlineSize = ObjectLogPositionSize; + + // Because this is IPU we are limited in inline value size by the record length less any optional length growth in the sizeInfo. + // We don't allow non-inline if we have a null objectIdMap. TODO: Need better awareness of actual inline value max length. + var existingOptionalSize = eTagLen + expirationLen + objectLogPositionLen; + + // sizeInfo.OptionalSize will be nonzero because we've not yet set ValueIsInline so calculate the sizeInfo OptionalSize values directly + // from its FieldInfo with keyOverflowInlineSize as a proxy for ObjectLogPosition. + sizeInfo.MaxInlineValueSize = existingValueLength + fillerLen - (sizeInfo.FieldInfo.eTagSize + sizeInfo.FieldInfo.expirationSize + keyOverflowInlineSize - existingOptionalSize); + + if (objectIdMap is null || (!sizeInfo.ValueIsObject && sizeInfo.FieldInfo.ValueSize <= sizeInfo.MaxInlineValueSize)) + sizeInfo.SetValueIsInline(); + var valueSize = sizeInfo.ValueIsInline ? sizeInfo.FieldInfo.ValueSize : ObjectIdMap.ObjectIdSize; + + // Record + sizeInfo.CalculateSizes(keyLength, valueSize); + } + + /// + /// Check if there is sufficient space to store an ETag in the log record + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly long GetValueHeapMemorySize() + { + if (Info.ValueIsInline) + return 0; + + if (Info.ValueIsObject) + return ValueObject.HeapMemorySize; + + var (_ /*length*/, dataAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + return objectIdMap.GetOverflowByteArray(*(int*)dataAddress).HeapMemorySize; + } + + /// + /// Check if there is sufficient space to grow by the additional space, both the value and whether we want to have optionals when done. + /// + /// The inline length of the new value + /// If we are going to set the ETag this is ; if we are removing the ETag this is 0; if we're not changing the ETag it's + /// If we are going to set the Expiration this is ; if we are removing the Expiration this is 0; if we're not changing the Expiration it's + /// The address of the pinned value + /// The current length of the value + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool CanGrowPinnedValue(int newValueLength, int newETagLen, int newExpirationLen, out long valueAddress, out int valueLength) + { + if (!Info.ValueIsInline) + ThrowTsavoriteException("Cannot call CanGrowInline when !ValueIsInline"); + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + (var keyLength, valueLength) = dataHeader.GetKVLengths(Info, out _ /*recordLength*/, out var eTagLen, out var expirationLen, out var objectLogPositionLen, out var fillerLen, out valueAddress); + + var growth = (newValueLength - valueLength) + (newETagLen - eTagLen) + (newExpirationLen - expirationLen); + return growth <= fillerLen; + } + + /// + /// Copy the record optional values (ETag, Expiration) + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool TryCopyOptionals(in TSourceLogRecord srcLogRecord, in RecordSizeInfo sizeInfo) + where TSourceLogRecord : ISourceLogRecord + { + var srcRecordInfo = srcLogRecord.Info; + + // If the source has optionals and the destination wants them, copy them over + if (!srcRecordInfo.HasETag || !sizeInfo.FieldInfo.HasETag) + _ = RemoveETag(); + else if (!TrySetETag(srcLogRecord.ETag)) + return false; + + if (!srcRecordInfo.HasExpiration || !sizeInfo.FieldInfo.HasExpiration) + _ = RemoveExpiration(); + else if (!TrySetExpiration(srcLogRecord.Expiration)) + return false; + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void ClearOptionals() + { + _ = RemoveExpiration(); + _ = RemoveETag(); + } + + /// + /// Clears any heap-allocated Value: Object or Overflow. Does not clear key (if it is Overflow). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void ClearValueIfHeap() + { + if (Info.ValueIsInline) + return; + + // If the key is Heap and we're not clearing it then we don't want to to change ObjectLogPosition and Filler, so just clear the value and return. + if (!Info.KeyIsInline) + { + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (valueLength, valueAddress) = dataHeader.GetValueFieldInfo(Info, out _ /*keyLength*/, out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, valueAddress, objectIdMap, isKey: false); + return; + } + + // The key is not overflow so we must remove ObjectLogPosition and update filler. + ClearHeapFields(clearKey: false); + } + + /// + /// Clears any heap-allocated field, Object or Overflow, in the Value and optionally the Key. If we go from + /// being false to true, then we need to adjust filler as well. + /// + public readonly void ClearHeapFields(bool clearKey) + { + if (Info.RecordIsInline) + return; + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var recordLength = dataHeader.GetRecordLength(); + var fillerLen = dataHeader.GetFillerLength(Info, recordLength); + + var (valueLength, valueAddress) = dataHeader.GetValueFieldInfo(Info, out var keyLength, out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + + // If the key is Heap and we're not clearing it then we don't want to to change ObjectLogPosition and Filler, so just clear the value and return. + if (!clearKey && !Info.KeyIsInline) + { + if (!Info.ValueIsInline) + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, valueAddress, objectIdMap, isKey: false); + return; + } + + // If we're here and the key is overflow we're clearing it. + if (!Info.KeyIsInline) + { + var keyAddress = valueAddress - keyLength; + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, keyAddress, objectIdMap, isKey: true); + } + if (!Info.ValueIsInline) + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, valueAddress, objectIdMap, isKey: false); + + // Now update filler to account for removal of ObjectLogPosition + dataHeader.SetFillerLength(ref InfoRef, recordLength, fillerLen + ObjectLogPositionSize); + } + + /// + /// For revivification or reuse: the record space has been retrieved from revivification or PendingContext, so prepare it to be passed to initial updaters, + /// based upon the sizeInfo's key and value lengths. + /// + /// This is 'readonly' because it does not alter the fields of this object, only what they point to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void PrepareForRevivification(ref RecordSizeInfo sizeInfo) + => RecordDataHeader.InitializeForRevivification(ref InfoRef, ref sizeInfo); + + /// + /// Sets the lengths of Overflow Keys and Values and Object values into the disk-image copy of the log record before the main-log page is flushed. + /// + /// The starting position of the serialized key and value data in the object log. + /// The serialized length of the value object if it is an object and not inline or overflow. Overflow + /// fields have their length known from the property. + /// + /// IMPORTANT: This is only to be called in the disk image copy of the log record, not in the actual log record itself. + /// + internal readonly void SetObjectLogRecordStartPositionAndLength(in ObjectLogFilePositionInfo objectLogFilePosition, ulong valueObjectLength) + { + if (Info.RecordIsInline) // ValueIsInline is true; if the record is fully inline, we should not be called here + { + Debug.Fail("Cannot call SetObjectLogRecordStartPositionAndLength for an inline record"); + return; + } + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + + if (Info.KeyIsOverflow) + { + var (keyLength, keyAddress) = dataHeader.GetKeyFieldInfo(); + var overflow = objectIdMap.GetOverflowByteArray(*(int*)keyAddress); + *(int*)keyAddress = overflow.Length; + } + + var (valueLength, valueAddress) = dataHeader.GetValueFieldInfo(Info); + + // Adding valueAddress and length is the same as GetOptionalStartAddress() but faster + var objectLogPositionPtr = (ulong*)GetObjectLogPositionAddress(valueAddress + valueLength); + *objectLogPositionPtr = objectLogFilePosition.word; + + if (Info.ValueIsOverflow) + { + var overflow = objectIdMap.GetOverflowByteArray(*(int*)valueAddress); + *(int*)valueAddress = overflow.Length; + } + else if (Info.ValueIsObject) + { + // Reuse the valueAddress space to store the low int of valueObjectLength, then store the high byte in the ObjectLogPosition + // (it is combined with the length that is stored in the ObjectId field data of the record). + *(uint*)valueAddress = (uint)(valueObjectLength & 0xFFFFFFFF); + ObjectLogFilePositionInfo.SetObjectSizeHighByte(objectLogPositionPtr, (int)(valueObjectLength >> 32)); + } + else if (Info.RecordIsInline) // ValueIsInline is true; if the record is fully inline, we should not be called here + { + Debug.Fail("Cannot call SetObjectLogRecordStartPositionAndLength for an inline record"); + return; + } + } + + /// + /// Returns the object log position for the start of the key (if any) and value (if any). + /// + /// Outputs key length; will always be for overflow + /// Outputs key length; will be for overflow or object + /// The object log position for this record + internal readonly ulong GetObjectLogRecordStartPositionAndLengths(out int keyLength, out ulong valueObjectLength) + { + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + if (Info.KeyIsOverflow) + { + (keyLength, var keyAddress) = dataHeader.GetKeyFieldInfo(); + keyLength = *(int*)keyAddress; + } + else // KeyIsInline is true; keyLength will be ignored + keyLength = 0; + + var (valueLength, valueAddress) = dataHeader.GetValueFieldInfo(Info); + if (Info.ValueIsOverflow) + valueObjectLength = (ulong)*(int*)valueAddress; + else if (Info.ValueIsObject) + { + // Get the high byte in the ObjectLogPosition (it is combined with the length that is stored in the ObjectId field data of the record). + // Adding valueAddress and length is the same as GetOptionalStartAddress() but faster + var objectLogPositionPtr = (ulong*)GetObjectLogPositionAddress(valueAddress + valueLength); + valueObjectLength = *(uint*)valueAddress | ((ulong)ObjectLogFilePositionInfo.GetObjectSizeHighByte(objectLogPositionPtr) << 32); + } + else // ValueIsInline is true; valueLength will be ignored + { + valueObjectLength = 0; + if (Info.RecordIsInline) // If the record is fully inline, we should not be called here + { + Debug.Fail("Cannot call GetObjectLogRecordStartPositionAndLength for an inline record"); + return 0; + } + } + + return *(ulong*)GetObjectLogPositionAddress(GetOptionalStartAddress()); + } + + /// + /// For recovery, we have already deserialized all objects and know their lengths: Overflow is in the Key or Value field, + /// and Object is in the ObjectLogPosition field. So we can set up the pagePositionInfo for this record directly rather than + /// re-serializing, which also keeps the objectLogTail consistent. + /// + /// The cumulative position on the page (starting from the PageHeader) + /// + /// IMPORTANT: This is only to be called in the disk image copy of the log record, not in the actual log record itself. + /// + /// The total "serialized" lengths from this LogRecord; will be 0 for inline records. Caller will adjust for + /// segment boundaries. + internal readonly ulong SetRecoveredObjectLogRecordStartPosition(ObjectLogFilePositionInfo pagePositionInfo) + { + if (Info.RecordIsInline) + { + Debug.Fail("Cannot call SetRecoveredObjectLogRecordStartPositionAndLengths for an inline record"); + return 0; + } + + // Adding valueAddress and length is the same as GetOptionalStartAddress() but faster + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (valueLength, valueAddress) = dataHeader.GetValueFieldInfo(Info); + var objectLogPositionPtr = (ulong*)GetObjectLogPositionAddress(valueAddress + valueLength); + ulong objectLengths = 0; + + // In case we're a ValueObject, store off the ulong at objectLogPositionPtr before overwriting it with the position in the log file. + var valueObjectLength = *objectLogPositionPtr; + *objectLogPositionPtr = pagePositionInfo.word; + + if (Info.KeyIsOverflow) + { + var (_ /*keyLength*/, keyAddress) = dataHeader.GetKeyFieldInfo(); + var overflow = objectIdMap.GetOverflowByteArray(*(int*)keyAddress); + objectLengths += (uint)overflow.Length; + } + + if (Info.ValueIsOverflow) + { + var overflow = objectIdMap.GetOverflowByteArray(*(int*)valueAddress); + objectLengths += (uint)overflow.Length; + } + else if (Info.ValueIsObject) + { + objectLengths += valueObjectLength; + + // Reuse the valueAddress space to store the low int of valueObjectLength, then store the high byte in the ObjectLogPosition + // (it is combined with the length that is stored in the ObjectId field data of the record). + *(uint*)valueAddress = (uint)(valueObjectLength & 0xFFFFFFFF); + ObjectLogFilePositionInfo.SetObjectSizeHighByte(objectLogPositionPtr, (int)(valueObjectLength >> 32)); + } + else if (Info.RecordIsInline) // ValueIsInline is true; if the record is fully inline, we should not be called here + { + Debug.Fail("Cannot call SetRecoveredObjectLogRecordStartPositionAndLengths for an inline record"); + return 0; + } + + // We no longer need the valueObjectLength in our objectLogPositionPtr, so now we overwrite that with the pagePositionInfo, + // then update pagePositionInfo. + return objectLengths; + } + + internal readonly void OnDeserializationError(bool keyWasSet) + { + // If the key was set, clear it. Then set things as inline so we don't try to release objects on Dispose(). + // This is a transient logRecord, so it is no problem to clear these fields. + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var (keyLength, keyAddress) = dataHeader.GetKeyFieldInfo(); + if (keyWasSet) + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, keyAddress, objectIdMap, isKey: true); + else if (!Info.KeyIsInline) + InfoRef.SetKeyIsInline(); + + // Value length may not be ObjectIdSize. + if (!Info.ValueIsInline) + { + var valueAddress = keyAddress + keyLength; + *(int*)valueAddress = ObjectIdMap.InvalidObjectId; + LogField.ClearObjectIdAndConvertToInline(ref InfoRef, valueAddress, objectIdMap, isKey: false); + } + } + + /// + /// Return the serialized size of the contained logRecord. + /// + public readonly int GetSerializedSize() + { + var recordSize = AllocatedSize; + if (Info.RecordIsInline) + return recordSize; + + _ = GetObjectLogRecordStartPositionAndLengths(out var keyLength, out var valueLength); + return recordSize + keyLength + (int)valueLength; + } + + public readonly long CalculateHeapMemorySize() + { + long size = 0; + if (!Info.Tombstone) + { + if (Info.KeyIsOverflow) + size += KeyOverflow.HeapMemorySize; + + if (Info.ValueIsOverflow) + size += ValueOverflow.HeapMemorySize; + else if (Info.ValueIsObject) + { + var (_ /*valueLength*/, valueAddress) = new RecordDataHeader((byte*)DataHeaderAddress).GetValueFieldInfo(Info); + var objectId = *(int*)valueAddress; + if (objectId != ObjectIdMap.InvalidObjectId) + { + var valueObject = objectIdMap.GetHeapObject(objectId); + if (valueObject is not null) // ignore deleted values being evicted (they are accounted for by InPlaceDeleter) + size += valueObject.HeapMemorySize; + } + } + } + return size; + } + + public readonly void Dispose() + { + if (IsSet) + ClearHeapFields(clearKey: true); + } + + public override readonly string ToString() + { + if (physicalAddress == 0) + return ""; + + string keyString, valueString; + try { keyString = SpanByte.ToShortString(Key, 12); } + catch (Exception ex) { keyString = $""; } + try { valueString = Info.ValueIsObject ? "obj" : ValueSpan.ToShortString(20); } + catch (Exception ex) { valueString = $""; } + + var dataHeader = new RecordDataHeader((byte*)DataHeaderAddress); + var keyOid = Info.KeyIsInline ? "na" : (*(int*)dataHeader.GetKeyFieldInfo().keyAddress).ToString(); + var valOid = Info.ValueIsInline ? "na" : (*(int*)dataHeader.GetValueFieldInfo(Info).valueAddress).ToString(); + + var eTagStr = Info.HasETag ? ETag.ToString() : "na"; + var expirStr = Info.HasExpiration ? Expiration.ToString() : "na"; + return $"ri {Info} | hdr: {dataHeader.ToString(keyString, valueString)} | OIDs k:{keyOid} v:{valOid} | ETag {eTagStr} Expir {expirStr}"; + } + + public bool TrySetValueObjectAndPrepareOptionals(IHeapObject srcValue, in object sizeInfo) => throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs index 153cd3f257b..664e0954cda 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/MallocFixedPageSize.cs @@ -13,46 +13,6 @@ namespace Tsavorite.core { - internal sealed class CountdownWrapper - { - // Separate event for sync code and tcs for async code: Do not block on async code. - private readonly CountdownEvent syncEvent; - private readonly TaskCompletionSource asyncTcs; - int remaining; - - internal CountdownWrapper(int count, bool isAsync) - { - if (isAsync) - { - asyncTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - remaining = count; - return; - } - syncEvent = new CountdownEvent(count); - } - - internal bool IsCompleted => syncEvent is null ? remaining == 0 : syncEvent.IsSet; - - internal void Wait() => syncEvent.Wait(); - internal async ValueTask WaitAsync(CancellationToken cancellationToken) - { - using var reg = cancellationToken.Register(() => asyncTcs.TrySetCanceled()); - await asyncTcs.Task.ConfigureAwait(false); - } - - internal void Decrement() - { - if (asyncTcs is not null) - { - Debug.Assert(remaining > 0); - if (Interlocked.Decrement(ref remaining) == 0) - asyncTcs.TrySetResult(0); - return; - } - syncEvent.Signal(); - } - } - /// /// Memory allocator for objects /// @@ -79,7 +39,7 @@ public sealed class MallocFixedPageSize : IDisposable internal static bool IsBlittable => Utility.IsBlittable(); private int checkpointCallbackCount; - private SemaphoreSlim checkpointSemaphore; + private TaskCompletionSource checkpointTcs; private readonly ConcurrentQueue freeList; @@ -307,12 +267,10 @@ private unsafe long InternalAllocate(int blockSize) /// public async ValueTask IsCheckpointCompletedAsync(CancellationToken token = default) { - var s = checkpointSemaphore; - await s.WaitAsync(token).ConfigureAwait(false); - s.Release(); + await checkpointTcs.Task.WaitAsync(token).ConfigureAwait(false); } - public SemaphoreSlim GetCheckpointSemaphore() => checkpointSemaphore; + public Task GetCheckpointTask() => checkpointTcs.Task; /// /// Public facing persistence API @@ -339,7 +297,7 @@ internal unsafe void BeginCheckpoint(IDevice device, ulong offset, out ulong num int numCompleteLevels = localCount >> PageSizeBits; int numLevels = numCompleteLevels + (recordsCountInLastLevel > 0 ? 1 : 0); checkpointCallbackCount = numLevels; - checkpointSemaphore = new SemaphoreSlim(0); + checkpointTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); uint alignedPageSize = PageSize * (uint)RecordSize; uint lastLevelSize = (uint)recordsCountInLastLevel * (uint)RecordSize; @@ -393,7 +351,7 @@ private unsafe void AsyncFlushCallback(uint errorCode, uint numBytes, object con if (Interlocked.Decrement(ref checkpointCallbackCount) == 0) { - checkpointSemaphore.Release(); + checkpointTcs.TrySetResult(true); } } diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/MemoryPageScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/MemoryPageScanIterator.cs deleted file mode 100644 index e07936553d7..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/MemoryPageScanIterator.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; - -namespace Tsavorite.core -{ - /// - /// Lightweight iterator for memory page (copied to buffer). GetNext() can be used outside epoch protection and locking, - /// but ctor must be called within epoch protection. - /// - /// - /// - internal sealed class MemoryPageScanIterator : ITsavoriteScanIterator - { - readonly AllocatorRecord[] page; - readonly long pageStartAddress; - readonly int recordSize; - readonly int start, end; - int offset; - - public MemoryPageScanIterator(AllocatorRecord[] page, int start, int end, long pageStartAddress, int recordSize) - { - this.page = new AllocatorRecord[page.Length]; - Array.Copy(page, start, this.page, start, end - start); - offset = start - 1; - this.start = start; - this.end = end; - this.pageStartAddress = pageStartAddress; - this.recordSize = recordSize; - } - - public long CurrentAddress => pageStartAddress + offset * recordSize; - - public long NextAddress => pageStartAddress + (offset + 1) * recordSize; - - public long BeginAddress => pageStartAddress + start * recordSize; - - public long EndAddress => pageStartAddress + end * recordSize; - - public void Dispose() - { - } - - public ref TKey GetKey() => ref page[offset].key; - public ref TValue GetValue() => ref page[offset].value; - - public bool GetNext(out RecordInfo recordInfo) - { - while (true) - { - offset++; - if (offset >= end) - { - recordInfo = default; - return false; - } - if (!page[offset].info.Invalid) - break; - } - - recordInfo = page[offset].info; - return true; - } - - public bool GetNext(out RecordInfo recordInfo, out TKey key, out TValue value) - { - var r = GetNext(out recordInfo); - if (r) - { - key = page[offset].key; - value = page[offset].value; - } - else - { - key = default; - value = default; - } - return r; - } - - /// - public override string ToString() => $"BA {BeginAddress}, EA {EndAddress}, CA {CurrentAddress}, NA {NextAddress}, start {start}, end {end}, recSize {recordSize}, pageSA {pageStartAddress}"; - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocator.cs new file mode 100644 index 00000000000..2c8983cb54c --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocator.cs @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Struct wrapper (for inlining) around the fixed-length Blittable allocator. + /// + public struct ObjectAllocator : IAllocator + where TStoreFunctions : IStoreFunctions + { + /// The wrapped class containing all data and most actual functionality. This must be the ONLY field in this structure so its size is sizeof(IntPtr). + private readonly ObjectAllocatorImpl _this; + + public ObjectAllocator(AllocatorSettings settings, TStoreFunctions storeFunctions) + { + // Called by TsavoriteKV via allocatorCreator; must pass a wrapperCreator to AllocatorBase + _this = new(settings, storeFunctions, @this => new ObjectAllocator(@this)); + } + + internal ObjectAllocator(object @this) + { + // Called by AllocatorBase via primary ctor wrapperCreator + _this = (ObjectAllocatorImpl)@this; + } + + /// + public readonly AllocatorBase GetBase() + where TAllocator : IAllocator + => (AllocatorBase)(object)_this; + + /// + public readonly bool HasObjectLog => true; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void InitializeRecord(TKey key, long logicalAddress, in RecordSizeInfo sizeInfo, ref LogRecord logRecord) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _this.InitializeRecord(key, logicalAddress, in sizeInfo, ref logRecord); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => _this.GetRMWCopyRecordSize(in srcLogRecord, ref input, varlenInput); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetRMWInitialRecordSize(key, ref input, varlenInput); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, value, ref input, varlenInput); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, value, ref input, varlenInput); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, in inputLogRecord, ref input, varlenInput); + + /// Get record size required for a new tombstone record + public readonly RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _this.GetDeleteRecordSize(key); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) => _this.PopulateRecordSizeInfo(ref sizeInfo); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void AllocatePage(int pageIndex) => _this.AllocatePage(pageIndex); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); + + /// + public readonly int OverflowPageCount => _this.OverflowPageCount; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly LogRecord CreateLogRecord(long logicalAddress) => _this.CreateLogRecord(logicalAddress); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly LogRecord CreateLogRecord(long logicalAddress, long physicalAddress) => _this.CreateLogRecord(logicalAddress, physicalAddress); + + /// + public readonly LogRecord CreateRemappedLogRecordOverPinnedTransientMemory(long logicalAddress, long physicalAddress) => _this.CreateRemappedLogRecordOverPinnedTransientMemory(logicalAddress, physicalAddress); + + /// + public readonly ObjectIdMap TransientObjectIdMap => _this.transientObjectIdMap; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) => _this.OnDispose(ref logRecord, disposeReason); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) => _this.OnDisposeDiskRecord(ref logRecord, disposeReason); + + /// + public readonly void EvictRecordsInRange(long startAddress, long endAddress, EvictionSource source) => _this.EvictRecordsInRange(startAddress, endAddress, source); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocatorImpl.cs new file mode 100644 index 00000000000..d841b98db9d --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocatorImpl.cs @@ -0,0 +1,1219 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + internal sealed unsafe class ObjectAllocatorImpl : AllocatorBase> + where TStoreFunctions : IStoreFunctions + { + /// For each in-memory page of this allocator we have an for keys that are too large to fit inline into the main log + /// and become overflow byte[], or are Object values; this is needed to root the objects for GC. + internal struct ObjectPage + { + internal readonly ObjectIdMap objectIdMap { get; init; } + + public ObjectPage() => objectIdMap = new(); + + internal readonly void Clear() => objectIdMap?.Clear(); // TODO: Ensure we have already called the RecordDisposer + + public override readonly string ToString() => $"oidMap {objectIdMap}"; + } + + /// The pages of the log, containing object storage. In parallel with AllocatorBase.pagePointers + internal ObjectPage[] objectPages; + + /// The position information for the next write to the object log. + ObjectLogFilePositionInfo objectLogTail; + + /// + /// We use the LastIssued here because we don't want to wait for IO to complete which is when + /// FlushedUntilAddress is updated. Instead, LastIssuedFlushedUntilAddress is the proxy for it: it's updated with the flushEndAddress + /// after the flush has been issued, without waiting for it to complete. + /// + long LastIssuedFlushedUntilAddress; + + /// + /// Dynamically extended Flush end address, used by + /// + long OngoingFlushedUntilAddress; + + /// + /// If the "noFlush" option on is true, we won't try to flush anything below that. + /// + long NoFlushUntilAddress; + + /// The lowest object-log segment in use; adjusted with Truncate to remain consistent with BeginAddress. + internal int lowestObjectLogSegmentInUse = 0; + + // Default to max sizes so testing a size as "greater than" will always be false + readonly int maxInlineKeySize; + readonly int maxInlineValueSize; + + readonly int numberOfFlushBuffers; + readonly int numberOfDeserializationBuffers; + + private readonly IDevice objectLogDevice; + + /// The free pages of the log + private readonly OverflowPool> freePagePool; + + /// Segment size + private long ObjectLogSegmentSize; + + /// + public override string ToString() => BaseToString($" (LI {LastIssuedFlushedUntilAddress}, OG {OngoingFlushedUntilAddress}, No {NoFlushUntilAddress})"); + + public ObjectAllocatorImpl(AllocatorSettings settings, TStoreFunctions storeFunctions, Func> wrapperCreator) + : base(settings, storeFunctions, wrapperCreator, settings.logger, transientObjectIdMap: new ObjectIdMap()) + { + objectLogDevice = settings.LogSettings.ObjectLogDevice; + + maxInlineKeySize = 1 << settings.LogSettings.MaxInlineKeySizeBits; + maxInlineValueSize = 1 << settings.LogSettings.MaxInlineValueSizeBits; + + ObjectLogSegmentSize = 1L << settings.LogSettings.ObjectLogSegmentSizeBits; + + freePagePool = new OverflowPool>(4, static p => { }); + pageHeaderSize = PageHeader.Size; + + if (settings.LogSettings.NumberOfFlushBuffers < LogSettings.kMinFlushBuffers || settings.LogSettings.NumberOfFlushBuffers > LogSettings.kMaxFlushBuffers || !IsPowerOfTwo(settings.LogSettings.NumberOfFlushBuffers)) + throw new TsavoriteException($"{nameof(settings.LogSettings.NumberOfFlushBuffers)} must be between {LogSettings.kMinFlushBuffers} and {LogSettings.kMaxFlushBuffers - 1} and a power of 2"); + numberOfFlushBuffers = settings.LogSettings.NumberOfFlushBuffers; + + if (settings.LogSettings.NumberOfDeserializationBuffers < LogSettings.kMinDeserializationBuffers || settings.LogSettings.NumberOfDeserializationBuffers > LogSettings.kMaxDeserializationBuffers || !IsPowerOfTwo(settings.LogSettings.NumberOfDeserializationBuffers)) + throw new TsavoriteException($"{nameof(settings.LogSettings.NumberOfDeserializationBuffers)} must be between {LogSettings.kMinDeserializationBuffers} and {LogSettings.kMaxDeserializationBuffers - 1} and a power of 2"); + numberOfDeserializationBuffers = settings.LogSettings.NumberOfDeserializationBuffers; + + if (settings.LogSettings.ObjectLogSegmentSizeBits is < LogSettings.kMinObjectLogSegmentSizeBits or > LogSettings.kMaxSegmentSizeBits) + throw new TsavoriteException($"{nameof(settings.LogSettings.ObjectLogSegmentSizeBits)} must be between {LogSettings.kMinObjectLogSegmentSizeBits} and {LogSettings.kMaxSegmentSizeBits}"); + objectLogTail = new(0, settings.LogSettings.ObjectLogSegmentSizeBits); + + objectPages = new ObjectPage[BufferSize]; + for (var ii = 0; ii < BufferSize; ii++) + objectPages[ii] = new(); + } + + /// Initialize allocator + [MethodImpl(MethodImplOptions.NoInlining)] + protected internal override void Initialize() + { + base.Initialize(); + LastIssuedFlushedUntilAddress = FlushedUntilAddress; + OngoingFlushedUntilAddress = 0; + NoFlushUntilAddress = 0; + } + + internal int OverflowPageCount => freePagePool.Count; + + /// + protected override void FreeAllAllocatedPages() + { + for (var index = 0; index < BufferSize; index++) + { + if (IsAllocated(index)) + FreePage(index); + } + } + + /// Allocate memory page, pinned in memory, and in sector aligned form, if possible + internal void AllocatePage(int index) + { + IncrementAllocatedPageCount(); + + if (freePagePool.TryGet(out var item)) + { + pageArrays[index] = item.array; + pagePointers[index] = item.pointer; + objectPages[index] = item.value; + } + else + { + // No free pages are available so allocate new + AllocatePinnedPageArray(index); + objectPages[index] = new(); + } + PageHeader.Initialize(pagePointers[index]); + } + + void ReturnPage(int index) + { + Debug.Assert(index < BufferSize); + if (pagePointers[index] != default) + { + var enqueued = freePagePool.TryAdd(new() + { + array = pageArrays[index], + pointer = pagePointers[index], + value = objectPages[index] + }); + + // We only need to clear the page if it's enqueued; otherwise we don't reuse the page, so can save the time + if (enqueued) + ClearPage(index, 0); + else + objectPages[index].Clear(); + pageArrays[index] = default; + pagePointers[index] = default; + _ = Interlocked.Decrement(ref AllocatedPageCount); + } + } + + internal void FreePage(long page) + { + // If the logSizeTracker is not active, then all pages are used once allocated so there's nothing to add to the overflow pool. + if (logSizeTracker is not null) + ReturnPage((int)(page % BufferSize)); + else + { + objectPages[page % BufferSize].Clear(); + ClearPage(page, 0); + } + } + + internal override void ClearPage(long page, int offset = 0) + { + var index = page % BufferSize; + + // Offset is nonzero only for RecoveryReset, to zero out the page past offset (which is tailAddress). + // In this case, we want to keep the objectPage information for the used (so far) part of the page. + if (offset == 0) + objectPages[index].Clear(); + base.ClearPage(index, offset); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal LogRecord CreateLogRecord(long logicalAddress) => CreateLogRecord(logicalAddress, GetPhysicalAddress(logicalAddress)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal LogRecord CreateLogRecord(long logicalAddress, long physicalAddress) => new(physicalAddress, objectPages[GetPageIndexForAddress(logicalAddress)].objectIdMap); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal LogRecord CreateRemappedLogRecordOverPinnedTransientMemory(long logicalAddress, long physicalAddress) + => LogRecord.CreateRemappedOverPinnedTransientMemory(physicalAddress, objectPages[GetPageIndexForAddress(logicalAddress)].objectIdMap, transientObjectIdMap); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ObjectIdMap GetObjectIdMap(long logicalAddress) => objectPages[GetPageIndexForAddress(logicalAddress)].objectIdMap; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void InitializeRecord(TKey key, long logicalAddress, in RecordSizeInfo sizeInfo, ref LogRecord logRecord) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => logRecord.InitializeRecord(key, in sizeInfo, GetObjectIdMap(logicalAddress)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + { + // Used by RMW to determine the length of copy destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetRMWModifiedFieldInfo(in srcLogRecord, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + { + // Used by RMW to determine the length of initial destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetRMWInitialFieldInfo(key, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + { + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, value, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + { + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, value, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + { + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, in inputLogRecord, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + // Used by Delete to determine the length of a new tombstone record. Does not require an ISessionFunctions method. + var sizeInfo = new RecordSizeInfo() + { + FieldInfo = new() + { + KeySize = key.KeyBytes.Length, + ValueSize = 0, // This will be inline, and with the length prefix and possible space when rounding up to kRecordAlignment, allows the possibility revivification can reuse the record for a Heap Field + HasETag = false, + HasExpiration = false + } + }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) + { + Debug.Assert(sizeInfo.word == 0, "RecordSizeInfo should not be resused"); + + // Object allocator may have Inline or Overflow Keys or Values; additionally, Values may be Object. Both non-inline cases are an objectId in the record. + // Key + if (sizeInfo.FieldInfo.KeySize <= maxInlineKeySize) + sizeInfo.SetKeyIsInline(); + var keySize = sizeInfo.KeyIsInline ? sizeInfo.FieldInfo.KeySize : ObjectIdMap.ObjectIdSize; + + // Value + sizeInfo.MaxInlineValueSize = maxInlineValueSize; + if (!sizeInfo.ValueIsObject && sizeInfo.FieldInfo.ValueSize <= sizeInfo.MaxInlineValueSize) + sizeInfo.SetValueIsInline(); + var valueSize = sizeInfo.ValueIsInline ? sizeInfo.FieldInfo.ValueSize : ObjectIdMap.ObjectIdSize; + + // Record + sizeInfo.CalculateSizes(keySize, valueSize); + } + + /// + /// Dispose an in-memory + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) + { + if (logRecord.IsSet) + { + // Decrement heap from the tracker before clearing fields. The amount depends on + // what's still alive on the record at dispose time: + // - Deleted: value only (key stays for chain traversal; key accounted at eviction). + // Tombstone is NOT yet set, so GetValueHeapMemorySize returns the correct value. + // - Elided / RevivificationFreeList: full remaining heap (key + value). The record + // is being removed from the chain entirely, so both key and value are freed. + // Eviction skips invalid records, so this is the last chance to account for them. + // - CAS failures / InsertAbandoned: no decrement needed — the record was never + // CAS'd into the chain, so +key/+value were never added to the tracker. + if (disposeReason == DisposeReason.Deleted) + { + var valueHeap = logRecord.GetValueHeapMemorySize(); + if (valueHeap != 0) + logSizeTracker?.IncrementSize(-valueHeap); + } + else if (disposeReason is DisposeReason.Elided or DisposeReason.RevivificationFreeList) + { + // Subtract whatever heap remains. The record is being removed from the chain + // entirely (or transferred to the freelist), so eviction will never visit it. + // For tombstoned records, CalculateHeapMemorySize returns 0, but key overflow + // is still alive and needs to be subtracted. + long remainingHeap; + if (!logRecord.Info.Tombstone) + remainingHeap = logRecord.CalculateHeapMemorySize(); + else + remainingHeap = logRecord.Info.KeyIsOverflow ? logRecord.KeyOverflow.HeapMemorySize : 0; + if (remainingHeap != 0) + logSizeTracker?.IncrementSize(-remainingHeap); + } + + storeFunctions.OnDispose(ref logRecord, disposeReason); + + logRecord.ClearHeapFields(disposeReason != DisposeReason.Deleted); + logRecord.ClearOptionals(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) + { + // Route to the store-level trigger; the app (e.g. Garnet) decides whether to Dispose() + // the value object. DiskLogRecord.Dispose() is responsible for releasing the record buffer. + storeFunctions.OnDisposeDiskRecord(ref logRecord, disposeReason); + } + + /// + /// Iterate records in the given logical address range and call + /// on each non-null, non-invalid, non-tombstoned record — including sealed source records that + /// may still own heap. Used during page eviction to allow cleanup of external resources. + /// The caller constrains / to lie on a single page + /// (see and + /// ), so this routine walks records + /// within that single page only. + /// + internal void EvictRecordsInRange(long startAddress, long endAddress, EvictionSource source) + { + var startPage = GetPage(startAddress); + var firstValidAddress = GetFirstValidLogicalAddressOnPage(startPage); + var address = startAddress < firstValidAddress ? firstValidAddress : startAddress; + var pageEndAddress = GetLogicalAddressOfStartOfPage(startPage + 1); + var stopAddress = endAddress < pageEndAddress ? endAddress : pageEndAddress; + + while (address < stopAddress) + { + var physicalAddress = GetPhysicalAddress(address); + var logRecord = new LogRecord(physicalAddress, objectPages[GetPageIndexForAddress(address)].objectIdMap); + var allocatedSize = logRecord.AllocatedSize; + + if (allocatedSize <= 0) + break; + var offset = GetOffsetOnPage(address); + if (offset == 0 || offset + allocatedSize > PageSize) + break; + + // Skip null and invalid records (elided/disposed, heap already cleaned up). + if (logRecord.Info.IsNull || logRecord.Info.Invalid) + { + address += allocatedSize; + continue; + } + + // Decrement the record's heap contribution in a single call. + // For non-tombstoned records, CalculateHeapMemorySize returns key + value heap. + // For tombstoned records, it returns 0 (by design), but the key overflow is still + // alive — value was already decremented at the delete site via OnDispose. + long heapSize; + if (!logRecord.Info.Tombstone) + { + heapSize = logRecord.CalculateHeapMemorySize(); + + if (storeFunctions.CallOnEvict) + storeFunctions.OnEvict(ref logRecord, source); + } + else + { + heapSize = logRecord.Info.KeyIsOverflow ? logRecord.KeyOverflow.HeapMemorySize : 0; + } + + if (heapSize != 0) + logSizeTracker?.IncrementSize(-heapSize); + + address += allocatedSize; + } + } + + /// + /// Call on original in-memory records + /// before they are flushed to disk. This allows the application to snapshot external resources + /// (e.g. BfTree data files) and set flags on the live record while it is still in memory. + /// + internal void FlushRecordsInRange(long startAddress, long endAddress) + { + var page = GetPage(startAddress); + var firstValidAddress = GetFirstValidLogicalAddressOnPage(page); + var address = startAddress < firstValidAddress ? firstValidAddress : startAddress; + + while (address < endAddress) + { + var physicalAddress = GetPhysicalAddress(address); + var logRecord = new LogRecord(physicalAddress, objectPages[GetPageIndexForAddress(address)].objectIdMap); + var allocatedSize = logRecord.AllocatedSize; + + if (allocatedSize <= 0) + break; + + var offset = GetOffsetOnPage(address); + if (offset + allocatedSize > PageSize) + break; + + if (logRecord.Info.Valid && !logRecord.Info.IsNull && !logRecord.Info.SkipOnScan && !logRecord.Info.Tombstone) + storeFunctions.OnFlush(ref logRecord, address); + + address += allocatedSize; + } + } + + /// + /// Dispose memory allocator + /// + public override void Dispose() + { + var localValues = Interlocked.Exchange(ref objectPages, null); + if (localValues != null) + { + freePagePool.Dispose(); + foreach (var value in localValues) + value.Clear(); + base.Dispose(); + } + } + + protected override void TruncateUntilAddressBlocking(long toAddress) + { + // First get the segment of the object log to remove. We've put the lowest object log position used by a page into its PageHeader. + // If toAddress is in the middle of a main log page, we must limit objectlog truncation to the lowest segment used by the that page. + // If toAddress is not past the PageHeader, then assume it is the start of the page and its PageHeader hasn't been written, so use + // the previous page. + var objectLogSegment = -1; + if (objectLogDevice is not null) + { + var addressOfStartOfMainLogPage = GetAddressOfStartOfPageOfAddress(toAddress); + if (GetOffsetOnPage(toAddress) <= PageHeader.Size && addressOfStartOfMainLogPage >= PageSize) + addressOfStartOfMainLogPage -= PageSize; + objectLogSegment = GetLowestObjectLogSegmentInUse(addressOfStartOfMainLogPage); + } + + // Now do the actual truncations. + base.TruncateUntilAddressBlocking(toAddress); + if (objectLogSegment >= 0) + { + objectLogDevice.TruncateUntilSegment(objectLogSegment); + _ = MonotonicUpdate(ref lowestObjectLogSegmentInUse, objectLogSegment, out _); + } + } + + protected override void RemoveSegment(int segment) + { + // if segment is not the last segment (which should be the case), we can use the page header of the start of the segment to get + // the highest object log segment to remove because we know its PageHeader has been written. Otherwise, we have to use the previous + // page's PageHeader to get the object log segment to remove. + var objectLogSegment = -1; + if (objectLogDevice is not null) + { + var addressOfStartOfMainLogPage = GetStartLogicalAddressOfSegment(segment); + if (segment >= device.EndSegment) + addressOfStartOfMainLogPage -= PageSize; + objectLogSegment = GetLowestObjectLogSegmentInUse(addressOfStartOfMainLogPage); + } + + // Now do the actual truncations; TruncateUntilSegment does not remove the passed segment. + base.RemoveSegment(segment); + if (objectLogSegment >= 0) + { + objectLogDevice.TruncateUntilSegment(objectLogSegment); + _ = MonotonicUpdate(ref lowestObjectLogSegmentInUse, objectLogSegment, out _); + } + } + + private int GetLowestObjectLogSegmentInUse(long addressOfStartOfMainLogPage) + { + Debug.Assert(objectLogDevice is not null, "GetHighestObjectLogSegmentToRemove should not be called if there is no objectLogDevice"); + var objectLogSegment = -1; + + // If we're on the first main-log page, we won't be able to remove any object log segments. + // If we're not past the PageHeader of the second page, then the PageHeader probably hasn't been written, so we can't read it. + if (addressOfStartOfMainLogPage <= PageSize + PageHeader.Size) + return objectLogSegment; + + var buffer = bufferPool.Get(sectorSize); + PageAsyncReadResult result = new() { handle = new CountdownEvent(1) }; + try + { + device.ReadAsync((ulong)addressOfStartOfMainLogPage, (IntPtr)buffer.aligned_pointer, (uint)sectorSize, AsyncReadPageCallback, result); + result.handle.Wait(); + if (result.numBytesRead >= PageHeader.Size) + { + var pageHeader = *(PageHeader*)buffer.aligned_pointer; + if (pageHeader.objectLogLowestPositionWord != ObjectLogFilePositionInfo.NotSet) + { + var objectLogPosition = new ObjectLogFilePositionInfo(pageHeader.objectLogLowestPositionWord, objectLogTail.SegmentSizeBits); // TODO verify SegmentSizeBits is correct + objectLogSegment = objectLogPosition.SegmentId; + } + } + } + finally + { + bufferPool.Return(buffer); + result.DisposeHandle(); + } + + return objectLogSegment; + } + + /// + internal override CircularDiskWriteBuffer CreateCircularFlushBuffers(IDevice objectLogDevice, ILogger logger) + { + var localObjectLogDevice = objectLogDevice ?? this.objectLogDevice; + return localObjectLogDevice is not null + ? new(bufferPool, IStreamBuffer.BufferSize, numberOfFlushBuffers, localObjectLogDevice, logger) + : null; + } + + /// + internal override CircularDiskReadBuffer CreateCircularReadBuffers(IDevice objectLogDevice, ILogger logger) + => new(bufferPool, IStreamBuffer.BufferSize, numberOfDeserializationBuffers, objectLogDevice ?? this.objectLogDevice, logger); + + /// + internal override CircularDiskReadBuffer CreateCircularReadBuffers() + => new(bufferPool, IStreamBuffer.BufferSize, numberOfDeserializationBuffers, objectLogDevice, logger); + + /// + internal override int LowestObjectLogSegmentInUse => lowestObjectLogSegmentInUse; + /// + internal override ObjectLogFilePositionInfo GetObjectLogTail() => objectLogTail; + /// + internal override void SetObjectLogTail(ObjectLogFilePositionInfo tail) + { + Debug.Assert(!objectLogTail.HasData, $"SetObjectLogTail should be called only when we have not already set objectLogTail, such as in Recovery"); + objectLogTail = tail; + } + + /// Object log segment size + public override long GetObjectLogSegmentSize() => ObjectLogSegmentSize; + + /// + [MethodImpl(MethodImplOptions.NoInlining)] + protected internal override void RecoveryReset(long tailAddress, long headAddress, long beginAddress, long readonlyAddress) + { + base.RecoveryReset(tailAddress, headAddress, beginAddress, readonlyAddress); + LastIssuedFlushedUntilAddress = readonlyAddress; + OngoingFlushedUntilAddress = 0; + NoFlushUntilAddress = 0; + } + + /// + [MethodImpl(MethodImplOptions.NoInlining)] + internal override void AsyncFlushPagesForReadOnly(long fromAddress, long untilAddress, bool noFlush = false) + { + // We do not need to ensure page alignment of the ReadOnlyAddres for correctness, and in fact that is impossible since we support setting it to whatever + // the current TailAddress is, but for normal flush operations we do set it to page alignment to eliminate concerns about rewriting partial sectors. + GetFlushPageRange(fromAddress, untilAddress, out var startPage, out var numPages); + + // Create the buffers we will use for all ranges of the flush. This calls our callback and disposes itself when the last write of a range completes. + var flushBuffers = CreateCircularFlushBuffers(objectLogDevice: null, logger); + + // Write each page (or partial page) in the range. + for (var flushPage = startPage; flushPage < (startPage + numPages); flushPage++) + { + // The result from PrepareFlushAsyncResult indicates whether we are to perform an actual flush--but asyncResult will be set anyway. + if (PrepareFlushAsyncResult(fromAddress, untilAddress, noFlush, flushPage, out var asyncResult)) + { + asyncResult.flushBuffers = flushBuffers; + + // TsavoriteKV using ObjectAllocator always moves ReadOnlyAddress in page alignment, so if we have a partial first page, it can be written + // in the same loop as full pages, because there are no adjacent fragments. Write the entire page up to asyncResult.untilAddress. + Debug.Assert(PendingFlush[GetPageIndexForAddress(asyncResult.fromAddress)].list.Count == 0, + $"Expected PendingFlush count {PendingFlush[GetPageIndexForAddress(asyncResult.fromAddress)].list.Count} to be 0 for ObjectAllocator"); + + WriteAsync(flushPage, AsyncFlushPageCallback, asyncResult); + } + } + } + + protected override void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult) + => WriteAsync(flushPage, (ulong)(AlignedPageSizeBytes * flushPage), (uint)PageSize, callback, asyncResult, device, objectLogDevice); + + protected override void WriteAsyncToDeviceForSnapshot(long startPage, long flushPage, int pageFlushSize, DeviceIOCompletionCallback callback, + PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long fuzzyStartLogicalAddress) + { + VerifyCompatibleSectorSize(device); + VerifyCompatibleSectorSize(objectLogDevice); + + var epochTaken = epoch.ResumeIfNotProtected(); + try + { + var headAddress = HeadAddress; + + if (headAddress >= asyncResult.untilAddress) + { + // Requested span on page is entirely unavailable in memory; ignore it and call the callback directly. + callback(0, 0, asyncResult); + return; + } + + // If requested page span is only partly available in memory, adjust the start position + // and mark as partial so WriteAsync recalculates the flush size from the adjusted range. + if (headAddress > asyncResult.fromAddress) + { + asyncResult.fromAddress = headAddress; + asyncResult.partial = true; + } + + // We are writing to a separate device which starts at startPage. Eventually, startPage becomes the basis of + // HybridLogRecoveryInfo.snapshotStartFlushedLogicalAddress, which is the page starting at offset 0 of the snapshot file. + WriteAsync(flushPage, (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), (uint)pageFlushSize, + callback, asyncResult, device, objectLogDevice, fuzzyStartLogicalAddress); + } + finally + { + if (epochTaken) + epoch.Suspend(); + } + } + + private void WriteAsync(long flushPage, ulong alignedMainLogFlushPageAddress, uint numBytesToWrite, + DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult, + IDevice device, IDevice objectLogDevice, long fuzzyStartLogicalAddress = long.MaxValue) + { + // We flush within the DiskStreamWriteBuffer, so we do not use the asyncResult here for IO (until the final callback), but it has necessary fields. + + // Short circuit if we are using a null device + if (device is NullDevice) + { + device.WriteAsync(IntPtr.Zero, 0, 0, numBytesToWrite, callback, asyncResult); + return; + } + + var pageStart = GetLogicalAddressOfStartOfPage(asyncResult.page); + var logPagePointer = (byte*)pagePointers[flushPage % BufferSize]; + + // asyncResult.fromAddress is either start of page or start of a record past the page header + Debug.Assert(asyncResult.fromAddress - pageStart is >= PageHeader.Size or 0, $"fromAddress ({asyncResult.fromAddress}, offset {asyncResult.fromAddress - pageStart}) must be 0 or after the PageHeader"); + int startOffset = (int)(asyncResult.fromAddress - pageStart), endOffset = startOffset + (int)numBytesToWrite; + var isFirstRecordOnPage = startOffset <= PageHeader.Size; + + // Write the object log position into the header if this is the first record on the page. If there are no records on the page, we will + // call through to WriteInlinePageAsync so we want the header updated regardless of whether we have objects (this may be a page with no + // objects after some pages with objects, and so we want Truncate() to know it has to preserve those object log segments). + if (isFirstRecordOnPage) + ((PageHeader*)logPagePointer)->SetLowestObjectLogPosition(objectLogTail); + + // Short circuit if we are not using flushBuffers and not in recovery (e.g. using ObjectAllocator for string-only purposes). + if (asyncResult.flushBuffers is null) + { + if (asyncResult.flushRequestState != FlushRequestState.Recovery) + { + WriteInlinePageAsync((nint)pagePointers[flushPage % BufferSize], (ulong)(AlignedPageSizeBytes * flushPage), (uint)AlignedPageSizeBytes, callback, asyncResult, device); + return; + } + Debug.Assert(!asyncResult.partial, "Partial flush should not be requested for recovery flushes"); + } + + Debug.Assert(asyncResult.page == flushPage, $"asyncResult.page {asyncResult.page} should equal flushPage {flushPage}"); + + // numBytesToWrite is calculated from start and end logical addresses, either for the full page or a subset of records (aligned to start and end of record boundaries), + // in the allocator page (including the objectId space for Overflow and Heap Objects). Note: "Aligned" in this discussion refers to sector (as opposed to record) alignment. + + // Initialize offsets into the allocator page based on full-page (including the page header), then override them if partial. + Debug.Assert(asyncResult.untilAddress - pageStart >= PageHeader.Size, $"untilAddress ({asyncResult.untilAddress}, offset {asyncResult.untilAddress - pageStart}) must be past PageHeader {flushPage}"); + if (asyncResult.partial) + { + // We're writing only a subset of the page. + endOffset = (int)(asyncResult.untilAddress - pageStart); + numBytesToWrite = (uint)(endOffset - startOffset); + } + + // Adjust so the first record on the page includes the page header. We've already asserted fromAddress such that startOffset is either 0 or >= PageHeader. + var logicalAddress = asyncResult.fromAddress; + var firstRecordOffset = startOffset; + if (isFirstRecordOnPage) + { + if (startOffset == 0) + { + // For the first record on the page the caller may have passed the address of the start of the page rather than the offset at the end of the PageHeader. + firstRecordOffset = PageHeader.Size; + logicalAddress += firstRecordOffset; + } + else + { + startOffset = 0; // Include the PageHeader in the page output + numBytesToWrite = (uint)(endOffset - startOffset); + } + } + else + Debug.Assert(asyncResult.flushRequestState != FlushRequestState.Recovery, "FlushRequestState.IsForRecovery should always be done an entire page at a time"); + + var alignedStartOffset = RoundDown(startOffset, (int)device.SectorSize); + var startPadding = startOffset - alignedStartOffset; + var alignedBufferSize = RoundUp(startPadding + (int)numBytesToWrite, (int)device.SectorSize); + + // If we are in snapshot checkpoint we will need to acquire the epoch whenever we access the log record or oidMap; we will not have the epoch + // when we enter here. If we are in recovery, we will not have the epoch either, but we don't need to acquire it as there are no other operations + // happening. Otherwise, we are here because we are moving the read-only address (FoldOver checkpoint is a special case of this). In that case + // we will have the epoch on entry, but we will not need to remain protected because ShiftHeadAddress always remains below FlushedUntilAddress + // so the actual log page, inluding ObjectIdMap, will remain valid until we complete this partial flush. So we release the epoch if we have it; + // we don't need it and don't want to hold it during the time-consuming actual flush. + var pulseEpoch = asyncResult.flushRequestState == FlushRequestState.Snapshot; + var protectEpochWhenDone = epoch.TrySuspend(); + + // Overflow Keys and Values are written to, and Object values are serialized to, this Stream, if we have flushBuffers. + ObjectLogWriter logWriter = null; + + // Do everything below here in the try{} to be sure the epoch is Resumed()d if we Suspended it. + SectorAlignedMemory srcBuffer = default; + try + { + // Create a local copy of the main-log page inline data. Space for ObjectIds and the ObjectLogPosition will be updated as we go + // (ObjectId space and a byte of the length-metadata space will combine for 5 bytes or 1TB of object size, which is our max). This does + // not change record sizes, so the logicalAddress space is unchanged. Also, we will not advance HeadAddress until this flush is complete + // and has updated FlushedUntilAddress, so we don't have to worry about the page being yanked out from underneath us (and Objects + // won't be disposed before we're done). TODO: Loop on successive subsets of the page's records to make this initial copy buffer smaller. + var objectIdMap = objectPages[flushPage % BufferSize].objectIdMap; + srcBuffer = bufferPool.Get(alignedBufferSize); + asyncResult.freeBuffer1 = srcBuffer; + + // Read back the first sector if the start is not aligned (this means we already wrote a partially-filled sector with ObjectLog fields set). + if (startPadding > 0) + { + // TODO: This will potentially overwrite partial sectors (with the same data) if this is a partial flush; a workaround would be difficult. + // TODO: Cache the last sector flushed in readBuffers so we can avoid this Read. + PageAsyncReadResult result = new() { handle = new CountdownEvent(1) }; + device.ReadAsync(alignedMainLogFlushPageAddress + (ulong)alignedStartOffset, (IntPtr)srcBuffer.aligned_pointer, (uint)sectorSize, AsyncReadPageCallback, result); + result.handle.Wait(); + result.DisposeHandle(); + } + + try + { + if (pulseEpoch) + epoch.Resume(); + + // Copy from the record start position (startOffset) in the main log page to the src buffer starting at its offset in the first sector (startPadding). + var allocatorPageSpan = new Span((byte*)logPagePointer + startOffset, (int)numBytesToWrite); + allocatorPageSpan.CopyTo(srcBuffer.TotalValidSpan.Slice(startPadding)); + srcBuffer.available_bytes = (int)numBytesToWrite + startPadding; + } + finally + { + if (pulseEpoch) + epoch.Suspend(); + } + + if (asyncResult.flushBuffers is not null) + { + logWriter = new(device, asyncResult.flushBuffers, storeFunctions); + _ = logWriter.OnBeginPartialFlush(objectLogTail); + } + + // Include page header when calculating end address. + var endPhysicalAddress = (long)srcBuffer.GetValidPointer() + startPadding + numBytesToWrite; + var physicalAddress = (long)srcBuffer.GetValidPointer() + firstRecordOffset - alignedStartOffset; + + // For recovery flushes we don't re-serialize; rather we just update the object lengths and positions in the log file using deserialized + // Overflow and/or Object information. That means we also have to track the increasing object log position "as if" we were re-serializing + // the objects (because it is recovery, the lengths will not change--even if this is a page from snapshot, in which case we still don't + // want to write to an object-log segment; that is ONLY done on OnPagesMarkedReadOnly. + ref var pageHeader = ref *(PageHeader*)srcBuffer.GetValidPointer(); + + var recoveryOngoingPageHeader = asyncResult.flushRequestState == FlushRequestState.Recovery ? pageHeader.GetLowestObjectLogPosition(objectLogTail.SegmentSizeBits) : default; + var endLogicalAddress = logicalAddress + (endPhysicalAddress - physicalAddress); + while (physicalAddress < endPhysicalAddress) + { + // LogRecord is in the *copy of* the log buffer. We will update it (for objectIds) without affecting the actual record in the log. + // Increment for next iteration; use allocatedSize because that is what LogicalAddress is based on. + var logRecord = new LogRecord(physicalAddress, objectIdMap); + var logRecordSize = logRecord.AllocatedSize; + var extraRecordOffset = 0; + + // Do not write Invalid records. This includes IsNull records. By the time we get here, ReadOnlyAddress has been advanced, so the + // record's state (IsValid, IsInNewVersion, inline data, etc.) will not change. + if (logRecord.Info.Valid) + { + // Do not write v+1 records (e.g. during a checkpoint) + if (logicalAddress < fuzzyStartLogicalAddress || !logRecord.Info.IsInNewVersion) + { + // Do not write objects for fully-inline records. This should always be false if we don't have a logWriter (i.e. no flushBuffers), + // which would be the case where we were created to be used for inline string records only. + if (logRecord.Info.RecordHasObjects) + { + if (asyncResult.flushRequestState != FlushRequestState.Recovery) + { + var recordStartPosition = logWriter.GetNextRecordStartPosition(); + Debug.Assert(asyncResult.flushRequestState != FlushRequestState.ReadOnly || !isFirstRecordOnPage || recordStartPosition.CurrentAddress == objectLogTail.CurrentAddress, + $"ObjectLogPosition mismatch on first record for ReadOnly flush: rec {recordStartPosition.CurrentAddress}, tail {objectLogTail.CurrentAddress}"); + + OverflowByteArray keyOverflow = default, valueOverflow = default; + IHeapObject valueObject = default; + try + { + if (pulseEpoch) + { + epoch.Resume(); + + // Check to see if HeadAddress (which can change while we're here) has moved past this record. + var headAddress = HeadAddress; + if (headAddress > logicalAddress) + { + if (headAddress <= endLogicalAddress) + { + // Jump ahead to HeadAddress. Recover() will start recovery at the last FlushedUntilAddress of the main log, + // which will never be less than HeadAddress. So we do not need to worry about whatever values are in the inline + // record space between the current logicalAddress and HeadAddress. + extraRecordOffset = (int)(headAddress - (logicalAddress + logRecordSize)); + // Skip object serialization + goto NextRecord; + } + else + { + asyncResult.flushRequestState = FlushRequestState.WriteNotIssued; + goto WritePage; + } + } + } + + if (logRecord.Info.KeyIsOverflow) + keyOverflow = logRecord.KeyOverflow; + + if (logRecord.Info.ValueIsOverflow) + valueOverflow = logRecord.ValueOverflow; + else if (logRecord.Info.ValueIsObject) + valueObject = logRecord.ValueObject; + } + finally + { + if (pulseEpoch) + epoch.Suspend(); + } + + var valueObjectLength = logWriter.WriteRecordObjects(in keyOverflow, in valueOverflow, in valueObject); + logRecord.SetObjectLogRecordStartPositionAndLength(recordStartPosition, valueObjectLength); + } + else + { + // In recovery we just need to update the disk-image LogRecord with the object lengths and file position, and then + // advance the recoveryOngoingPageHeader position. This advancement will also take care of segment breaks if needed. + var objectLengths = logRecord.SetRecoveredObjectLogRecordStartPosition(recoveryOngoingPageHeader); + recoveryOngoingPageHeader.Advance(objectLengths); + } + + // Do this for both cases so it's clear when debugging + isFirstRecordOnPage = false; + } + } + else + { + // Mark v+1 records as invalid to avoid deserializing them on recovery + logRecord.InfoRef.SetInvalid(); + } + } // endif record id Valid + + NextRecord: + logicalAddress += logRecordSize + extraRecordOffset; // advance in main log + physicalAddress += logRecordSize + extraRecordOffset; // advance in source buffer + } + + WritePage: + // We are done with the per-record objectlog flushes and we've updated the copy of the allocator page. Now write that updated page + // to the main log file unless we are to skip it because HeadAddress advanced. + if (asyncResult.flushRequestState != FlushRequestState.WriteNotIssued) + { + if (asyncResult.partial) + { + // We're writing only a subset of the page, so update our count of bytes to write. + var aligned_end = (int)RoundUp(asyncResult.untilAddress - alignedStartOffset, (int)device.SectorSize); + numBytesToWrite = (uint)(aligned_end - alignedStartOffset); + } + + // Finally write the main log page as part of OnPartialFlushComplete, or directly if we had no flushBuffers. + // TODO: This will potentially overwrite partial sectors if this is a partial flush; a workaround would be difficult. + if (logWriter is not null) + logWriter.OnPartialFlushComplete(srcBuffer.GetValidPointer(), alignedBufferSize, device, alignedMainLogFlushPageAddress + (uint)alignedStartOffset, callback, asyncResult, ref objectLogTail); + else + device.WriteAsync((IntPtr)srcBuffer.GetValidPointer(), alignedMainLogFlushPageAddress + (uint)alignedStartOffset, (uint)alignedBufferSize, callback, asyncResult); + } + } + finally + { + if (protectEpochWhenDone) + epoch.Resume(); + logWriter?.Dispose(); + } + } + + /// + /// Action to be performed when pages move into the immutable region. + /// Seal: make sure there are no longer any threads writing to the page + /// Flush: send page to secondary store + /// + internal override void OnPagesMarkedReadOnly(long newSafeReadOnlyAddress, bool noFlush = false) + { + Debug.Assert(newSafeReadOnlyAddress > HeadAddress); + Debug.Assert(newSafeReadOnlyAddress <= GetTailAddress()); + if (noFlush) + _ = MonotonicUpdate(ref NoFlushUntilAddress, newSafeReadOnlyAddress, out _); + if (MonotonicUpdate(ref SafeReadOnlyAddress, newSafeReadOnlyAddress, out var oldSafeReadOnlyAddress)) + { + // This thread is responsible for [oldSafeReadOnlyAddress -> newSafeReadOnlyddress] + while (true) + { + var _ongoingFlushedUntilAddress = OngoingFlushedUntilAddress; + + // If we are closing in the middle of an ongoing OPMROWorker loop, exit. + if (_ongoingFlushedUntilAddress >= newSafeReadOnlyAddress) + break; + + // We'll continue the loop if we fail the CAS here; that means another thread extended the Ongoing range. + if (Interlocked.CompareExchange(ref OngoingFlushedUntilAddress, newSafeReadOnlyAddress, _ongoingFlushedUntilAddress) == _ongoingFlushedUntilAddress) + { + // If _ongoingFlushedUntilAddress != 0 then another thread is runnning the OPMROWorker loop and will see the OnGoingFlushedUntilAddress increment to + // include newSafeReadOnlyAddress so we are done here. Otherwise, this thread is responsible for flushing [LastIssuedFlushedUntilAddress -> newSafeHeadAddress] + // and any other ranges that OnGoingFlushedUntilAddress is incremented to, and we are done here when that concludes. + if (_ongoingFlushedUntilAddress == 0) + OnPagesMarkedReadOnlyWorker(); + return; + } + _ = Thread.Yield(); + } + } + } + + private void OnPagesMarkedReadOnlyWorker() + { + while (true) + { + var flushStartAddress = LastIssuedFlushedUntilAddress; + var flushEndAddress = OngoingFlushedUntilAddress; + + // Notify the application per record before flushing, so it can snapshot external + // resources (e.g. BfTree data files) and/or set flags on the live in-memory records. + // This runs on the ORIGINAL records (not a copy), under epoch protection. + if (storeFunctions.CallOnFlush) + FlushRecordsInRange(flushStartAddress, flushEndAddress); + + // Debug.WriteLine("SafeReadOnly shifted from {0:X} to {1:X}", oldSafeReadOnlyAddress, newSafeReadOnlyAddress); + if (onReadOnlyObserver != null) + { + // This scan does not need a store because it does not lock; it is epoch-protected so by the time it runs no current thread + // will have seen a record below the new ReadOnlyAddress as "in mutable region". + using var iter = Scan(store: null, flushStartAddress, flushEndAddress, DiskScanBufferingMode.NoBuffering); + onReadOnlyObserver?.OnNext(iter); + } + + var noFlushUntilAddress = NoFlushUntilAddress; + if (flushEndAddress > noFlushUntilAddress && flushStartAddress < noFlushUntilAddress) + { + // NoFlushUntilAddress is in the middle of the flush range, so we flush in two parts: <= NoFUA (noFlush) and > NoFUA (!noFlush) + AsyncFlushPagesForReadOnly(flushStartAddress, noFlushUntilAddress, noFlush: true); + AsyncFlushPagesForReadOnly(noFlushUntilAddress, flushEndAddress, noFlush: false); + } + else + { + // We're entirely above or below NoFUA, so we can flush in one go with the appropriate noFlush value + AsyncFlushPagesForReadOnly(flushStartAddress, flushEndAddress, noFlush: flushEndAddress <= NoFlushUntilAddress); + } + + var updatedLIFUA = MonotonicUpdate(ref LastIssuedFlushedUntilAddress, flushEndAddress, out var oldLastIssuedFlushedUntilAddress); + Debug.Assert(updatedLIFUA, $"Failed to update LIFUA"); + Debug.Assert(oldLastIssuedFlushedUntilAddress == flushStartAddress, $"Expected LastIssuedFlushedUntilAddress to be {flushStartAddress} but was {oldLastIssuedFlushedUntilAddress}"); + + // End if we have exhausted co-operative work. This includes the case where OngoingFUA and flushEndAddress are already 0. + if (Interlocked.CompareExchange(ref OngoingFlushedUntilAddress, 0, flushEndAddress) == flushEndAddress) + break; + _ = Thread.Yield(); + } + } + + private void AsyncReadPageCallback(uint errorCode, uint numBytes, object context) + { + if (errorCode != 0) + logger?.LogError($"{nameof(AsyncReadPageCallback)} error: {{errorCode}}", errorCode); + + // Set the page status to flushed + var result = (PageAsyncReadResult)context; + result.numBytesRead = numBytes; + _ = result.handle.Signal(); + } + + /// + /// This override of the base function reads Overflow keys or values, or Object values. + private protected override bool VerifyRecordFromDiskCallback(ref AsyncIOContext ctx, out long prevAddressToRead, out int prevLengthToRead) + { + // If this fails it is either too-short main-log record or a key mismatch. Let the top-level retry handle it. This will always + // use the transientObjectIdMap (unless we are copying to tail, in which case we will remap to the allocator page's objectIdMap). + if (!base.VerifyRecordFromDiskCallback(ref ctx, out prevAddressToRead, out prevLengthToRead)) + return false; + + // If the record is inline, we have no Overflow or Objects to retrieve. + ref var diskLogRecord = ref ctx.diskLogRecord; + if (diskLogRecord.Info.RecordIsInline) + return true; + + var startPosition = new ObjectLogFilePositionInfo(ctx.diskLogRecord.logRecord.GetObjectLogRecordStartPositionAndLengths(out var keyLength, out var valueLength), objectLogTail.SegmentSizeBits); + var totalBytesToRead = (ulong)keyLength + valueLength; + + // 'using' is OK here as we complete the object reads before returning. + using var readBuffers = CreateCircularReadBuffers(objectLogDevice, logger); + + var logReader = new ObjectLogReader(readBuffers, storeFunctions); + logReader.OnBeginReadRecords(startPosition, totalBytesToRead); + if (logReader.ReadRecordObjects(ref diskLogRecord.logRecord, ctx.requestKey, startPosition.SegmentSizeBits)) + { + // Success. The deserialized heap object's Dispose() will be invoked when the DiskLogRecord + // is disposed (ObjectIdMap.Free → IHeapObject.Dispose), unless the object is transferred out + // (e.g. via CopyToTail) in which case the transient ObjectIdMap slot is cleared without disposing. + // Default the output arguments for reading a previous record. + prevAddressToRead = 0; + return true; + } + + // Ensure we have finished all object reads + logReader.OnEndReadRecords(); + + // If readBuffer.Read returned false it was due to an Overflow key mismatch or an Invalid record, so get the previous record. + prevAddressToRead = (*(RecordInfo*)ctx.record.GetValidPointer()).PreviousAddress; + return false; + } + + protected override void ReadAsync(ulong alignedSourceAddress, IntPtr destinationPtr, uint aligned_read_length, + DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device) + { + //TODO("Add CancellationToken to the ReadAsync and WriteAsync paths"); + + asyncResult.callback = callback; + asyncResult.destinationPtr = destinationPtr; + asyncResult.maxAddressOffsetOnPage = aligned_read_length; + + device.ReadAsync(alignedSourceAddress, destinationPtr, aligned_read_length, AsyncReadPageWithObjectsCallback, asyncResult); + } + + private void AsyncReadPageWithObjectsCallback(uint errorCode, uint numBytes, object context) + { + if (errorCode != 0) + logger?.LogError($"{nameof(AsyncReadPageWithObjectsCallback)} error: {{errorCode}}", errorCode); + + var result = (PageAsyncReadResult)context; + var pageStartAddress = (long)result.destinationPtr; + + // Iterate all records in range to determine how many bytes we need to read from objlog. + ObjectLogFilePositionInfo startPosition = new(), endPosition = new(); + var endKeyLength = 0; + ulong endValueLength = 0; + ulong totalBytesToRead = 0; + var recordAddress = pageStartAddress + PageHeader.Size; + var endAddress = pageStartAddress + result.maxAddressOffsetOnPage; + + while (recordAddress < endAddress) + { + // Increment for next iteration; use allocatedSize because that is what LogicalAddress is based on. + var logRecord = new LogRecord(recordAddress); + recordAddress += logRecord.AllocatedSize; + + if (logRecord.Info.RecordHasObjects && logRecord.Info.Valid) + { + if (!startPosition.IsSet) + startPosition = new(logRecord.GetObjectLogRecordStartPositionAndLengths(out _, out _), objectLogTail.SegmentSizeBits); + endPosition = new(logRecord.GetObjectLogRecordStartPositionAndLengths(out endKeyLength, out endValueLength), objectLogTail.SegmentSizeBits); + } + } + + // The page may not have contained any records with objects + if (startPosition.IsSet) + { + endPosition.Advance((ulong)endKeyLength + endValueLength); + totalBytesToRead = endPosition - startPosition; + + // Iterate all records again to actually do the deserialization. + result.readBuffers.nextFileReadPosition = startPosition; + recordAddress = pageStartAddress + PageHeader.Size; + var logReader = new ObjectLogReader(result.readBuffers, storeFunctions); + logReader.OnBeginReadRecords(startPosition, totalBytesToRead); + + var objectIdMapToUse = result.isForRecovery ? objectPages[result.page % BufferSize].objectIdMap : transientObjectIdMap; + + while (recordAddress < endAddress) + { + // Increment for next iteration; use allocatedSize because that is what LogicalAddress is based on. + var logRecord = new LogRecord(recordAddress, objectIdMapToUse); + recordAddress += logRecord.AllocatedSize; + + if (logRecord.Info.RecordHasObjects && logRecord.Info.Valid) + { + _ = logReader.ReadRecordObjects(ref logRecord, default(EmptyKey), startPosition.SegmentSizeBits); + // CalculateHeapMemorySize returns 0 for tombstones, but eviction subtracts + // key overflow for tombstoned records. Add it here so the tracker stays balanced. + if (logRecord.Info.Tombstone) + { + if (logRecord.Info.KeyIsOverflow) + logSizeTracker?.IncrementSize(logRecord.KeyOverflow.HeapMemorySize); + } + else + { + logSizeTracker?.UpdateSize(in logRecord, add: true); + } + } + } + + // Ensure we have finished all object reads + logReader.OnEndReadRecords(); + } + + // Call the "real" page read callback + result.callback(errorCode, numBytes, context); + result.Free(); + return; + } + + /// + /// Iterator interface for scanning Tsavorite log + /// + /// + public override ITsavoriteScanIterator Scan(TsavoriteKV> store, + long beginAddress, long endAddress, DiskScanBufferingMode diskScanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool includeClosedRecords = false) + => new ObjectScanIterator>(store, this, beginAddress, endAddress, epoch, diskScanBufferingMode, includeClosedRecords: includeClosedRecords); + + /// + /// Implementation for push-scanning Tsavorite log, called from LogAccessor + /// + internal override bool Scan(TsavoriteKV> store, + long beginAddress, long endAddress, ref TScanFunctions scanFunctions, DiskScanBufferingMode scanBufferingMode) + { + using ObjectScanIterator> iter = new(store, this, beginAddress, endAddress, epoch, scanBufferingMode, includeClosedRecords: false, logger: logger); + return PushScanImpl(beginAddress, endAddress, ref scanFunctions, iter); + } + + /// + /// Implementation for push-scanning Tsavorite log with a cursor, called from LogAccessor + /// + internal override bool ScanCursor(TsavoriteKV> store, + ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, + bool resetCursor = true, bool includeTombstones = false) + { + using ObjectScanIterator> iter = new(store, this, cursor, endAddress, epoch, DiskScanBufferingMode.SinglePageBuffering, + includeClosedRecords: maxAddress < long.MaxValue, logger: logger); + return ScanLookup>>(store, scanCursorState, ref cursor, count, scanFunctions, iter, validateCursor, + maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); + } + + /// + /// Implementation for push-iterating key versions, called from LogAccessor + /// + internal override bool IterateKeyVersions(TsavoriteKV> store, + TKey key, long beginAddress, ref TScanFunctions scanFunctions) + { + using ObjectScanIterator> iter = new(store, this, beginAddress, epoch, logger: logger); + return IterateHashChain(store, key, beginAddress, ref scanFunctions, iter); + } + + /// + internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver observer) + { + using var iter = new ObjectScanIterator>(store: null, this, beginAddress, endAddress, epoch, DiskScanBufferingMode.NoBuffering, InMemoryScanBufferingMode.NoBuffering, + includeClosedRecords: false, assumeInMemory: true, logger: logger); + observer?.OnNext(iter); + } + + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectIdMap.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectIdMap.cs new file mode 100644 index 00000000000..233642a846e --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectIdMap.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Maps the ObjectId in the ObjectAllocator's Value field to the actual object in the object multi-level array. + /// This may be either a byte[] Span-overflow allocation, or an IHeapObject. + /// + public class ObjectIdMap + { + /// We will never return a negative index from Allocate + public const int InvalidObjectId = -1; + + /// Size of the object Id + public const int ObjectIdSize = sizeof(int); + + // For this class, the "page" is an object. + internal MultiLevelPageArray objectArray; + + internal SimpleConcurrentStack freeSlots; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ObjectIdMap() + { + // entriesPerPage comes from ObjectAllocator's minimum pagesize / expected record size so is the maximum possible number of records. + // Records may be larger due to key size but we have limits on that so it is unlikely we will waste very much of this allocation. + objectArray = new(); + freeSlots = new(); + } + + internal int Count => objectArray.Count; + + /// Reserve a slot and return its ID. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Allocate() + { + if (freeSlots.TryPop(out var objectId)) + { + Debug.Assert(objectId < objectArray.tail, $"objectId {objectId} retrieved from freelist must be less than tail {objectArray.tail}"); + return objectId; + } + return objectArray.Allocate(); + } + + /// Reserve a slot, place the Overflow into it, and return the slot's ID. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int AllocateAndSet(OverflowByteArray element) + { + var id = Allocate(); + Set(id, element); + return id; + } + + /// Reserve a slot, place the Object into it, and return the slot's ID. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int AllocateAndSet(IHeapObject element) + { + var id = Allocate(); + Set(id, element); + return id; + } + + /// Free a slot for reuse by another record on this page (e.g. when sending a record to the revivification freelist, on a failed CAS, on record disposal, etc.). + /// The slot is cleared so its previous occupant (byte[] overflow or IHeapObject) becomes unreachable via the map and eligible for GC. If the application needs + /// to run on an IHeapObject (e.g. to release external resources), it should do so in + /// before the containing record is cleared. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Free(int objectId) + { + if (objectId != InvalidObjectId) + { + objectArray.Set(objectId, default); + freeSlots.Push(objectId); + } + } + + /// Returns the slot's object as an IHeapObject. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal IHeapObject GetHeapObject(int objectId) => Unsafe.As(objectArray.Get(objectId)); + + /// Returns the slot's object as an . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OverflowByteArray GetOverflowByteArray(int objectId) => new(Unsafe.As(objectArray.Get(objectId))); + + /// Sets the slot's object. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Set(int objectId, IHeapObject element) => objectArray.Set(objectId, element); + + /// Sets the slot's object. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Set(int objectId, OverflowByteArray element) => objectArray.Set(objectId, element.Array); + + /// Clear the array. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear() + { + objectArray?.Clear(1 << MultiLevelPageArray.PrimaryClearRetainedChapterSizeBits); + freeSlots.Clear(1 << MultiLevelPageArray.FreeListClearRetainedChapterSizeBits); + } + + /// + public override string ToString() => $"objectArray: {(objectArray is not null ? objectArray.ToString() : "")}; freeSlots: {(freeSlots is not null ? freeSlots.ToString() : "")}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectScanIterator.cs new file mode 100644 index 00000000000..56c5fed1e76 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectScanIterator.cs @@ -0,0 +1,450 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ + /// + /// Scan iterator for hybrid log + /// + internal sealed unsafe class ObjectScanIterator : ScanIteratorBase, ITsavoriteScanIterator, IPushScanIterator + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + private readonly TsavoriteKV store; + private readonly AllocatorBase hlogBase; + private readonly BlittableFrame frame; + + private SectorAlignedMemory recordBuffer; + private readonly bool assumeInMemory; + + private DiskLogRecord diskLogRecord; + + /// + /// Constructor + /// + /// + /// The fully derived log implementation + /// + /// + /// + /// + /// + /// Epoch to use for protection; may be null if is true. + /// Provided address range is known by caller to be in memory, even if less than HeadAddress + /// + internal ObjectScanIterator(TsavoriteKV store, AllocatorBase hlogBase, + long beginAddress, long endAddress, LightEpoch epoch, DiskScanBufferingMode diskScanBufferingMode, + InMemoryScanBufferingMode memScanBufferingMode = InMemoryScanBufferingMode.NoBuffering, + bool includeClosedRecords = false, bool assumeInMemory = false, ILogger logger = null) + : base(beginAddress == 0 ? hlogBase.GetFirstValidLogicalAddressOnPage(0) : beginAddress, endAddress, diskScanBufferingMode, memScanBufferingMode, + includeClosedRecords, epoch, hlogBase.LogPageSizeBits, logger: logger) + { + this.store = store; + this.hlogBase = hlogBase; + this.assumeInMemory = assumeInMemory; + if (frameSize > 0) + frame = new BlittableFrame(frameSize, hlogBase.PageSize, hlogBase.GetDeviceSectorSize()); + InitializeReadBuffers(hlogBase); + } + + /// + /// Constructor for use with tail-to-head push iteration of the passed key's record versions + /// + internal ObjectScanIterator(TsavoriteKV store, AllocatorBase hlogBase, + long beginAddress, LightEpoch epoch, ILogger logger = null) + : base(beginAddress == 0 ? hlogBase.GetFirstValidLogicalAddressOnPage(0) : beginAddress, hlogBase.GetTailAddress(), + DiskScanBufferingMode.SinglePageBuffering, InMemoryScanBufferingMode.NoBuffering, false, epoch, hlogBase.LogPageSizeBits, logger: logger) + { + this.store = store; + this.hlogBase = hlogBase; + assumeInMemory = false; + if (frameSize > 0) + frame = new BlittableFrame(frameSize, hlogBase.PageSize, hlogBase.GetDeviceSectorSize()); + } + + #region TODO Unify with SpanByteScanIterator + /// + public bool SnapCursorToLogicalAddress(ref long cursor) + { + Debug.Assert(currentAddress == -1, "SnapCursorToLogicalAddress must be called before GetNext()"); + Debug.Assert(nextAddress == cursor, "SnapCursorToLogicalAddress should have nextAddress == cursor"); + + if (!InitializeGetNextAndAcquireEpoch(out var stopAddress)) + return false; + try + { + if (!LoadPageIfNeeded(out var headAddress, out var currentPage, stopAddress)) + return false; + beginAddress = nextAddress = SnapToLogicalAddressBoundary(ref cursor, headAddress, currentPage); + } + finally + { + epoch?.Suspend(); + } + return true; + } + + private bool InitializeGetNextAndAcquireEpoch(out long stopAddress) + { + if (diskLogRecord.IsSet) + { + hlogBase._wrapper.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); + } + diskLogRecord = default; + currentAddress = nextAddress; + + // Acquire the epoch BEFORE sampling Initializing / TailAddress / HeadAddress / + // pagePointers, so that any allocator state we read is consistent with the epoch + // we hold. + epoch?.Resume(); + + // If a concurrent Reset is rebuilding the allocator, terminate the iteration + // cleanly — Reset is a wholesale wipe, the records we were iterating are gone, + // and the address range we were stepping through no longer maps to live data. + // Also avoids dereferencing the non-monotonic mid-Initialize state (HeadAddress + // already rewound to FirstValidAddress while TailPageOffset still holds the + // pre-Reset tail and pagePointers[i] are mostly 0). + if (hlogBase.Initializing) + { + epoch?.Suspend(); + stopAddress = 0; + return false; + } + + stopAddress = endAddress < hlogBase.GetTailAddress() ? endAddress : hlogBase.GetTailAddress(); + if (currentAddress >= stopAddress) + { + epoch?.Suspend(); + return false; + } + + // Success; caller will suspend the epoch as needed. + return true; + } + + private bool LoadPageIfNeeded(out long headAddress, out long currentPage, long stopAddress) + { + headAddress = hlogBase.HeadAddress; + + if (currentAddress < hlogBase.BeginAddress && !assumeInMemory) + currentAddress = hlogBase.BeginAddress; + + // If currentAddress < headAddress and we're not buffering and not guaranteeing the records are in memory, fail. + if (frameSize == 0 && currentAddress < headAddress && !assumeInMemory) + { + // Caller will suspend the epoch. + throw new TsavoriteException("Iterator address is less than log HeadAddress in memory-scan mode"); + } + + currentPage = hlogBase.GetPage(currentAddress); + if (currentAddress < headAddress && !assumeInMemory) + _ = BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, stopAddress); + + // Success; keep the epoch held for GetNext (SnapCursorToLogicalAddress will Suspend()). + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal long SnapToLogicalAddressBoundary(ref long logicalAddress, long headAddress, long currentPage) + { + var offset = hlogBase.GetOffsetOnPage(logicalAddress); + + // Subtracting offset means this physicalAddress is at the start of the page. Adjust for PageHeader. + long totalSizes = PageHeader.Size; + if (currentPage == 0) + { + if (logicalAddress < hlogBase.BeginAddress) + return logicalAddress = hlogBase.BeginAddress; + totalSizes = (int)hlogBase.BeginAddress; + } + var physicalAddress = GetPhysicalAddress(logicalAddress, headAddress, currentPage, offset) - offset + totalSizes; + + while (totalSizes <= offset) + { + var allocatedSize = new LogRecord(physicalAddress).AllocatedSize; + if (totalSizes + allocatedSize > offset) + break; + totalSizes += allocatedSize; + physicalAddress += allocatedSize; + } + + return logicalAddress += totalSizes - offset; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + long GetPhysicalAddress(long currentAddress, long headAddress, long currentPage, long offset) + => currentAddress >= headAddress || assumeInMemory + ? hlogBase.GetPhysicalAddress(currentAddress) + : frame.GetPhysicalAddress(currentPage, offset); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + long GetPhysicalAddressAndAllocatedSize(long currentAddress, long headAddress, long currentPage, long offset, out long allocatedSize) + { + var physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); + + // We are just getting inline sizes so no need for ObjectIdMap + var logRecord = new LogRecord(physicalAddress); + allocatedSize = logRecord.AllocatedSize; + return logRecord.physicalAddress; + } + #endregion TODO Unify with SpanByteScanIterator + + /// + /// Get next record in iterator + /// + /// True if record found, false if end of scan + public bool GetNext() + { + while (true) + { + if (!InitializeGetNextAndAcquireEpoch(out var stopAddress)) + return false; + + try + { + if (!LoadPageIfNeeded(out var headAddress, out var currentPage, stopAddress)) + return false; + + var offset = hlogBase.GetOffsetOnPage(currentAddress); + var physicalAddress = GetPhysicalAddressAndAllocatedSize(currentAddress, headAddress, currentPage, offset, out var allocatedSize); + var recordInfo = LogRecord.GetInfo(physicalAddress); + + // If record does not fit on page, skip to the next page. Offset should always be at least PageHeader.Size; if it's zero, it means + // our record size aligned perfectly with end of page, so we must move to the next page (skipping its PageHeader). + if (offset == 0 || offset + allocatedSize > hlogBase.PageSize) + { + var nextPage = hlogBase.GetPage(currentAddress); + nextAddress = hlogBase.GetFirstValidLogicalAddressOnPage(offset == 0 ? nextPage : nextPage + 1); + continue; + } + + nextAddress = currentAddress + allocatedSize; + + var skipOnScan = !includeClosedRecords && recordInfo.SkipOnScan; + if (skipOnScan || recordInfo.IsNull) + continue; + + if (currentAddress >= headAddress || assumeInMemory) + { + // TODO: for this PR we always buffer the in-memory records; pull iterators require it, and currently push iterators are implemented on top of pull. + // Copy the entire record into bufferPool memory so we don't have a ref to log data outside epoch protection. + OperationStackContext stackCtx = default; + try + { + // Lock to ensure no value tearing while copying to temp storage. + if (currentAddress >= headAddress && store is not null) + { + var logRecord = hlogBase._wrapper.CreateLogRecord(currentAddress, physicalAddress); + store.LockForScan(ref stackCtx, logRecord); + } + + if (recordBuffer == null) + recordBuffer = hlogBase.bufferPool.Get((int)allocatedSize); + else if (recordBuffer.AlignedTotalCapacity < (int)allocatedSize) + { + recordBuffer.Return(); + recordBuffer = hlogBase.bufferPool.Get((int)allocatedSize); + } + + // These objects are still alive in the log, so do not dispose the value object if any. + // Don't pass the recordBuffer to diskLogRecord; we reuse that here. + var remapPtr = recordBuffer.GetValidPointer(); + Buffer.MemoryCopy((byte*)physicalAddress, remapPtr, allocatedSize, allocatedSize); + var memoryLogRecord = hlogBase._wrapper.CreateRemappedLogRecordOverPinnedTransientMemory(currentAddress, (long)remapPtr); + diskLogRecord = new DiskLogRecord(in memoryLogRecord); + } + finally + { + if (stackCtx.recSrc.HasLock) + store.UnlockForScan(ref stackCtx); + } + } + else + { + // We advance a record at a time in the IO frame so set the diskLogRecord to the current frame offset and advance nextAddress. + // DiskLogRecord.Dispose() invokes IHeapObject.Dispose on any deserialized value object for all callers + // (pending-op ctx, scan iteration, cluster streaming), unless the object is transferred out (e.g. via CopyToTail). + var logRecord = new LogRecord(physicalAddress, hlogBase._wrapper.TransientObjectIdMap); + diskLogRecord = new(logRecord); + // Fire OnDiskRead so app can invalidate stale TreeHandles, etc., on records loaded from disk. + if (hlogBase.storeFunctions.CallOnDiskRead) + hlogBase.storeFunctions.OnDiskRead(ref diskLogRecord.logRecord); + } + } + finally + { + // Success + epoch?.Suspend(); + } + + return true; + } + } + + /// + /// Get previous record and keep the epoch held while we call the user's scan functions + /// + /// True if record found, false if end of scan + bool IPushScanIterator.BeginGetPrevInMemory(TKey key, out LogRecord logRecord, out bool continueOnDisk) + { + while (true) + { + // "nextAddress" is reused as "previous address" for this operation. + currentAddress = nextAddress; + var headAddress = hlogBase.HeadAddress; + if (currentAddress < headAddress) + { + logRecord = default; + continueOnDisk = currentAddress >= hlogBase.BeginAddress; + return false; + } + + epoch?.Resume(); + + logRecord = hlogBase._wrapper.CreateLogRecord(currentAddress); + nextAddress = logRecord.Info.PreviousAddress; + + // Do not SkipOnScan here; we Seal previous versions. + if (logRecord.Info.IsNull || !hlogBase.storeFunctions.KeysEqual(logRecord, key)) + { + epoch?.Suspend(); + continue; + } + + // Success; defer epoch?.Suspend(); to EndGet + continueOnDisk = false; + return true; + } + } + + void IPushScanIterator.EndGetPrevInMemory() => epoch?.Suspend(); + + #region ISourceLogRecord + /// + public ref RecordInfo InfoRef => ref diskLogRecord.InfoRef; + /// + public RecordInfo Info => diskLogRecord.Info; + + /// + public byte RecordType => diskLogRecord.RecordType; + + /// + public ReadOnlySpan Namespace => diskLogRecord.Namespace; + + /// + public ObjectIdMap ObjectIdMap => diskLogRecord.ObjectIdMap; + + /// + public bool IsSet => diskLogRecord.IsSet; + + /// + public ReadOnlySpan Key => diskLogRecord.Key; + + /// + public bool IsPinnedKey => diskLogRecord.IsPinnedKey; + + /// + public byte* PinnedKeyPointer => diskLogRecord.PinnedKeyPointer; + + /// + public OverflowByteArray KeyOverflow + { + get => diskLogRecord.KeyOverflow; + set => diskLogRecord.KeyOverflow = value; + } + + #region IKey + public bool IsPinned => false; + + public ReadOnlySpan KeyBytes + => Key; + + public bool HasNamespace => diskLogRecord.HasNamespace; + + public ReadOnlySpan NamespaceBytes => diskLogRecord.NamespaceBytes; + #endregion + + /// + public Span ValueSpan => diskLogRecord.ValueSpan; + + /// + public IHeapObject ValueObject => diskLogRecord.ValueObject; + + /// + public bool IsPinnedValue => diskLogRecord.IsPinnedValue; + + /// + public byte* PinnedValuePointer => diskLogRecord.PinnedValuePointer; + + /// + public OverflowByteArray ValueOverflow + { + get => diskLogRecord.ValueOverflow; + set => diskLogRecord.ValueOverflow = value; + } + + /// + public SpanByteAndMemory ValueSpanByteAndMemory => diskLogRecord.ValueSpanByteAndMemory; + + /// + public long ETag => diskLogRecord.ETag; + + /// + public long Expiration => diskLogRecord.Expiration; + + /// + public void ClearValueIfHeap() { } // Not relevant for "iterator as logrecord" + + /// + public bool IsMemoryLogRecord => false; + + /// + public unsafe ref LogRecord AsMemoryLogRecordRef() => throw new InvalidOperationException("Cannot cast a DiskLogRecord to a memory LogRecord."); + + /// + public bool IsDiskLogRecord => true; + + /// + public unsafe ref DiskLogRecord AsDiskLogRecordRef() => ref Unsafe.AsRef(in diskLogRecord); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordFieldInfo GetRecordFieldInfo() => diskLogRecord.GetRecordFieldInfo(); + + /// + public int AllocatedSize => diskLogRecord.AllocatedSize; + + /// + public int ActualSize => diskLogRecord.ActualSize; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long CalculateHeapMemorySize() => diskLogRecord.CalculateHeapMemorySize(); + #endregion // ISourceLogRecord + + /// + /// Dispose iterator + /// + public override void Dispose() + { + base.Dispose(); + if (diskLogRecord.IsSet) + hlogBase._wrapper.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + recordBuffer?.Return(); + recordBuffer = null; + //TODOobjDispose("Dispose objects in frame"); + frame?.Dispose(); + } + + internal override void AsyncReadPageFromDeviceToFrame(CircularDiskReadBuffer readBuffers, long readPage, long untilAddress, TContext context, out CountdownEvent completed, + long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) + => hlogBase.AsyncReadPageFromDeviceToFrame(readBuffers, readPage, untilAddress, AsyncReadPageFromDeviceToFrameCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice, cts); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskReadBuffer.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskReadBuffer.cs new file mode 100644 index 00000000000..58bcb072f6f --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskReadBuffer.cs @@ -0,0 +1,270 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + /// + /// This class drives object-deserialization reading from the disk. It has multiple buffers and reads buffers ahead of the current one + /// ahead while deserialization logic is running. + public class CircularDiskReadBuffer : IDisposable + { + internal readonly SectorAlignedBufferPool bufferPool; + internal readonly int bufferSize; + internal readonly IDevice objectLogDevice; + internal readonly ILogger logger; + + DiskReadBuffer[] buffers; + int currentIndex; + + bool disposed; + + /// Device address to do the next read from (segment and offset); set at the start of a record by + /// and incremented with each buffer read; all of these should be aligned to sector size, so this address remains sector-aligned. + internal ObjectLogFilePositionInfo nextFileReadPosition; + + /// Track the remaining length to be read for one or more records for Object values, and we can also read some or all of Overflow values into the buffer. + ulong unreadLengthRemaining; + + internal CircularDiskReadBuffer(SectorAlignedBufferPool bufferPool, int bufferSize, int numBuffers, IDevice objectLogDevice, ILogger logger) + { + this.bufferPool = bufferPool; + this.bufferSize = bufferSize; + this.objectLogDevice = objectLogDevice; + this.logger = logger; + + buffers = new DiskReadBuffer[numBuffers]; + currentIndex = 0; + } + + internal DiskReadBuffer GetCurrentBuffer() + { + if (disposed) + throw new ObjectDisposedException(nameof(CircularDiskReadBuffer)); + return buffers[currentIndex]; + } + + int GetNextBufferIndex(int curIndex) + { + var index = curIndex + 1; + return index >= buffers.Length ? 0 : index; + } + + private DiskReadBuffer CreateBuffer(int bufferIndex) + { + DiskReadBuffer buffer = new(bufferPool.Get(bufferSize), objectLogDevice, logger); + buffers[bufferIndex] = buffer; + return buffer; + } + + /// + /// Prepare the and local variables to read the next buffer (or as much of it as we need) and issue the read. + /// This is called by OnBeginReadRecords and when we are leaving a buffer with more data, to fill that buffer so it is available when we + /// wrap around to it again. For both of these, we do not have to worry that there is pending IO in the buffer. + /// + /// The index into of the that will do the reading + /// The "actual" read start position in the buffer (relative to start of buffer), which will + /// become the "current position" of the buffer + private void DoReadBuffer(int bufferIndex, int unalignedReadStartPosition) + { + var buffer = buffers[bufferIndex]; + if (buffer is null) + buffer = CreateBuffer(bufferIndex); + else + { + Debug.Assert(buffer.countdownEvent.CurrentCount == 0, $"Unexpected countdownEvent.CurrentCount ({buffer.countdownEvent.CurrentCount}) when preparing to read into buffer"); + buffer.Initialize(); + } + + var alignedReadStartPosition = RoundDown(unalignedReadStartPosition, (int)objectLogDevice.SectorSize); + var bufferStartPosition = unalignedReadStartPosition - alignedReadStartPosition; + + // See how much to read. We have two limits: the total size requested for this ReadAsync operation, and the segment size. + var unalignedReadLength = bufferSize - alignedReadStartPosition; + if ((ulong)unalignedReadLength > unreadLengthRemaining) + unalignedReadLength = (int)unreadLengthRemaining; + + Debug.Assert(IsAligned(nextFileReadPosition.Offset, (int)objectLogDevice.SectorSize), $"filePosition.Offset ({nextFileReadPosition.Offset}) is not sector-aligned"); + var segmentIsComplete = false; + if (nextFileReadPosition.Offset + (ulong)unalignedReadLength >= nextFileReadPosition.SegmentSize) + { + unalignedReadLength = (int)(nextFileReadPosition.SegmentSize - nextFileReadPosition.Offset); + Debug.Assert(IsAligned(unalignedReadLength, (int)objectLogDevice.SectorSize), $"unalignedReadLength ({unalignedReadLength}) is not sector-aligned at segment end"); + segmentIsComplete = true; + } + + // We may not have had a sector-aligned amount of remaining unread data. + var alignedReadLength = RoundUp(unalignedReadLength, (int)objectLogDevice.SectorSize); + buffer.ReadFromDevice(nextFileReadPosition, bufferStartPosition, (uint)alignedReadLength, ReadFromDeviceCallback); + + // Advance the filePosition. This used aligned read length so may advance it past end of record but that's OK because + // filePosition is for the "read buffer-sized chunks" logic while data transfer via Read() uses buffer.currentPosition. + // Note: If segmentIsComplete, this increment results in nextFileReadPosition.Offset == SegmentSize, which will mask off to a 0. + nextFileReadPosition.Offset += (uint)alignedReadLength; + + Debug.Assert(nextFileReadPosition.Offset <= nextFileReadPosition.SegmentSize, $"filePosition.Offset ({nextFileReadPosition.Offset}) must be <= filePosition.SegmentSize ({nextFileReadPosition.SegmentSize})"); + if (segmentIsComplete) + nextFileReadPosition.AdvanceToNextSegment(); + + unreadLengthRemaining -= (uint)unalignedReadLength; + } + + /// + /// Called when one or more records are to be read via ReadAsync. + /// + /// The initial file position to read + /// The cumulative length of all object-log entries for the span of records to be read. We read ahead for all record + /// in the ReadAsync call. + internal void OnBeginReadRecords(ObjectLogFilePositionInfo startFilePosition, ulong totalLength) + { + if (disposed) + throw new ObjectDisposedException(nameof(CircularDiskReadBuffer)); + + Debug.Assert(totalLength > 0, "TotalLength cannot be 0"); + nextFileReadPosition = startFilePosition; + unreadLengthRemaining = totalLength; + + // Initialize all buffers + for (var ii = 0; ii < buffers.Length; ii++) + buffers[ii]?.Initialize(); + currentIndex = 0; + + // Do an initial read to fill the buffers, at least as much as we have. Again, totalLength applies to all records in the ReadAsync range, + // whether one or many. First align the initial read. recordStartPosition is the padding between rounded-down-to-align-readStart and recordStart. + var alignedReadPosition = RoundDown(nextFileReadPosition.Offset, (int)objectLogDevice.SectorSize); + var recordStartPosition = (int)(nextFileReadPosition.Offset - alignedReadPosition); + unreadLengthRemaining += (uint)recordStartPosition; + nextFileReadPosition.Offset -= (uint)recordStartPosition; + + // Load all the buffers as long as we have more unread data. Leave currentIndex at 0. + for (var ii = 0; ii < buffers.Length; ii++) + { + if (unreadLengthRemaining == 0) + break; + DoReadBuffer(ii, recordStartPosition); + recordStartPosition = 0; // After the first read, subsequent reads start on an aligned address + } + } + + /// + /// Called when one or more records with Objects have been read and via ReadAsync, e.g. being processed by AsyncReadPageWithObjectsCallback, + /// and we have completed reading and deserializing those objects. + /// + internal void OnEndReadRecords() + { + for (var ii = 0; ii < buffers.Length; ii++) + { + Debug.Assert(buffers[ii] is null || !buffers[ii].HasInFlightRead, $"All reads should have been completed by OnEndReadRecords()"); + } + } + + internal bool OnBeginRecord(ObjectLogFilePositionInfo recordFilePosition) + { + var buffer = buffers[currentIndex] ?? throw new TsavoriteException($"Internal error in read buffer sequencing; empty buffer[{currentIndex}] encountered with unreadLengthRemaining {unreadLengthRemaining}"); + + // Because each partial flush ends with a sector-aligning write, we may have a record start position greater than our ongoing buffer.currentPosition + // incrementing. It should never be less. recordFilePosition is only guaranteed to be sector-aligned if it's the first record after a partial flush. + if (!buffer.HasData && !buffer.WaitForDataAvailable()) + return false; + + while (true) + { + var bufferFilePosition = buffer.GetCurrentFilePosition(); + Debug.Assert(recordFilePosition.word >= bufferFilePosition.word, $"Record file position ({recordFilePosition}) should be >= ongoing position {bufferFilePosition}"); + Debug.Assert(recordFilePosition.SegmentId == bufferFilePosition.SegmentId, $"Record file segment ({recordFilePosition.SegmentId}) should == ongoing position {bufferFilePosition.SegmentId}"); + var increment = recordFilePosition - bufferFilePosition; + Debug.Assert(increment < objectLogDevice.SectorSize, $"Increment {increment} must be less than SectorSize ({objectLogDevice.SectorSize})"); + + // We might cleanly align to the start of the next buffer, if there was a flush that ended on a buffer boundary. + // Otherwise, we should always be within the current buffer. We should only do this "continue" once. + if (buffer.currentPosition + (int)increment < buffer.endPosition) + { + buffer.currentPosition += (int)increment; + break; + } + + Debug.Assert(buffer.currentPosition + (int)increment == buffer.endPosition, $"Increment {increment} overflows buffer (curPos {buffer.currentPosition}, endPos {buffer.endPosition}) by more than alignment"); + if (!MoveToNextBuffer(out buffer)) + break; + } + return true; + } + + /// + /// Begin the deserialization process for a single record. + /// + internal void OnBeginDeserialize() + { + // Currently nothing + } + + /// + /// Move to the next buffer and see if it has data. + /// + /// The next buffer + /// + internal bool MoveToNextBuffer(out DiskReadBuffer nextBuffer) + { + // If we have more data to read, "backfill" this buffer with a read before departing it, else initialize it. + if (unreadLengthRemaining > 0) + DoReadBuffer(currentIndex, unalignedReadStartPosition: 0); + else + buffers[currentIndex].Initialize(); + + // Move to the next buffer and wait for any in-flight read to complete. If there is no pending IO and the buffer is + // empty, we are done with this read op. + currentIndex = GetNextBufferIndex(currentIndex); + nextBuffer = buffers[currentIndex]; + if (nextBuffer is not null && nextBuffer.WaitForDataAvailable()) + return true; + + Debug.Assert(unreadLengthRemaining == 0, $"unreadLengthRemaining ({unreadLengthRemaining}) was not 0 when WaitForDataAvailable returned false"); + return false; + } + + internal void ReadFromDeviceCallback(uint errorCode, uint numBytes, object context) + { + if (errorCode != 0) + logger?.LogError($"{nameof(ReadFromDeviceCallback)} error: {{errorCode}}", errorCode); + + // Finish setting up the buffer + var buffer = (DiskReadBuffer)context; + + buffer.endPosition += (int)numBytes; + if (buffer.endPosition == 0) + Debug.Assert(buffer.currentPosition == 0, $"buffer.currentPosition ({buffer.currentPosition}) must be 0 if buffer.endPosition ({buffer.endPosition}) is 0"); + else + Debug.Assert(buffer.endPosition > buffer.currentPosition, $"buffer.endPosition ({buffer.endPosition}) must be >= buffer.currentPosition ({buffer.currentPosition})"); + + // Signal the buffer's event to indicate the data is available. + _ = buffer.countdownEvent.Signal(); + } + + public void Dispose() + { + disposed = true; + + // Atomic swap to avoid clearing twice. + var localBuffers = Interlocked.Exchange(ref buffers, null); + if (localBuffers == null) + return; + + for (var ii = 0; ii < localBuffers.Length; ii++) + localBuffers[ii]?.Dispose(); + + // Restore the now-cleared buffers array. + //buffers = localBuffers; + } + + /// + public override string ToString() + => $"currIdx {currentIndex}; bufSize {bufferSize}; filePosition {nextFileReadPosition}; SecSize {(int)objectLogDevice.SectorSize}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskWriteBuffer.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskWriteBuffer.cs new file mode 100644 index 00000000000..0fd422c0537 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/CircularDiskWriteBuffer.cs @@ -0,0 +1,295 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + /// + /// This class drives object-serialization writing to the disk. It is reused by multiple "partial flushes": ranges on a single page (rare) or + /// full pages. We create one instance for all ranges of a top-level Flush() call; each partial range will call , + /// do its flushes, and call . This reuse makes the most efficient use of the buffer allocations. It also + /// requires tracking of "in-flight" device writes at multiple levels: + /// + /// : A tracks how many in-flight device writes are associated with that buffer. + /// : A separate instance tracks total in-flight device writes + /// for each and pair, both at the level + /// and without buffer association, such as direct writes from pinned byte[] spans. This lets us call the main-page callback once the main-page + /// and all associated object log writes are complete. + /// + /// This also contains a counter of how many instances are active (i.e. how many partial flush + /// completion write batches are in-flight); when this hits 0, we can call . + /// + /// + /// + public class CircularDiskWriteBuffer : IDisposable + { + internal readonly SectorAlignedBufferPool bufferPool; + internal readonly int bufferSize; + internal readonly IDevice device; + internal readonly ILogger logger; + + DiskWriteBuffer[] buffers; + + /// Index of the current buffer + int currentIndex; + + /// Device address to write to (segment and offset); incremented with each buffer flush or out-of-line write by the caller; all of these should be aligned to sector size, + /// so this address remains sector-aligned. + internal ObjectLogFilePositionInfo filePosition; + + /// If true, we own the file position and have initialized it, and we ignore the allocator's on PartialFlush begin and end. This is done for + /// writes to an objectLog device that is not the allocator's. If false, then we are using the allocator's and will initialize from it when begging a PartialFlush + /// and update it when ending the PartialFlush. + bool ownFilePosition; + + /// Countdown event for global count of all buffers and all direct writes. Also triggers the external callback of a partial-flush sequence. + /// This is passed to all disk-write operations; multiple pending flushes may be in-flight with the callback unset; when the final flush (which may be a buffer-span, a direct write, or the + /// final sector-aligning partial-flush completion flush), it allows the final pending flush to complete to know it *is* the final one and the callback can be called. + internal CountdownCallbackAndContext countdownCallbackAndContext; + + /// If true, has been called. Coordinates with to indicate when we can call . + bool disposed; + + /// Tracks the number of in-flight partial flush completion writes. Coordinates with to indicate when we can call . + long numInFlightWrites; + + internal CircularDiskWriteBuffer(SectorAlignedBufferPool bufferPool, int bufferSize, int numBuffers, IDevice device, ILogger logger) + { + this.bufferPool = bufferPool; + this.bufferSize = bufferSize; + this.device = device; + this.logger = logger; + + buffers = new DiskWriteBuffer[numBuffers]; + currentIndex = 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal DiskWriteBuffer GetCurrentBuffer() => buffers[currentIndex]; + + internal DiskWriteBuffer MoveToAndInitializeNextBuffer() + { + currentIndex = (currentIndex + 1) & (buffers.Length - 1); + return GetAndInitializeCurrentBuffer(); + } + + private DiskWriteBuffer GetOrAllocateCurrentBuffer() + => buffers[currentIndex] ?? (buffers[currentIndex] = new DiskWriteBuffer(bufferPool.Get(bufferSize), device, logger)); + + internal DiskWriteBuffer GetAndInitializeCurrentBuffer() + { + var buffer = GetOrAllocateCurrentBuffer(); + + // By this time the next device file write position has been updated, even if some of the preceding writes are still in-flight. + var endPosition = filePosition.SegmentSize - filePosition.Offset; + if (endPosition > (uint)bufferSize) + endPosition = (uint)bufferSize; + buffer.WaitUntilFreeAndInitialize((int)endPosition); + return buffer; + } + + internal ObjectLogFilePositionInfo GetNextRecordStartPosition() + { + var startFilePos = filePosition; + var buffer = GetCurrentBuffer(); + if (buffer is not null) + startFilePos.Offset += (uint)(buffer.currentPosition - buffer.flushedUntilPosition); + return startFilePos; + } + + internal void InitializeOwnObjectLogFilePosition(long segmentSize) + { + filePosition = new(word: 0, segSizeBits: GetLogBase2(segmentSize)); + ownFilePosition = true; + } + + /// Resets start positions for the next partial flush. + internal DiskWriteBuffer OnBeginPartialFlush(ObjectLogFilePositionInfo filePos) + { + // We start every partial flush with the first buffer, starting at position 0. + if (!ownFilePosition) + { + filePosition = filePos; + Debug.Assert(IsAligned(filePosition.Offset, (int)device.SectorSize), $"OnBeginPartialFlush file flush position ({filePosition}) is not sector-aligned"); + } + + disposed = false; + currentIndex = 0; + countdownCallbackAndContext = new(); + return GetAndInitializeCurrentBuffer(); + } + + /// Called when a Write is completed. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void OnRecordComplete() + { + // Currently nothing to do. We do not do end-of-record alignment in the ObjectLog file. + } + + /// + /// Finish all the current partial flush, including flushing any as-yet-unflushed data in the current buffer then calling the caller's callbacks + /// so flushedUntilAddresses can be updated. When this function exits, there will be IOs in flight. + /// + /// This write to the device is sector-aligned, which means the next fragment will probably rewrite the sector, since the currentPosition is probably + /// somewhere in the middle of the sector. + /// Starting pointer of the main log page span to write + /// Length of the main log page span to write + /// The main log device to write to + /// The offset in the main log to write at + /// Callback sent to the initial Flush() command. Called when we are done with this partial flush operation. + /// It usually signals the event so the caller knows the flush is complete and it can continue. + /// Context sent to . + /// The ending file position after the partial flush is complete + internal unsafe void OnPartialFlushComplete(byte* mainLogPageSpanPtr, int mainLogPageSpanLength, IDevice mainLogDevice, ulong alignedMainLogFlushAddress, + DeviceIOCompletionCallback externalCallback, object externalContext, ref ObjectLogFilePositionInfo endObjectLogFilePosition) + { + // Lock this with a reference until we have set the callback and issue the write. This callback is for the main log page write, and + // when the countdownCallbackAndContext.Decrement hits 0 again, we're done with this partial flush range and will call the external callback. + countdownCallbackAndContext.Increment(); + countdownCallbackAndContext.Set(externalCallback, externalContext, (uint)mainLogPageSpanLength); + + // Issue the last ObjectLog write for this partial flush. + var buffer = GetCurrentBuffer(); + Debug.Assert(IsAligned(alignedMainLogFlushAddress, (int)device.SectorSize), "alignedMainLogFlushAddress is not aligned to sector size"); + Debug.Assert(IsAligned(buffer.flushedUntilPosition, (int)device.SectorSize), $"flushedUntilPosition {buffer.flushedUntilPosition} is not sector-aligned"); + Debug.Assert(buffer.currentPosition >= buffer.flushedUntilPosition, $"buffer.currentPosition {buffer.currentPosition} must be >= buffer.flushedUntilPosition {buffer.flushedUntilPosition}"); + + if (buffer.currentPosition > buffer.flushedUntilPosition) + { + // We have something to flush. First ensure sector-alignment of the flush; we'll "waste" some space to do so. This is necessary to avoid rewriting sectors, + // which can be a problem for some devices due to inefficiencies in rewriting or inability to back up (or both). + var sectorEnd = RoundUp(buffer.currentPosition, (int)device.SectorSize); + if (sectorEnd > buffer.currentPosition) + { + // Prepare to flush the final piece to disk by zero-initializing the sector-alignment padding. + new Span(buffer.memory.GetValidPointer() + buffer.currentPosition, sectorEnd - buffer.currentPosition).Clear(); + buffer.currentPosition = sectorEnd; + } + + // Now write the buffer to the device. + _ = Interlocked.Increment(ref numInFlightWrites); + buffer.FlushToDevice(ref filePosition, FlushToDeviceCallback, CreateDiskWriteCallbackContext()); + } + + // Update the object log file position for the caller, unless we are using our own. + if (!ownFilePosition) + endObjectLogFilePosition = filePosition; + + // Write the main log page to the mainLogDevice. + FlushToMainLogDevice(mainLogPageSpanPtr, mainLogPageSpanLength, mainLogDevice, alignedMainLogFlushAddress, CreateDiskWriteCallbackContext()); + + // We added a count to countdownCallbackAndContext at the start, and the callback state creation also added a count. Remove the one we added at the start. + // If the write in FlushToMainLogDeviced completed fast, this decrement here may be the final one. + _ = countdownCallbackAndContext.Decrement(); + } + + internal DiskWriteCallbackContext CreateDiskWriteCallbackContext() => new(countdownCallbackAndContext); + internal DiskWriteCallbackContext CreateDiskWriteCallbackContext(RefCountedPinnedGCHandle refGcHandle) => new(countdownCallbackAndContext, refGcHandle); + internal DiskWriteCallbackContext CreateDiskWriteCallbackContext(GCHandle gcHandle) => new(countdownCallbackAndContext, gcHandle); + + /// Flush the current buffer. If we are in an operation that filled previous buffers, those will have been flushed already by earlier calls. + internal void FlushCurrentBuffer() + { + var buffer = GetCurrentBuffer(); + var writeCallbackContext = CreateDiskWriteCallbackContext(); + _ = Interlocked.Increment(ref numInFlightWrites); + buffer.FlushToDevice(ref filePosition, FlushToDeviceCallback, writeCallbackContext); + } + + /// Flush to disk for a span that is not associated with a particular buffer, such as fully-interior spans of a large overflow key or value. + internal unsafe void FlushToDevice(byte* spanPtr, int spanLength, DiskWriteCallbackContext writeCallbackContext) + { + Debug.Assert(IsAligned(spanLength, (int)device.SectorSize), "Span is not aligned to sector size"); + + _ = Interlocked.Increment(ref numInFlightWrites); + device.WriteAsync((IntPtr)spanPtr, filePosition.SegmentId, filePosition.Offset, (uint)spanLength, FlushToDeviceCallback, writeCallbackContext); + filePosition.Offset += (uint)spanLength; + } + + /// Flush a main-log page span to the main log device. This lets us coordinate the callbacks to be called on the last write, regardless of whether + /// that write is to main or object log. + internal unsafe void FlushToMainLogDevice(byte* spanPtr, int spanLength, IDevice mainLogDevice, ulong alignedMainLogFlushAddress, DiskWriteCallbackContext writeCallbackContext) + { + Debug.Assert(IsAligned(spanLength, (int)device.SectorSize), "Span is not aligned to sector size"); + Debug.Assert(IsAligned(alignedMainLogFlushAddress, (int)device.SectorSize), "mainLogAlignedDeviceOffset is not aligned to sector size"); + + _ = Interlocked.Increment(ref numInFlightWrites); + mainLogDevice.WriteAsync((IntPtr)spanPtr, alignedMainLogFlushAddress, (uint)spanLength, FlushToDeviceCallback, writeCallbackContext); + } + + private void FlushToDeviceCallback(uint errorCode, uint numBytes, object context) + { + if (errorCode != 0) + logger?.LogError($"{nameof(FlushToDeviceCallback)} error: {{errorCode}}", errorCode); + + // Try to signal the event; if we have finished the last write for this buffer, the count will hit zero and Set the event so any Waits we do on it will succeed. + // We don't wait on the result of individual device writes; we may wait due to a call (e.g. FlushAndEvict()) with a "wait" parameter set to true. + var writeCallbackContext = (DiskWriteCallbackContext)context; + + // If this returns 0 we have finished all in-flight writes for the writeCallbackContext.countdownCallbackAndContext instance, but there may be more instances + // active even if we have been disposed, so adjust and check the global count, and if *that* is zero, check the disposed state (being disposed ensures that no + // further partial flush ranges will be sent). + _ = Interlocked.Decrement(ref numInFlightWrites); + if (writeCallbackContext.Release() == 0 && numInFlightWrites == 0 && disposed) + ClearBuffers(); + } + + /// + public void Dispose() + { + // If we are here, then we have returned from the partial-flush loop and will not be incrementing numInFlightRangeBatches again, so if it is 0 we are + // done and can free the buffers. If numInFlightWrites > 0, then we have pending writes and will free the buffers when they complete, but will not initiate + // any further writes. For this class, "disposed" means "we're done issuing writes". And filePosition must be preserved; checkpoints will retrieve it later, + // and chained partial flushes will append to it. + disposed = true; + if (numInFlightWrites == 0) + ClearBuffers(); + } + + private void ClearBuffers() + { + // We should have no data to flush--the last partial flush should have ended with PartialFlushComplete which flushes the last of the data for that flush fragment, + // and we wait for that to finish before calling the caller's callback. However, we may have to wait for flushed data to complete; this may be from either the + // just-completed partial-flush range, or even from the range before that if the most recent range did not use all buffers; at the time this is called there may + // be one or more in-flight countdownCallbackAndContexts. So we just wait. + + // Atomic swap to avoid clearing twice, because the 'disposed' testing isn't atomic. + var localBuffers = Interlocked.Exchange(ref buffers, null); + if (localBuffers == null) + return; + + for (var ii = 0; ii < localBuffers.Length; ii++) + { + ref var buffer = ref localBuffers[ii]; + if (buffer is not null) + { + buffer.Wait(); + buffer.Dispose(); + buffer = null; + } + } + + // Restore the now-cleared buffers array. + buffers = localBuffers; + } + + /// + public override string ToString() + { + var result = $"currIdx {currentIndex}; bufSize {bufferSize}; filePos {filePosition}, SecSize {(int)device.SectorSize}"; + var buffer = GetCurrentBuffer(); + if (buffer is not null) + result += $"; currBuf: [{buffer}]"; + return result; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskReadBuffer.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskReadBuffer.cs new file mode 100644 index 00000000000..96574a83fb3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskReadBuffer.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ + internal sealed unsafe class DiskReadBuffer : IDisposable + { + internal readonly IDevice device; + internal readonly ILogger logger; + + /// Signals when reads are complete. Allows multiple reads of buffer subsections to be enqueed, and uses a ManualResetEvent + /// so it remains signaled until Reset() is called (currently only one read for a single span of the buffer is done). + internal CountdownEvent countdownEvent; + + /// The buffer to read (part of) the page image into. + internal SectorAlignedMemory memory; + + /// + /// This is the initialization value for ; it means there is no data available for this buffer and no + /// in-flight read (we issue reads ahead of the buffer-array traversal, so this means that by the time we got to this buffer all + /// the data had already been read. + /// + const int NoPosition = -1; + + /// Current read position (we do not support write in this buffer). This class only supports Read and no Seek, + /// so currentPosition is always where will read from next. + /// This will be either 0 or greater than or equal to . + internal int currentPosition; + + /// Non-inclusive last position in this buffer; the number of byte read. If equals this, then we are out of space and + /// must move to the next buffer. + internal int endPosition; + + /// + /// The starting position in the file that we read this buffer from. + /// + internal ObjectLogFilePositionInfo startFilePosition; + + internal int AvailableLength => endPosition - currentPosition; + + internal ReadOnlySpan AvailableSpan => new(memory.GetValidPointer() + currentPosition, endPosition - currentPosition); + + internal DiskReadBuffer(SectorAlignedMemory memory, IDevice device, ILogger logger) + { + this.memory = memory; + countdownEvent = new CountdownEvent(0); // Start with 0; we'll increment at the time of read + this.device = device; + this.logger = logger; + Initialize(); + } + + internal void Initialize() + { + currentPosition = endPosition = NoPosition; + } + + internal ReadOnlySpan GetTailSpan(int start) => new(memory.GetValidPointer() + start, currentPosition - start); + + /// + /// Read the first chunk of an Object deserialization from the device. + /// + /// Sector-aligned position in the device + /// Start position in the buffer (relative to start of buffer) + /// Number of bytes to read + /// The callback. + internal void ReadFromDevice(ObjectLogFilePositionInfo filePosition, int startPosition, uint alignedReadLength, DeviceIOCompletionCallback callback) + { + IncrementOrResetCountdown(ref countdownEvent); + startFilePosition = filePosition; + + currentPosition = startPosition; + endPosition = 0; + device.ReadAsync(filePosition.SegmentId, filePosition.Offset, (IntPtr)memory.aligned_pointer, (uint)alignedReadLength, callback, context: this); + } + + internal static void IncrementOrResetCountdown(ref CountdownEvent countdownEvent) => DiskWriteBuffer.IncrementOrResetCountdown(ref countdownEvent); + + internal bool HasData => endPosition > 0; + + internal bool WaitForDataAvailable() + { + // Because we have issued reads ahead of the buffer wrap, if the currentPosition is NoPosition, we're done. + if (currentPosition == NoPosition) + return false; + if (!HasData) + countdownEvent.Wait(); + return true; + } + + internal bool HasInFlightRead => countdownEvent is not null && !countdownEvent.IsSet; + + internal ObjectLogFilePositionInfo GetCurrentFilePosition() + { + var bufferFilePos = startFilePosition; + bufferFilePos.Offset += (uint)currentPosition; + + // We only read from one segment into one buffer, so we should never exceed the segment size with this increment. + Debug.Assert(bufferFilePos.Offset < bufferFilePos.SegmentSize, $"Incremented bufferFilePos.Offset {bufferFilePos.Offset} should be < bufferFilePos.SegmentSize {bufferFilePos.SegmentSize}"); + return bufferFilePos; + } + + public void Dispose() + { + memory?.Return(); + memory = null; + countdownEvent?.Dispose(); + countdownEvent = null; + } + + /// + public override string ToString() + => $"currPos {currentPosition}; endPos {endPosition}; avLen {AvailableLength}; countDown {countdownEvent?.CurrentCount}; buf: {memory}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskWriteBuffer.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskWriteBuffer.cs new file mode 100644 index 00000000000..809d7a3f18c --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/DiskWriteBuffer.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; +using Microsoft.Extensions.Logging; + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + internal sealed unsafe class DiskWriteBuffer : IDisposable + { + /// Signals when writes are complete. Allows multiple writes of buffer subsections to be enqueed, and uses a ManualResetEvent + /// so it remains signaled until Reset() is called. + CountdownEvent countdownEvent; + + /// The buffer to build the page image for writing. + internal SectorAlignedMemory memory; + + /// Current write position (we do not support read in this buffer). This class only supports Write and no Seek, + /// so currentPosition equals the current length. Relevant for object serialization only; reset to 0 at start of DoSerialize(). + /// is the current length, since it is *past* the last byte copied to the buffer. + internal int currentPosition; + + /// Last position flushed to in this buffer (i.e. 0->flushedUntilPosition have been flushed). This allows using the buffer for + /// multiple small writes sandwiched around large internal direct writes from overflow. + /// If is less than then there is unflushed data in the buffer. + internal int flushedUntilPosition; + + /// The end position of the buffer. Usually the size of the buffer, but may be less if we're at the end of a segment. + /// This should always be sector-aligned, because we start the partial flush sector-aligned and this is either a bufferSize past + /// the previous buffer or a sector-aligned distance from the end of the segment; the latter may be less than the end of the buffer + /// due to directly writing internal spans for Keys and Values that are less than a full buffer size. + internal int endPosition; + + internal int RemainingCapacity => endPosition - currentPosition; // DiskPageHeader.Size is included in currentPosition + + /// The remaining space in the buffer, from to . + internal Span RemainingSpan => new(memory.GetValidPointer(), RemainingCapacity); + + internal readonly IDevice device; + internal readonly ILogger logger; + + internal DiskWriteBuffer(SectorAlignedMemory memory, IDevice device, ILogger logger) + { + this.memory = memory; + this.device = device; + this.logger = logger; + } + + internal void WaitUntilFreeAndInitialize(int endPosition) + { + Debug.Assert(IsAligned(endPosition, (int)device.SectorSize), $"endPosition {endPosition} is not sector-aligned"); + + // First wait for any pending write in this buffer to complete. If this is our first time in this buffer there won't be a CountdownEvent yet; + // we defer that because we may not need all the buffers in the circular buffer. + countdownEvent?.Wait(); + + // Initialize fields. + this.endPosition = endPosition; + memory.valid_offset = 0; + currentPosition = 0; + flushedUntilPosition = 0; + } + + internal static CountdownEvent IncrementOrResetCountdown(ref CountdownEvent countdownEvent) + { + if (countdownEvent is null) + countdownEvent = new(1); + else if (!countdownEvent.TryAddCount(1)) + { + // This means we've enqueued one or more earlier writes which have completed. + // First wait to be sure the callback has signaled the contained event, then Reset the event with a new count. + countdownEvent.Wait(); // This should usually be immediate + countdownEvent.Reset(1); + } + return countdownEvent; + } + + internal void FlushToDevice(ref ObjectLogFilePositionInfo filePosition, DeviceIOCompletionCallback callback, DiskWriteCallbackContext pageWriteCallbackContext) + { + Debug.Assert(currentPosition <= endPosition, $"currentPosition ({currentPosition}) cannot exceed endPosition ({endPosition})"); + + // We are flushing the buffer. currentPosition must already be sector-aligned; either it is at endPosition (which is always sector-aligned), + // which is the normal "buffer is full so flush it" handling, or it is less than endPosition which means it is called from one of: + // a. OnPartialFlushComplete, in which case the caller has sector-aligned it before calling this + // b. OverflowByteArray sector-aligning writes at the beginning or end, which means we copied a sector-aligned number of bytes to the buffer. + Debug.Assert(IsAligned(currentPosition, (int)device.SectorSize), $"currentPosition ({currentPosition}) is not sector-aligned"); + Debug.Assert(IsAligned(filePosition.Offset, (int)device.SectorSize), $"FlushToDevice starting file flush position ({filePosition}) is not sector-aligned"); + pageWriteCallbackContext.SetBufferCountdownEvent(IncrementOrResetCountdown(ref countdownEvent)); + + var flushLength = (uint)(currentPosition - flushedUntilPosition); + Debug.Assert(IsAligned(flushLength, (int)device.SectorSize), $"flushLength {flushLength} is not sector-aligned"); + Debug.Assert(flushLength <= filePosition.RemainingSizeInSegment, $"flushLength ({flushLength}) cannot be greater than filePosition.RemainingSize ({filePosition.RemainingSizeInSegment})"); + + var spanPtr = memory.GetValidPointer() + flushedUntilPosition; + device.WriteAsync((IntPtr)spanPtr, filePosition.SegmentId, filePosition.Offset, flushLength, callback, pageWriteCallbackContext); + flushedUntilPosition = currentPosition; + + // This does not use .Advance() because we are already checking boundary conditions and calling .AdvanceToNextSegment() in ObjectLogWriter. + filePosition.Offset += flushLength; + } + + internal void Wait() => countdownEvent?.Wait(); + + public void Dispose() + { + memory?.Return(); + memory = null; + + Debug.Assert(countdownEvent is null || countdownEvent.CurrentCount == 0, $"Unexpected count ({countdownEvent.CurrentCount}) remains"); + countdownEvent?.Dispose(); + countdownEvent = null; + } + + /// + public override string ToString() + { + var countdownString = countdownEvent?.CurrentCount.ToString() ?? "null"; + return $"currPos {currentPosition}; endPos {endPosition}; remCap {RemainingCapacity}; flushedUntilPos {flushedUntilPosition}; countDown {countdownString}; buf: {memory}"; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/IStreamBuffer.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/IStreamBuffer.cs new file mode 100644 index 00000000000..863ff0e4f29 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/IStreamBuffer.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; + +namespace Tsavorite.core +{ + /// + /// This interface abstracts the process of writing a full buffer to storage or network, or reading (up to) a certain number of bytes. + /// + public interface IStreamBuffer : IDisposable + { + /// Indicates that the value is continued in the next chunk, after the current length. + internal const int ValueChunkContinuationBit = 1 << 31; + + /// Indicates that the value is completed in the current chunk (there is no next chunk). + internal const int NoValueChunkContinuationBit = 0; + + /// The size of the buffer used for writing data to and reading it from the disk. Must be a sector multiple. + internal const int BufferSize = 1 << LogSettings.kMinObjectLogSegmentSizeBits; + + /// Initial IO size to read. + internal static int InitialIOSize => Environment.SystemPageSize; + + /// + /// We use these buffers for only read or only write operations, never both at the same time. + /// + bool IsForWrite { get; } + + /// + /// Write a full buffer to storage or network and reset the buffer to the starting position. Note that this may also reset the + /// underlying buffer pointer. + /// + /// Optional cancellation token + void FlushAndReset(CancellationToken cancellationToken = default); + + /// + /// Write span of bytes to the storage or network buffer. Actual flushing (e.g. to disk) is done as needed.. + /// + /// The data span to write to the device. + /// Optional cancellation token + /// This implements the standard Stream functionality, called from the Value Serializer + void Write(ReadOnlySpan data, CancellationToken cancellationToken = default); + + /// + /// Read more bytes from the disk or network, up to , and store in the buffer. It may not read all bytes + /// depending on the internal buffer management. + /// + /// The span to receive data from the device + /// Optional cancellation token + /// This implements the standard Stream functionality, called from the Value Serializer + /// The number of bytes read into , which may be less than . + int Read(Span destinationSpan, CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogFilePositionInfo.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogFilePositionInfo.cs new file mode 100644 index 00000000000..01281701933 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogFilePositionInfo.cs @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Represents the information about the segment and offset of a location within the object log file. + /// + internal struct ObjectLogFilePositionInfo + { + /// Indicates the word has not been set. + internal const ulong NotSet = ulong.MaxValue; + + /// Maximum number of bytes to use for segment + offset. 7 bytes gives a 72PB Object Log size range. + private const int NumSegmentAndOffsetBytes = 7; + + /// Maximum number of bits to use for segment + offset. 7 bytes gives a 72PB Object Log size range. + private const int NumSegmentAndOffsetBits = NumSegmentAndOffsetBytes * sizeof(long); + + /// Maximum number of bytes to use for segment + offset. 7 bytes gives a 72PB Object Log size range. + private const ulong SegmentAndOffsetMask = (1UL << NumSegmentAndOffsetBits) - 1; + + /// Object log segment size bits + internal int SegmentSizeBits; + + /// The word containing the data. + internal ulong word; + + internal readonly bool IsSet => SegmentSizeBits != 0; + internal readonly bool HasData => word != 0 && word != NotSet; + + /// + /// Default initialization; leaves IsSet false. ObjectLogFilePositionInfo must be instantiated by new(), not default; we don't have arrays of this, + /// and fields are initalized with some overload of new(). + /// + public ObjectLogFilePositionInfo() + { + SegmentSizeBits = 0; + word = NotSet; + } + + /// + /// Initialize the ObjectLogFilePositionInfo with the given word (containing segment and offset) and segment size bits. + /// + internal ObjectLogFilePositionInfo(ulong word, int segSizeBits) + { + SegmentSizeBits = segSizeBits; + this.word = word; + } + + internal readonly void Serialize(StreamWriter writer) + { + writer.WriteLine(SegmentSizeBits); + writer.WriteLine(word); + } + + internal void Deserialize(StreamReader reader) + { + var value = reader.ReadLine(); + SegmentSizeBits = int.Parse(value); + value = reader.ReadLine(); + word = ulong.Parse(value); + } + + /// The high byte is combined with the Value object length stored in the Value field when serialized, yielding 40 bits or 1TB max single object size. + public int ObjectSizeHighByte + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + readonly get { return (int)((word >> NumSegmentAndOffsetBits) & 0xFFUL); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set + { + if (value > byte.MaxValue) + throw new ArgumentOutOfRangeException(nameof(value), $"Object size high byte must be less than or equal to {byte.MaxValue}."); + word = (word & ~(0xFFUL << NumSegmentAndOffsetBits)) | ((ulong)value << NumSegmentAndOffsetBits); + } + } + + /// The high byte is combined with the Value object length stored in the Value field when serialized, yielding 40 bits or 1TB max single object size. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe void SetObjectSizeHighByte(ulong* wordPtr, int value) + { + if (value > byte.MaxValue) + throw new ArgumentOutOfRangeException(nameof(value), $"Object size high byte must be less than or equal to {byte.MaxValue}."); + *wordPtr = (*wordPtr & ~(0xFFUL << NumSegmentAndOffsetBits)) | ((ulong)value << NumSegmentAndOffsetBits); + } + + /// The high byte is combined with the Value object length stored in the Value field when serialized, yielding 40 bits or 1TB max single object size. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static unsafe int GetObjectSizeHighByte(ulong* wordPtr) => (int)((*wordPtr >> NumSegmentAndOffsetBits) & 0xFFUL); + + /// The offset within the current . + public ulong Offset + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + readonly get + { + var mask = (ulong)(1L << SegmentSizeBits) - 1L; + return word & mask; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set + { + var mask = (ulong)(1L << SegmentSizeBits) - 1L; + Debug.Assert((value & ~mask) <= SegmentSize, $"New Offset ({value & ~mask}) exceeds max segment size"); + word = (word & ~mask) | (value & mask); + } + } + + /// The current segment in the file. + public int SegmentId + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + readonly get + { + var mask = (ulong)((1L << (NumSegmentAndOffsetBits - SegmentSizeBits)) - 1L); + return (int)((word >> SegmentSizeBits) & mask); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set + { + var mask = (ulong)((1L << (NumSegmentAndOffsetBits - SegmentSizeBits)) - 1L); + word = (word & ~(mask << SegmentSizeBits)) | (((ulong)value & mask) << SegmentSizeBits); + } + } + + private readonly int MaxSegmentId + { + get + { + var seg = 1L << (NumSegmentAndOffsetBits - SegmentSizeBits); + return seg > int.MaxValue ? int.MaxValue : (int)seg; + } + } + + public void Advance(ulong size) + { + // Does it fit in the current segment? + var remaining = SegmentSize - Offset; + if (size < remaining) + { + Offset += size; + return; + } + + // Note: If size == remaining, we will advance to the start of the next segment. + size -= remaining; + + // Move to the next segment(s). + long nextSegmentId = SegmentId + (int)(size / SegmentSize) + 1; + if (nextSegmentId > MaxSegmentId) + throw new InvalidDataException($"Advancing position by {size:N} bytes exceeds maximum object log segment."); + + SegmentId = (int)nextSegmentId; + Offset += size & (SegmentSize - 1); + } + + public void AdvanceToNextSegment() + { + long nextSegmentId = SegmentId + 1; + if (nextSegmentId > MaxSegmentId) + throw new InvalidDataException($"Advancing to next segment exceeds maximum object log segment."); + SegmentId = (int)nextSegmentId; + Offset = 0; + } + + public readonly ulong CurrentAddress => ((ulong)SegmentId << SegmentSizeBits) | Offset; + + public static ulong operator -(ObjectLogFilePositionInfo left, ObjectLogFilePositionInfo right) + { + Debug.Assert(left.SegmentSizeBits == right.SegmentSizeBits, "Segment size bits must match to compute distance"); + Debug.Assert((left.word & SegmentAndOffsetMask) >= (right.word & SegmentAndOffsetMask), "comparison position must be greater"); + var segmentDiff = (ulong)(left.SegmentId - right.SegmentId); + if (segmentDiff == 0) + return left.Offset - right.Offset; + return ((segmentDiff - 1) * left.SegmentSize) + (left.SegmentSize - right.Offset) + left.Offset; + } + + public readonly ulong SegmentSize => 1UL << SegmentSizeBits; + + public readonly ulong RemainingSizeInSegment => SegmentSize - Offset; + + /// + public override readonly string ToString() => $"Segment# {SegmentId}; Offset {Offset:N0}; SegBits {SegmentSizeBits}; SegSize {SegmentSize:N0}; RemSize {RemainingSizeInSegment:N0}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogReader.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogReader.cs new file mode 100644 index 00000000000..f9cbbe21058 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogReader.cs @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Tsavorite.core +{ + /// + /// The class that manages IO read of ObjectAllocator records. It manages the read buffer at two levels: + /// + /// At the higher level, called by IO routines, it manages the overall record reading, including issuing additional reads as the buffer is drained. + /// At the lower level, it provides the stream for the valueObjectSerializer, which is called via Deserialize() by the higher level. + /// + /// + internal unsafe partial class ObjectLogReader : IStreamBuffer + where TStoreFunctions : IStoreFunctions + { + IObjectSerializer valueObjectSerializer; + PinnedMemoryStream> pinnedMemoryStream; + + /// The current record header; used for chunks to identify when they need to extract the optionals after the final chunk. + internal RecordInfo recordInfo; + + /// The circular buffer we cycle through for object-log deserialization. + readonly CircularDiskReadBuffer readBuffers; + + /// The implementation to use + internal readonly TStoreFunctions storeFunctions; + + /// If true, we are in the Deserialize call. If not we ignore things like etc. + bool inDeserialize; + + /// The cumulative length of object data read from the device during deserialization. + internal ulong deserializedLength; + + /// The total capacity of the buffer. + public bool IsForWrite => false; + +#pragma warning disable IDE0290 // Use primary constructor + public ObjectLogReader(CircularDiskReadBuffer readBuffers, TStoreFunctions storeFunctions) + { + this.readBuffers = readBuffers; + this.storeFunctions = storeFunctions ?? throw new ArgumentNullException(nameof(storeFunctions)); + } + + /// + /// Called when one or more records with Objects have been read via ReadAsync, e.g. being processed by AsyncReadPageWithObjectsCallback. + /// + /// The initial file position to read + /// The cumulative length of all object-log entries for the span of records to be read. We read ahead for all record + /// in the ReadAsync call. + internal void OnBeginReadRecords(ObjectLogFilePositionInfo filePosition, ulong totalLength) + { + inDeserialize = false; + deserializedLength = 0UL; + readBuffers.OnBeginReadRecords(filePosition, totalLength); + } + + /// + /// Called when one or more records with Objects have been read and via ReadAsync, e.g. being processed by AsyncReadPageWithObjectsCallback, + /// and we have completed reading and deserializing those objects. + /// + internal void OnEndReadRecords() => readBuffers.OnEndReadRecords(); + + /// + public void FlushAndReset(CancellationToken cancellationToken = default) => throw new InvalidOperationException("FlushAndReset is not supported for DiskStreamReadBuffer"); + + /// + public void Write(ReadOnlySpan data, CancellationToken cancellationToken = default) => throw new InvalidOperationException("Write is not supported for DiskStreamReadBuffer"); + + /// + /// Get the object log entries for Overflow Keys and Values and Object Values for the input . We do not create the log record here; + /// that was already done by the caller from a single-record disk IO or from Recovery. + /// + /// If there is an Overflow key, read it and if we have a compare it and return false if it does not match. + /// Otherwise, store the Key Overflow in the transient in . + /// If we don't have , this is either ReadAtAddress (which is an implicit match) or Scan or Restore. + /// If we have an Overflow or Object value, read and store it in the transient in . + /// + /// + /// The initial record read from disk from Pending IO, so it is of size or less. + /// The requested key, if not ReadAtAddress; we will compare to see if it matches the record. + /// Number of bits in segment size + /// False if requestedKey is set and we read an Overflow key and it did not match; otherwise true + [MethodImpl(MethodImplOptions.NoInlining)] + public bool ReadRecordObjects(ref LogRecord logRecord, TKey requestedKey, int segmentSizeBits) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + Debug.Assert(logRecord.Info.RecordHasObjects, "Inline records should have been checked by the caller"); + if (readBuffers is null) + throw new TsavoriteException("ReadBuffers are required to ReadRecordObjects"); + + // This is only called when we expect data to be there so throw if we don't have any. + var positionWord = logRecord.GetObjectLogRecordStartPositionAndLengths(out var keyLength, out var valueLength); + if (!readBuffers.OnBeginRecord(new ObjectLogFilePositionInfo(positionWord, segmentSizeBits))) + throw new TsavoriteException("ReadRecordObjects found no data available in ReadBuffers"); + + // TODO: Optimize the reading of large internal sector-aligned parts of Overflow Keys and Values to read directly into the overflow, similar to how ObjectLogWriter writes + // directly from overflow. This requires changing the read-ahead in CircularDiskReadBuffer.OnBeginReadRecords and the "backfill" in CircularDiskReadBuffer.MoveToNextBuffer. + + // Note: Similar logic to this is in DiskLogRecord.Deserialize. + var keyWasSet = false; + try + { + if (logRecord.Info.KeyIsOverflow) + { + // This assignment also allocates the slot in ObjectIdMap. The RecordDataHeader length info should be unchanged from ObjectIdSize. + logRecord.KeyOverflow = new OverflowByteArray(keyLength, startOffset: 0, endOffset: 0, zeroInit: false); + _ = Read(logRecord.KeyOverflow.Span); + if (!requestedKey.IsEmpty && !storeFunctions.KeysEqual(requestedKey, logRecord)) + return false; + } + + if (logRecord.Info.ValueIsOverflow) + { + // This assignment also allocates the slot in ObjectIdMap. The RecordDataHeader length info should be unchanged from ObjectIdSize. + logRecord.ValueOverflow = new OverflowByteArray((int)valueLength, startOffset: 0, endOffset: 0, zeroInit: false); + _ = Read(logRecord.ValueOverflow.Span); + } + else if (logRecord.Info.ValueIsObject) + { + // Info.ValueIsObject is true. DoDeserialize() also allocates the slot in ObjectIdMap and updates the value length to be ObjectIdSize. + DoDeserialize(ref logRecord); + } + return true; + } + catch + { + logRecord.OnDeserializationError(keyWasSet); + throw; + } + } + + /// + public int Read(Span destinationSpan, CancellationToken cancellationToken = default) + { + // This is called by valueObjectSerializer.Deserialize() to read up to destinationSpan.Length bytes. + // It is also currently called internally for Overflow. + var prevCopyLength = 0; + var destinationSpanAppend = destinationSpan.Slice(prevCopyLength); + + // Read from the circular buffer. + var buffer = readBuffers.GetCurrentBuffer(); + if (buffer is null || !buffer.HasData) + return 0; + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); // IDevice does not support cancellation, so just check this here + + var copyLength = buffer.AvailableLength; + if (copyLength > destinationSpanAppend.Length) + copyLength = destinationSpanAppend.Length; + + if (copyLength > 0) + { + buffer.AvailableSpan.Slice(0, copyLength).CopyTo(destinationSpanAppend); + buffer.currentPosition += copyLength; + if (inDeserialize) + deserializedLength += (uint)copyLength; + if (copyLength == destinationSpanAppend.Length) + return destinationSpan.Length; + } + + prevCopyLength += copyLength; + if (buffer.AvailableLength == 0) + { + if (!readBuffers.MoveToNextBuffer(out buffer)) + return prevCopyLength; + } + destinationSpanAppend = destinationSpan.Slice(prevCopyLength); + } + } + + void DoDeserialize(ref LogRecord logRecord) + { + deserializedLength = 0; + inDeserialize = true; + + // If we haven't yet instantiated the serializer do so now. + if (valueObjectSerializer is null) + { + pinnedMemoryStream = new(this); + valueObjectSerializer = storeFunctions.CreateValueObjectSerializer(); + valueObjectSerializer.BeginDeserialize(pinnedMemoryStream); + } + + valueObjectSerializer.Deserialize(out var valueObject); + logRecord.SetDeserializedValueObject(valueObject, deserializedLength); + OnDeserializeComplete(valueObject); + } + + void OnDeserializeComplete(IHeapObject valueObject) + { + // TODO add size tracking; do not track deserialization size changes if we are deserializing to a frame + + inDeserialize = false; + deserializedLength = 0UL; + } + + /// + public void Dispose() + { + pinnedMemoryStream?.Dispose(); + valueObjectSerializer?.EndDeserialize(); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogWriter.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogWriter.cs new file mode 100644 index 00000000000..c6a57f7a54c --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/ObjectLogWriter.cs @@ -0,0 +1,293 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Threading; +using static Tsavorite.core.Utility; + +namespace Tsavorite.core +{ + /// + /// The class that manages IO writing of Overflow and Object Keys and Values for records. It manages the write buffer at two levels: + /// + /// At the higher level, called by routines, it manages the overall Key and Value writing, including flushing to disk as the buffer is filled. + /// At the lower level, it provides the stream for the valueObjectSerializer, which is called via Serialize() by the higher level. + /// + /// + /// This handles only Overflow Keys and Values, and Object Values; inline Keys and Values (of any length) are written to the main log device as part of the main log record. + internal unsafe partial class ObjectLogWriter : IStreamBuffer + where TStoreFunctions : IStoreFunctions + { + readonly IDevice device; + IObjectSerializer valueObjectSerializer; + PinnedMemoryStream> pinnedMemoryStream; + + /// The circular buffer we cycle through for parallelization of writes. + internal CircularDiskWriteBuffer flushBuffers; + + /// The implementation to use + internal readonly TStoreFunctions storeFunctions; + + /// The current buffer being written to in the circular buffer list. + internal DiskWriteBuffer writeBuffer; + + /// For object serialization, the cumulative length of the value bytes. + ulong valueObjectBytesWritten; + + /// The maximum number of key or value bytes to copy into the buffer rather than enqueue a DirectWrite. + internal const int MaxCopySpanLen = 128 * 1024; + + /// If true, we are in the Serialize call. If not we ignore things like etc. + bool inSerialize; + + /// The total capacity of the buffer. + public bool IsForWrite => true; + + /// Constructor. Creates the circular buffer pool. +#pragma warning disable IDE0290 // Use primary constructor + public ObjectLogWriter(IDevice device, CircularDiskWriteBuffer flushBuffers, TStoreFunctions storeFunctions) + { + this.device = device ?? throw new ArgumentNullException(nameof(device)); + this.flushBuffers = flushBuffers ?? throw new ArgumentNullException(nameof(flushBuffers)); + this.storeFunctions = storeFunctions; + } + + /// + /// This is a no-op because we have already flushed under control of the Write() and OnPartialFlushComplete() methods. + public void FlushAndReset(CancellationToken cancellationToken = default) { } + + internal ObjectLogFilePositionInfo GetNextRecordStartPosition() => flushBuffers.GetNextRecordStartPosition(); + + /// Resets start positions for the next partial flush. + internal DiskWriteBuffer OnBeginPartialFlush(ObjectLogFilePositionInfo filePosition) + { + valueObjectBytesWritten = 0; + inSerialize = false; + writeBuffer = flushBuffers.OnBeginPartialFlush(filePosition); + return writeBuffer; + } + + /// + /// Finish all the current partial flushes, then write the main log page (or page fragment). + /// + /// Starting pointer of the main log page span to write + /// Length of the main log page span to write + /// The main log device to write to + /// The offset in the main log to write at; aligned to sector + /// Callback sent to the initial Flush() command. Called when we are done with this partial flush operation. + /// Context sent to . + /// The ending file position after the partial flush is complete + internal void OnPartialFlushComplete(byte* mainLogPageSpanPtr, int mainLogPageSpanLength, IDevice mainLogDevice, ulong alignedMainLogFlushAddress, + DeviceIOCompletionCallback externalCallback, object externalContext, ref ObjectLogFilePositionInfo endFilePosition) + => flushBuffers.OnPartialFlushComplete(mainLogPageSpanPtr, mainLogPageSpanLength, mainLogDevice, alignedMainLogFlushAddress, + externalCallback, externalContext, ref endFilePosition); + + /// + /// Write Overflow and Object Keys and values in a to the device. + /// + /// This only writes Overflow and Object Keys and Values; inline portions of the record are written separately by the caller. + /// The number of bytes written for the value object, if any. + public ulong WriteRecordObjects(in OverflowByteArray keyOverflow, in OverflowByteArray valueOverflow, in IHeapObject valueObject) + { + // If the key is overflow, start with that + if (!keyOverflow.IsEmpty) + WriteDirect(keyOverflow); + + // Now do value overflow or object, if either is present + if (!valueOverflow.IsEmpty) + WriteDirect(valueOverflow); + else if (valueObject is not null) + DoSerialize(valueObject); + + // Signal completion. + flushBuffers.OnRecordComplete(); + return valueObjectBytesWritten; + } + + /// Start off the write using the full span of the . + /// The to write. + void WriteDirect(OverflowByteArray overflow) => WriteDirect(overflow, overflow.ReadOnlySpan, refCountedGCHandle: default); + + /// Write the of the . + /// The to write. + /// The span of to write. Initially it is the full ; if the write + /// spans segments, then it is a recursive call for the last segment's fraction. + /// The refcounted GC handle if this is a recursive call + void WriteDirect(OverflowByteArray overflow, ReadOnlySpan fullDataSpan, RefCountedPinnedGCHandle refCountedGCHandle) + { + if (overflow.Length <= MaxCopySpanLen) + Write(fullDataSpan); + else + { + // 1. Write the sector-aligning start fragment into the buffers and flush the current buffer (if we cross a buffer boundary, + // previous buffers will already have been flushed). + var dataStart = 0; + var copyLength = RoundUp(writeBuffer.currentPosition, (int)device.SectorSize) - writeBuffer.currentPosition; + if (copyLength != 0) + { + Debug.Assert(refCountedGCHandle is null, $"If refCountedGCHandle is not null then buffer.currentPosition ({writeBuffer.currentPosition}) should already be sector-aligned"); + Write(fullDataSpan.Slice(dataStart, copyLength)); + dataStart += copyLength; + flushBuffers.FlushCurrentBuffer(); + } + + // 2. Flush the sector-aligned span interior. We are writing direct to the device from a byte[], so we have to pin the array. + // We may have to split across multiple segments. + var interiorLen = RoundDown(overflow.Array.Length - dataStart, (int)device.SectorSize); + var segmentRemainingLen = flushBuffers.filePosition.RemainingSizeInSegment; + var gcHandle = (refCountedGCHandle is null) ? GCHandle.Alloc(overflow.Array, GCHandleType.Pinned) : default; + var localGcHandle = refCountedGCHandle?.gcHandle ?? gcHandle; + var overflowStartPtr = (byte*)localGcHandle.AddrOfPinnedObject() + overflow.StartOffset; + if ((uint)interiorLen <= segmentRemainingLen) + { + // We have enough room in the segment to write the full interior span in one chunk. + var writeCallback = refCountedGCHandle is null + ? flushBuffers.CreateDiskWriteCallbackContext(gcHandle) + : flushBuffers.CreateDiskWriteCallbackContext(refCountedGCHandle); + flushBuffers.FlushToDevice(overflowStartPtr + dataStart, interiorLen, writeCallback); + dataStart += interiorLen; + } + else + { + // Multi-segment write so we will need to refcount the GCHandle. SegmentRemainingLength is <= int.MaxValue so we can cast it to int. + // TODO: This and other segment-limiting logic could be pushed down into StorageDeviceBase, which could iterate on the segments. + // However this could have complications with e.g. callback and countdown counts (there would be more than one callback invocation + // on that; this could be handled by defining some way for the StorageDeviceBase to know the calback uses a CountdownEvent and + // incrementing that count, or by having a local callback, similarly to how CircularDiskWriteBuffer handles multiple possibly-concurrent + // writes before calling the main callback, that handles doing the "final" callback). In this case we could defer the "segment id" logic + // to StorageDeviceBase, and just have a ulong position, from which we could compute the segment id (e.g. for truncation), and + // ObjectLogFilePositionInfo would be simplified. + Debug.Assert(segmentRemainingLen <= int.MaxValue, $"segmentRemainingLen ({segmentRemainingLen}) should be <= int.MaxValue"); + + // Create the refcounted pinned GCHandle with a refcount of 1, so that if a read completes while we're still setting up, we won't get an early unpin. + refCountedGCHandle ??= new RefCountedPinnedGCHandle(gcHandle, initialCount: 1); + + // Copy chunks to segments and advance the segment. + while (interiorLen > (int)segmentRemainingLen) + { + var writeCallback = flushBuffers.CreateDiskWriteCallbackContext(refCountedGCHandle); + flushBuffers.FlushToDevice(overflowStartPtr + dataStart, (int)segmentRemainingLen, writeCallback); + dataStart += (int)segmentRemainingLen; + + Debug.Assert(flushBuffers.filePosition.RemainingSizeInSegment == 0, $"Expected to be at end of segment but there were {flushBuffers.filePosition.RemainingSizeInSegment} bytes remaining"); + flushBuffers.filePosition.AdvanceToNextSegment(); + segmentRemainingLen = flushBuffers.filePosition.RemainingSizeInSegment; + } + + // Now we know we will fit in the last segment, so call recursively to optimize the "copy vs. direct" final fragment. + // First adjust the endPosition in case we don't have a full buffer of space remaining in the segment. + if ((ulong)writeBuffer.RemainingCapacity > flushBuffers.filePosition.RemainingSizeInSegment) + writeBuffer.endPosition = (int)flushBuffers.filePosition.RemainingSizeInSegment - writeBuffer.currentPosition; + WriteDirect(overflow, fullDataSpan.Slice(dataStart), refCountedGCHandle); + } + + // 3. Copy the end sector-aligning fragment to the buffers. + if (dataStart < overflow.Length) + Write(fullDataSpan.Slice(dataStart)); + } + + // Release the initial refcount on this, if we created it. This will let it final-release when all writes are complete. + refCountedGCHandle?.Release(); + } + + /// + public void Write(ReadOnlySpan data, CancellationToken cancellationToken = default) + { + // This is called by valueObjectSerializer.Serialize() as well as internally. No other calls should write data to flushBuffer.memory in a way + // that increments flushBuffer.currentPosition, since we manage chained-chunk continuation and DiskPageHeader offsetting here. + + // Copy to the buffer. If it does not fit in the remaining capacity, we will write as much as does, flush the buffer, and move to next buffer. + var dataStart = 0; + var segmentRemainingLen = flushBuffers.filePosition.SegmentSize - flushBuffers.GetNextRecordStartPosition().Offset; + while (data.Length - dataStart > 0) + { + Debug.Assert(writeBuffer.RemainingCapacity > 0, + $"RemainingCapacity {writeBuffer.RemainingCapacity} should not be 0 (data.Length {data.Length}, dataStart {dataStart}); this should have already triggered an OnChunkComplete call, which would have reset the buffer"); + cancellationToken.ThrowIfCancellationRequested(); // IDevice does not support cancellation, so just check this here + + // If it won't all fit in the remaining buffer, write as much as will. + var requestLength = (uint)(data.Length - dataStart); + if (requestLength > writeBuffer.RemainingCapacity) + requestLength = (uint)writeBuffer.RemainingCapacity; + + // If it won't all fit in the remaining segment, write as much as will. + if ((ulong)requestLength > segmentRemainingLen) + requestLength = (uint)segmentRemainingLen; + segmentRemainingLen -= requestLength; + + data.Slice(dataStart, (int)requestLength).CopyTo(writeBuffer.memory.TotalValidSpan.Slice(writeBuffer.currentPosition)); + dataStart += (int)requestLength; + writeBuffer.currentPosition += (int)requestLength; + if (inSerialize) + { + valueObjectBytesWritten += requestLength; + if (valueObjectBytesWritten >= IHeapObject.MaxSerializedObjectSize) + throw new TsavoriteException($"Object serialized size currently at {valueObjectBytesWritten} which exceeds max serialization limit of {IHeapObject.MaxSerializedObjectSize}"); + } + + // See if we're at the end of the buffer or segment. + if (writeBuffer.RemainingCapacity == 0 || segmentRemainingLen == 0) + OnBufferComplete(); + + if (segmentRemainingLen == 0) + { + flushBuffers.filePosition.AdvanceToNextSegment(); + segmentRemainingLen = flushBuffers.filePosition.RemainingSizeInSegment; + } + } + } + + /// At the end of a buffer, do any processing, flush the current buffer, and move to the next buffer. + /// Called during Serialize(). + void OnBufferComplete() + { + // This should only be called when the object serialization hits the end of the buffer; for partial buffers we will call + // OnSerializeComplete() after the Serialize() call has returned. "End of buffer" ends before lengthSpaceReserve if any. + Debug.Assert(writeBuffer.currentPosition == writeBuffer.endPosition, $"CurrentPosition {writeBuffer.currentPosition} must be at writeBuffer.endPosition {writeBuffer.endPosition})."); + + flushBuffers.FlushCurrentBuffer(); + writeBuffer = flushBuffers.MoveToAndInitializeNextBuffer(); + } + + void DoSerialize(IHeapObject valueObject) + { + // valueCumulativeLength is only relevant for object serialization; we increment it on all device writes to avoid "if", so here we reset it to the appropriate + // "start at 0" by making it the negative of currentPosition. Subsequently if we write e.g. an int, we'll have Length and Position = (-currentPosition + currentPosition + 4). + inSerialize = true; + valueObjectBytesWritten = 0; + + // If we haven't yet instantiated the serializer do so now. + if (valueObjectSerializer is null) + { + pinnedMemoryStream = new(this); + valueObjectSerializer = storeFunctions.CreateValueObjectSerializer(); + valueObjectSerializer.BeginSerialize(pinnedMemoryStream); + } + + valueObjectSerializer.Serialize(valueObject); + OnSerializeComplete(valueObject); + } + + void OnSerializeComplete(IHeapObject valueObject) + { + inSerialize = false; + } + + /// + public int Read(Span destinationSpan, CancellationToken cancellationToken = default) => throw new InvalidOperationException("Read is not supported for DiskStreamWriteBuffer"); + + /// + public void Dispose() + { + var localMemoryStream = Interlocked.Exchange(ref pinnedMemoryStream, null); + if (localMemoryStream is not null) + { + // End serialization before disposing the pinned memory stream as it may try to flush final data which would use the pinnedMemoryStream. + valueObjectSerializer?.EndSerialize(); + localMemoryStream.Dispose(); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/PinnedMemoryStream.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/PinnedMemoryStream.cs new file mode 100644 index 00000000000..3ae3295b2a9 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/PinnedMemoryStream.cs @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + /// + /// This is a simple stream over a pinned memory buffer, such as a SectorAlignedMemory or network buffer. + /// + internal class PinnedMemoryStream : Stream + where TStreamBuffer : class, IStreamBuffer + { + TStreamBuffer streamBuffer; + + public PinnedMemoryStream(TStreamBuffer streamBuffer) + { + if (streamBuffer is null) + throw new ArgumentNullException(nameof(streamBuffer)); + this.streamBuffer = streamBuffer; + } + + /// Whether the stream is opened for Read + public override bool CanRead => !streamBuffer.IsForWrite; + + /// This stream implementation cannot Seek + public override bool CanSeek => false; + + /// Whether the stream is opened for Write + public override bool CanWrite => streamBuffer.IsForWrite; + + /// + protected override void Dispose(bool disposing) + { + // Note: this may be called from the localStreamBuffer's Dispose, so we have to be sure not to double-Dispose anything. + var localStreamBuffer = Interlocked.Exchange(ref streamBuffer, null); + if (localStreamBuffer is not null) + { + localStreamBuffer.Dispose(); + base.Dispose(disposing); + } + } + + /// Flush the internal buffer + public override void Flush() => streamBuffer.FlushAndReset(); + + /// Flush the internal buffer asynchronously + public override Task FlushAsync(CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + + try + { + streamBuffer.FlushAndReset(cancellationToken); + return Task.CompletedTask; + } + catch (Exception ex) + { + return Task.FromException(ex); + } + } + + /// The amount of data in the internal streamBuffer. Not supported because we chunk and thus may not have all data. + public override long Length + { + get => throw new NotSupportedException("Stream does not support get_Length."); + } + + /// The current position of the stream seeking; not supported + public override long Position + { + get => throw new NotSupportedException("Stream does not support get_Position."); + set => throw new NotSupportedException("Stream does not support set_Position."); + } + + /// Copy data from the internal streamBuffer into the buffer; the streamBuffer handles Flush, Reset, and Read more + /// (e.g. from disk or network) as needed. + /// Buffer to copy the bytes into. + /// Index in the buffer to start copying to. + /// Desired number of bytes to copy to the buffer. + /// Number of bytes actually read. + public override int Read(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + return streamBuffer.Read(new Span(buffer, offset, count)); + } + + /// Copy data from the internal streamBuffer into the destination span; the streamBuffer handles Flush, Reset, and Read more + /// (e.g. from disk or network) as needed. + public override int Read(Span destinationSpan) => streamBuffer.Read(destinationSpan); + + /// Asynchronously copy data from the internal streamBuffer into the memory buffer; the streamBuffer handles Flush, Reset, and Read more + /// (e.g. from disk or network) as needed. + /// Buffer to read the bytes to. + /// Token that can be used to cancel this operation. + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (cancellationToken.IsCancellationRequested) + return ValueTask.FromCanceled(cancellationToken); + + try + { + return new ValueTask(Read(buffer.Span)); + } + catch (Exception ex) + { + return ValueTask.FromException(ex); + } + } + + /// Returns the byte at the current streamBuffer position and advances the position + /// The byte read (as an int) + public override unsafe int ReadByte() + { + byte b = default; + return streamBuffer.Read(new Span(ref b)) > 0 ? b : -1; + } + + /// Seeking is not supported in this stream. + public override long Seek(long offset, SeekOrigin loc) => throw new InvalidOperationException("Stream does not support Seek."); + + /// Seeking is not supported in this stream. + public override void SetLength(long value) => throw new InvalidOperationException("Stream does not support SetLength."); + + /// Write the buffer to the stream; the streamBuffer handles Flush, Reset, and Writing iteratively + /// (e.g. to disk or network) as needed. + /// Buffer to write the bytes from. + /// Index in the buffer to start writing from. + /// Desired number of bytes to write from the buffer. + public override void Write(byte[] buffer, int offset, int count) + { + ValidateBufferArguments(buffer, offset, count); + streamBuffer.Write(new ReadOnlySpan(buffer, offset, count)); + } + + /// Write the buffer to the stream; the streamBuffer handles Flush, Reset, and Writing iteratively + /// (e.g. to disk or network) as needed. + public override void Write(ReadOnlySpan destinationSpan) => streamBuffer.Write(destinationSpan); + + /// Asynchronously write the buffer to the stream; the streamBuffer handles Flush, Reset, and Writing iteratively + /// (e.g. to disk or network) as needed. + /// Buffer to write the bytes from. + /// Index in the buffer to start writing from. + /// Desired number of bytes to write from the buffer. + /// Task that can be awaited + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken = default) + { + ValidateBufferArguments(buffer, offset, count); + return WriteAsync(new ReadOnlySpan(buffer, offset, count), cancellationToken).AsTask(); + } + + /// Asynchronously write the buffer to the stream; the streamBuffer handles Flush, Reset, and Writing iteratively + /// (e.g. to disk or network) as needed. + /// Buffer to write the bytes from. + /// Token that can be used to cancel the operation. + public override ValueTask WriteAsync(ReadOnlyMemory memoryBuffer, CancellationToken cancellationToken = default) + => WriteAsync(memoryBuffer.Span, cancellationToken); + + private ValueTask WriteAsync(ReadOnlySpan destinationSpan, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return ValueTask.FromCanceled(cancellationToken); + + try + { + streamBuffer.Write(destinationSpan, cancellationToken); + return ValueTask.CompletedTask; + } + catch (Exception ex) + { + return ValueTask.FromException(ex); + } + } + + /// Writes a byte at the next streamBuffer position and advances the position + public override unsafe void WriteByte(byte value) + => streamBuffer.Write(new ReadOnlySpan(ref value)); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/RecordDataHeader.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/RecordDataHeader.cs new file mode 100644 index 00000000000..f644f46d00a --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ObjectSerialization/RecordDataHeader.cs @@ -0,0 +1,496 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using static Tsavorite.core.Utility; + +namespace Tsavorite.core +{ + /// + /// The header describing the data layout of the record. The record must be pinned. + /// The layout is: + /// + /// Length indicator byte: flag bits and the number of bytes in KeyLength and RecordLength (3 bits, for 0-4). The layout of this indicator byte is: + /// + /// Indicator bits: 2 bits for any flags we want to add. + /// Number of bytes in Filler: 2 bits. This indicates the extra space available in the record if the used record length does not take the full allocated + /// space, either on initial creation or due to later value shriking or removal of Optional fields. These two bits are ignored unless RecordInfo.HasFiller + /// is set, in which case the value in these two bits must be nonzero, from 1-4. The 2 bits covers 0-3 and are offset by 1, as they must be nonzero + /// if RecordInfo.HasFiller is set, so there are 1-4 Filler length bytes possible. These 1-4 values are interpreted as: + /// + /// 1-3: this is the number of bytes in the Filler, as there is not enough Filler space for a full int. + /// 4: there are enough extra bytes to hold an int, and that int contains the actual number of Filler bytes. This int is the last 4 bytes + /// of the RecordLength; i.e. the last int before . + /// + /// + /// Number of bytes in KeyLength: 2 bits. May be inline length or if Overflow. The 2 bits covers 0-3 and + /// is offset by 1, as there must always be at least one key byte, so there are 1-4 KeyLength bytes possible. KeyLength is immutable for the life of + /// a record, but may be changed by revivification. + /// Number of bytes in RecordLength: 2 bits. Includes Key and Value length (and other attributes such as optionals) so is nonzero. The 2 bits covers 0-3 and + /// is offset by 1, as there must always be one byte. RecordLength is immutable for the life of the log page, including record revivification; even though + /// namespace and key lengths and optionals may change, the record length does not. + /// + /// + /// Namespace byte (with encoding indicating if there are many extra namespace bytes; if so, they precede the Key data bytes). + /// Record type byte; interpreted by caller + /// RecordLength. The entire allocated record size, from the start of the RecordInfo to the end of the allocation; rounded up to Constants.kRecordAlignment. + /// It must precede KeyLength so it never has to change location, e.g. if Revivification changes the number of KeyLenght bytes. + /// KeyLength. The length of the key + /// Namespace extra data, if any + /// Key data bytes + /// Content, consisting of: + /// + /// Value bytes, if any; there may be none, e.g. creating a Key to server as a Tombstone or as a lock. + /// Optional fields, consisting of: + /// + /// ETag, if present + /// Expiration, if present + /// ObjectLog position, if the Key is Overflow or the Value is Overflow or Object + /// + /// Filler, if present + /// + /// We do not store ValueLength explicitly; it is derived from RecordLength minus the sizes of Namespace extra bytes if any, Key, Optionals if any, and Filler. + /// + /// + /// + public unsafe struct RecordDataHeader : IKey + { +#pragma warning disable IDE1006 // Naming Styles: Must begin with uppercase letter + // When assigning these bits, use the highest # in kReservedBitMask# + const ulong kReservedBitMask1 = 0 << 7; // Reserved bit + const ulong kReservedBitMask2 = 0 << 6; // Reserved bit + + // The bottom 6 bits are length bytecounts + /// + /// 2 bits (4, 5) for the number of bytes for the Filler Length. There must always be a filler, so we can store the filler size indicator as 2 bits + /// which when offset by 1 allows for 1-4 bytes. If the value is 4, then there are enough bytes to hold an int, and that int is the last 4 + /// bytes of the record and contains the actual filler length. Otherwise, the value is between 1-3 and is the actual filler length. + /// + const int kFillerLengthIndicatorBitMask = (1 << kFillerLengthIndicatorBits) - 1; + const int kFillerLengthIndicatorBits = 2; + const int kFillerLengthIndicatorShift = kRecordLengthIndicatorBits + kKeyLengthBits; + + /// + /// 2 bits (2, 3) for the number of bytes for the RecordLength. This must always be nonzero up to 1 << ), which is + /// in 4 bytes, and 2 bits covers 0-3 which when adding 1 allows for 1-4 bytes. + /// + const int kRecordLengthIndicatorBitMask = (1 << kRecordLengthIndicatorBits) - 1; + const int kRecordLengthIndicatorBits = 2; + const int kRecordLengthIndicatorShift = kKeyLengthBits; // Shift bits in the indicator byte + const int kRecordLengthShiftInHeader = NumIndicatorBytes * 8; // Shift bytes when storing or retrieving the actual length + + /// + /// 2 bits (0, 1) for the number of bytes for the KeyLength. There must always be a key, so we can store the max key size (which is limited by 1 << + /// and thus allows 4 bytes), and 2 bits covers 0-3 which when adding 1 allows for 1-4 bytes. + /// + const int kKeyLengthIndicatorBitMask = (1 << kKeyLengthBits) - 1; + const int kKeyLengthBits = 2; + const int kKeyLengthIndicatorShift = 0; + // keyLengthShiftInHeader is calculated in the code, as it relies on kRecordLengthShiftInHeader +#pragma warning restore IDE1006 // Naming Styles + + /// The maximum number of key length bytes; . Anything over this becomes overflow. + internal const int MaxKeyLengthBytes = 1 << kKeyLengthBits; + + /// The maximum number of value length bytes; see . + internal const int MaxRecordLengthBytes = 1 << kRecordLengthIndicatorBits; + + /// The minimum number of total data header bytes--NumIndicatorBytes, 1 byte KeyLength, 1 byte RecordLength + public const int MinHeaderBytes = NumIndicatorBytes + 2; + /// The maximum number of total data header bytes--NumIndicatorBytes, 4 bytes KeyLength, 4 bytes RecordLength + internal const int MaxHeaderBytes = NumIndicatorBytes + 8; + /// The number of data header indicator bytes; currently 3 for the length indicator, Namespace, RecordType. + internal const int NumIndicatorBytes = 3; + + /// If the is not set, then the bits + /// contain the full namespace as a single byte; otherwise those bits are the length of the extended namespace data preceding the key data. + internal const byte ExtendedNamespaceIndicatorBit = 1 << 7; + /// If the is not set, then the bits + /// contain the full namespace as a single byte; otherwise those bits are the length of the extended namespace data preceding the key data. + internal const byte NamespaceIndicatorMask = ExtendedNamespaceIndicatorBit - 1; + + /// Offset of the nameSpace byte in the header. + internal const byte NamespaceOffsetInHeader = 1; + /// Offset of the recordType byte in the header. + internal const byte RecordTypeOffsetInHeader = 2; + + /// Pointer to the first byte of the header, which is the length indicator byte. + internal byte* HeaderPtr; + + /// + public override readonly string ToString() => ToString("na", "na"); + + internal readonly string ToString(string keyString, string valueString) + { + if (HeaderPtr == null) + return ""; + var (numKeyLengthBytes, numRecordLengthBytes) = DeconstructKVByteLengths(out var headerLength); + var recordLength = GetRecordLength(); + var fillerLength = GetFillerLength(recordLength); + var (keyLength, keyAddress) = GetKeyFieldInfo(); + var (valueLength, valueAdress) = GetValueFieldInfo(*RecordInfoPtr, out _ /*keyLength*/, out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + var fillerLenStr = (*RecordInfoPtr).HasFiller ? fillerLength.ToString() : "na"; + + return $"rec b:{numRecordLengthBytes}/o:na/l:{recordLength}" + + $" | key b:{numKeyLengthBytes}/o:{keyAddress - (long)RecordInfoPtr}/l:{keyLength} {keyString}" + + $" | val b:na/o:{valueAdress - (long)RecordInfoPtr}/l:{valueLength}, {valueString}" + + $" | filLen {fillerLenStr} Namespace b:{NamespaceByte}/x:{ExtendedNamespaceLength}, RecordType {RecordType}"; + } + + internal RecordDataHeader(byte* indicatorPtr) => HeaderPtr = indicatorPtr; + + private readonly RecordInfo* RecordInfoPtr => (RecordInfo*)(HeaderPtr - RecordInfo.Size); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int GetByteCount(long value) => ((sizeof(long) * 8) - BitOperations.LeadingZeroCount((ulong)(value | 1)) + 7) / 8; + + internal readonly int ExtendedNamespaceLength + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + var nameSpace = *(HeaderPtr + NamespaceOffsetInHeader); + return (nameSpace & ExtendedNamespaceIndicatorBit) == 0 ? 0 : nameSpace & NamespaceIndicatorMask; + } + } + + /// Get or set the RecordType byte. Throws an exception if out of range or if there is a conflicting specification for extended-length nameSpace. + public readonly byte NamespaceByte + { + get + { + var nameSpace = *(HeaderPtr + NamespaceOffsetInHeader); + if ((nameSpace & ExtendedNamespaceIndicatorBit) != 0) + throw new TsavoriteException("Cannot get NamespaceByte when ExtendedNamespaceFlag is set"); + return nameSpace; + } + set + { + if (value > sbyte.MaxValue) + throw new TsavoriteException($"NamespaceByte value {value} exceeds max allowable {sbyte.MaxValue}"); + *(HeaderPtr + NamespaceOffsetInHeader) = value; + } + } + + /// Get or set the RecordType byte + public readonly byte RecordType + { + get => *(HeaderPtr + RecordTypeOffsetInHeader); + set => *(HeaderPtr + RecordTypeOffsetInHeader) = value; + } + + #region IKey + + /// + public readonly bool IsPinned => true; + + /// + public readonly ReadOnlySpan KeyBytes + { + get + { + var ptr = HeaderPtr - RecordInfo.Size; + + var (numKeyLengthBytes, numRecordLengthBytes) = DeconstructKVByteLengths(out var headerLength); + var offsetToKeyStart = GetOffsetToKeyStart(headerLength); + + var keyStartPtr = ptr + offsetToKeyStart; + var keyLength = GetKeyLength(numKeyLengthBytes, numRecordLengthBytes); + + return new ReadOnlySpan(keyStartPtr, keyLength); + } + } + + /// + public readonly bool HasNamespace + { + get + { + // True if non-0 OR ExtendedNamespaceIndicatorBit is et + var nameSpace = *(HeaderPtr + NamespaceOffsetInHeader); + return nameSpace != 0; + } + } + + /// + public readonly ReadOnlySpan NamespaceBytes + { + get + { + Debug.Assert(HasNamespace, "Should call if !HasNamespace"); + + var nameSpacePtr = (HeaderPtr + NamespaceOffsetInHeader); + + if (((*nameSpacePtr) & ExtendedNamespaceIndicatorBit) == 0) + { + // Single byte namespace + return new ReadOnlySpan(nameSpacePtr, 1); + } + else + { + throw new TsavoriteException($"Extended namespaces not yet implemented"); + } + } + } + + #endregion + + internal readonly int Initialize(ref RecordInfo recordInfo, in RecordSizeInfo sizeInfo, out long keyAddress, out long namespaceAddress, out long valueAddress) + { + // Format of indicator byte is high->low: <2 bits reserved><2 bits encoded filler length><2 bits key length byte count - 1><2 bits record length byte count - 1> + var keyLength = sizeInfo.InlineKeySize; + var valueLength = sizeInfo.InlineValueSize; + var recordLength = sizeInfo.AllocatedInlineRecordSize; + var numRecordLengthBytes = sizeInfo.RecordLengthBytes; + Debug.Assert(numRecordLengthBytes == GetByteCount(recordLength), "RecordLengthBytes does not match RecordLength"); + var numKeyLengthBytes = sizeInfo.KeyLengthBytes; + Debug.Assert(numKeyLengthBytes == GetByteCount(keyLength), "KeyLengthBytes does not match KeyLength"); + + // If this was from revivification, we should have <= keyLengthBytes and == recordLengthBytes. Don't change keyLengthBytes, as that would move the RecordLength + // field in the header and that might not be an atomic update if it crosses a ulong boundary. + if (sizeInfo.IsRevivifiedRecord) + { + var (revivKeyLenBytes, revivRecLenBytes) = DeconstructKVByteLengths(out _ /*headerLength*/); + if (numKeyLengthBytes > revivKeyLenBytes || numRecordLengthBytes != revivRecLenBytes) + ThrowTsavoriteException($"In revivification, cannot exceed previous KeyLengthBytes {revivKeyLenBytes} or change RecordLengthBytes {revivRecLenBytes}"); + numKeyLengthBytes = revivKeyLenBytes; + } + + // Fill in the indicator byte. + *HeaderPtr = (byte)(((numRecordLengthBytes - 1) << kRecordLengthIndicatorShift) | ((numKeyLengthBytes - 1) << kKeyLengthIndicatorShift)); + + // TODO: Pass in the actual SpanNamespace to VarLenMethods to set sizeInfo.FieldInfo.ExtendedNamespaceSize. Here we are only concerned + // with setting the correct length indicators; LogRecord.InitializeRecord will set the actual data for it. sizeInfo.FieldInfo.ExtendedNamespaceSize + // has been verified by RecordSizeInfo.CalculateSizes to be within byte range. + var extendedNamespaceSize = sizeInfo.FieldInfo.ExtendedNamespaceSize; + namespaceAddress = (long)HeaderPtr + NamespaceOffsetInHeader; + *(byte*)namespaceAddress = (byte)(extendedNamespaceSize > 0 ? (ExtendedNamespaceIndicatorBit | (extendedNamespaceSize & NamespaceIndicatorMask)) : 0); + *(HeaderPtr + RecordTypeOffsetInHeader) = sizeInfo.FieldInfo.RecordType; + + // Calculate and store the filler length, if any. Filler includes any space for optionals that won't have been set this early in the initialization process. + // If sizeInfo indicates the record is not inline, that won't have been reflected in RecordInfo yet and thus not in optionals, but we need to reserve the + // ObjectLogPosition space and not let it be part of FillerLength. Do this here after we have initialized the nameSpace byte. + var headerLength = NumIndicatorBytes + numKeyLengthBytes + numRecordLengthBytes; + SetFillerLength(ref recordInfo, recordLength, fillerLength: recordLength - RecordInfo.Size - headerLength - extendedNamespaceSize - keyLength - valueLength - sizeInfo.ObjectLogPositionSize); + + // Set RecordLength into the header. Header format is (low->high): . + // RecordLength will always fit in the header word. Zero out bits before we assign them in case we have non-zeroinitialized space. + var recordLengthMask = (1UL << (numRecordLengthBytes * 8)) - 1; + *(ulong*)HeaderPtr = (*(ulong*)HeaderPtr & ~(recordLengthMask << kRecordLengthShiftInHeader)) | (((ulong)recordLength & recordLengthMask) << kRecordLengthShiftInHeader); + + // Set KeyLength into the header. The key length actual bytes may fit along with everything else in the header into a single ulong; otherwise the key length bytes + // overflow the ulong. To access they key length, offset IndicatorPtr to align to point to the bytes of a ulong with the KeyLength space as high bytes (remembering + // that in little endian, the high bytes are the "rightmost" bytes of a byte*). If the entire header (including key length) fits in a ulong this will back up into + // the RecordInfo space; otherwise, we will subtract the negative adjustment and thus "advance" IndicatorPtr. (We don't advance to make KeyLength the low bits, + // because that could encounter end-of-record if length is zero). Zero out bits before we assign them in case we have non-zeroinitialized space. + var keyLengthMask = (1UL << (numKeyLengthBytes * 8)) - 1; + var ptrBackup = sizeof(ulong) - NumIndicatorBytes - numRecordLengthBytes - numKeyLengthBytes; // If negative, the pointer advances + var keyLenPtr = (ulong*)(HeaderPtr - ptrBackup); + var keyLengthShiftInHeader = (sizeof(ulong) - numKeyLengthBytes) * 8; + *keyLenPtr = (*keyLenPtr & ~(keyLengthMask << keyLengthShiftInHeader)) | (((ulong)keyLength & keyLengthMask) << keyLengthShiftInHeader); + + keyAddress = (long)RecordInfoPtr + GetOffsetToKeyStart(headerLength); + valueAddress = keyAddress + keyLength; + + return headerLength; + } + + internal readonly void InitializeForRevivification(ref RecordInfo recordInfo, ref RecordSizeInfo sizeInfo) + { + Debug.Assert(recordInfo.Invalid, "Expected record to be Invalid in InitializeForRevivification"); + Debug.Assert(recordInfo.KeyIsInline, "Expected Key to be inline in InitializeForRevivification"); + Debug.Assert(recordInfo.ValueIsInline, "Expected Value to be inline in InitializeForRevivification"); + Debug.Assert(!recordInfo.HasETag && !recordInfo.HasExpiration, "Expected no optionals in InitializeForRevivification"); + + // See Initialize() for formatting notes. + // The keyLengthBytes and RecordLength must be less than or equal to those before revivification (even if we could fit a larger Key, any movement + // of RecordLength might not be atomic if it crosses the ulong boundary, so we just don't allow it). + var (numKeyLengthBytes, numRecordLengthBytes) = DeconstructKVByteLengths(out var headerLength); + var keyLength = GetKeyLength(numKeyLengthBytes, numRecordLengthBytes); + Debug.Assert(GetByteCount(sizeInfo.InlineKeySize) <= numKeyLengthBytes, "Cannot exceed previous Key size bytes in InitializeForRevivification"); + var recordLength = GetRecordLength(numRecordLengthBytes); + Debug.Assert(sizeInfo.AllocatedInlineRecordSize <= recordLength, "Cannot exceed previous Record size in InitializeForRevivification"); + + // We have no optionals, so just set up with key length and recordLength; no filler. + recordInfo.ClearHasFiller(); + *HeaderPtr = (byte)(*HeaderPtr & ~(kFillerLengthIndicatorBitMask << kFillerLengthIndicatorShift)); + + *(HeaderPtr + NamespaceOffsetInHeader) = 0; + *(HeaderPtr + RecordTypeOffsetInHeader) = 0; + + // RecordLength is already set and we don't set key here; we wait for Revivification to do that. But we must update the sizeInfo + // to ensure the AllocatedInlineRecordSize retains recordLength when LogRecord.InitializeRecord is called. + sizeInfo.AllocatedInlineRecordSize = recordLength; + sizeInfo.SetIsRevivifiedRecord(); + } + + /// Set the record length; this is ONLY to be used for temporary copies (e.g. serialization for Migration and Replication). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly void SetRecordLength(int newRecordLength) + { + // This might leave extra bytes in the record length field if the new length uses fewer bytes than the previous length but this is only + // temporary so it is acceptable. + var recordLengthMask = (1UL << (DeconstructKVByteLengths(out _ /*headerLength*/).numRecordLengthBytes * 8)) - 1; + *(ulong*)HeaderPtr = (*(ulong*)HeaderPtr & ~(recordLengthMask << kRecordLengthShiftInHeader)) | (((ulong)newRecordLength & recordLengthMask) << kRecordLengthShiftInHeader); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (int numKeyLengthBytes, int numRecordLengthBytes) DeconstructKVByteLengths(out int headerLength) + { + var indicator = *HeaderPtr; + var numRecordLengthBytes = ((indicator >> kRecordLengthIndicatorShift) & kRecordLengthIndicatorBitMask) + 1; // RecordLength does not allow zero, so add 1 + var numKeyLengthBytes = ((indicator >> kKeyLengthIndicatorShift) & kKeyLengthIndicatorBitMask) + 1; // KeyLength does not allow zero, so add 1 + headerLength = NumIndicatorBytes + numKeyLengthBytes + numRecordLengthBytes; + return (numKeyLengthBytes, numRecordLengthBytes); + } + + /// Get the offset of the key, relative to the start. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetOffsetToKeyStart(int headerLength) => RecordInfo.Size + headerLength + ExtendedNamespaceLength; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetFillerLength(RecordInfo recordInfo, int recordLength) + => recordInfo.HasFiller ? GetFillerLength(recordLength) : 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetFillerLength(RecordInfo recordInfo, out int recordLength) + { + recordLength = GetRecordLength(DeconstructKVByteLengths(out _ /*headerLength*/).numRecordLengthBytes); + return recordInfo.HasFiller ? GetFillerLength(recordLength) : 0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private readonly int GetFillerLength(int recordLength) + { + var fillerLen = (*HeaderPtr >> kFillerLengthIndicatorShift) & kFillerLengthIndicatorBitMask; + return fillerLen < 3 ? fillerLen + 1 : *(int*)((long)RecordInfoPtr + recordLength - LogRecord.FillerLengthSize); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly void SetFillerLength(ref RecordInfo recordInfo, int recordLength, int fillerLength) + { + Debug.Assert(fillerLength >= 0, $"Filler length {fillerLength} must be nonnegative"); + + if (fillerLength > 0) + { + recordInfo.SetHasFiller(); + if (fillerLength <= 3) + { + // Store only the indicator bits; we don't have space for the full int. Mask out previous bits in the filler space first. + *HeaderPtr = (byte)((*HeaderPtr & ~(kFillerLengthIndicatorBitMask << kFillerLengthIndicatorShift)) | ((fillerLength - 1) << kFillerLengthIndicatorShift)); + } + else + { + // Store the indicator bits as 3, and the filler length in the int at the end of the record. 3 is "all bits set" in the filler space, so we don't need to mask out previous bits there. + *HeaderPtr |= 3 << kFillerLengthIndicatorShift; + *(int*)((long)RecordInfoPtr + recordLength - LogRecord.FillerLengthSize) = fillerLength; + } + } + else + { + recordInfo.ClearHasFiller(); + *HeaderPtr = (byte)(*HeaderPtr & ~(kFillerLengthIndicatorBitMask << kFillerLengthIndicatorShift)); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetRecordLength() => GetRecordLength(DeconstructKVByteLengths(out _ /*headerLength*/).numRecordLengthBytes); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetRecordLength(int numRecordLengthBytes) + { + // See notes in Initialize() about layout of RecordLength in header and for the "set" side of this--keep them in sync. + var recordLengthMask = (1UL << (numRecordLengthBytes * 8)) - 1; + return (int)((*(ulong*)HeaderPtr >> kRecordLengthShiftInHeader) & recordLengthMask); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly int GetKeyLength(int numKeyLengthBytes, int numRecordLengthBytes) + { + var keyLengthMask = (1UL << (numKeyLengthBytes * 8)) - 1; + var ptrBackup = sizeof(ulong) - NumIndicatorBytes - numRecordLengthBytes - numKeyLengthBytes; // If negative, the pointer advances + var keyLenPtr = (ulong*)(HeaderPtr - ptrBackup); + var keyLengthShiftInHeader = (sizeof(ulong) - numKeyLengthBytes) * 8; + return (int)((*keyLenPtr >> keyLengthShiftInHeader) & keyLengthMask); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (int keyLength, int valueLength) GetKVLengths(RecordInfo recordInfo) + => GetKVLengths(recordInfo, out _ /* recordLength */, out _ /* eTagLen */, out _ /* expirationLen */, out _ /* objectLogPositionLen */, out _ /* fillerLen */, out _ /*valueAddress*/); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (int keyLength, int valueLength) GetKVLengths(RecordInfo recordInfo, out int recordLength, out int eTagLen, out int expirationLen, out int objectLogPositionLen, out int fillerLen, out long valueAddress) + { + var (numKeyLengthBytes, numRecordLengthBytes) = DeconstructKVByteLengths(out var headerLength); + + // Include changeable fields that are set or cleared by the caller, and the objectLogPosition which is indirectly set by the caller when changing + // the state of recordInfo.RecordIsInline. Namespace is not included; immutable and conceptually part of the key, it is not part of the record content. + // Returning these is useful for length-change calculations, and we must retrieve them anyway to determine object size. + eTagLen = recordInfo.HasETag ? LogRecord.ETagSize : 0; + expirationLen = recordInfo.HasExpiration ? LogRecord.ExpirationSize : 0; + objectLogPositionLen = recordInfo.RecordIsInline ? 0 : LogRecord.ObjectLogPositionSize; + + // See note in Initialize about layout of lengths in header + var keyLength = GetKeyLength(numKeyLengthBytes, numRecordLengthBytes); + recordLength = GetRecordLength(numRecordLengthBytes); + fillerLen = GetFillerLength(recordInfo, recordLength); + + // The value length is the recordLength minus everything other than the value. To get valueAddress, back up the HeaderPtr to the start of the RecordInfo then add key offset and size. + var keyOffset = RecordInfo.Size + headerLength + ExtendedNamespaceLength; + valueAddress = (long)HeaderPtr - RecordInfo.Size + keyOffset + keyLength; + return (keyLength, recordLength - keyOffset - keyLength - recordInfo.GetOptionalSize() - fillerLen); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (int keyLength, long keyAddress) GetKeyFieldInfo() => GetKeyFieldInfo(out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (int keyLength, long keyAddress) GetKeyFieldInfo(out int numKeyLengthBytes, out int numRecordLengthBytes) + { + (numKeyLengthBytes, numRecordLengthBytes) = DeconstructKVByteLengths(out var headerLength); + + // See note in Initialize about layout of lengths in header + var keyLength = GetKeyLength(numKeyLengthBytes, numRecordLengthBytes); + var keyAddress = (long)(HeaderPtr + headerLength + ExtendedNamespaceLength); + return (keyLength, keyAddress); + } + + /// + /// Gets the value field information for an in-memory or on-disk with object size changes to value length restored (objects have been read). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (long valueLength, long valueAddress) GetValueFieldInfo(RecordInfo recordInfo) + => GetValueFieldInfo(recordInfo, out _ /*keyLength*/, out _ /*numKeyLengthBytes*/, out _ /*numRecordLengthBytes*/); + + /// + /// Gets the value field information for an in-memory or on-disk with object size changes to value length restored (objects have been read). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal readonly (long valueLength, long valueAddress) GetValueFieldInfo(RecordInfo recordInfo, out int keyLength, out int numKeyLengthBytes, out int numRecordLengthBytes) + { + (keyLength, var keyAddress) = GetKeyFieldInfo(out numKeyLengthBytes, out numRecordLengthBytes); + var headerLength = NumIndicatorBytes + numKeyLengthBytes + numRecordLengthBytes; + + var recordLength = GetRecordLength(numRecordLengthBytes); + var fillerLength = GetFillerLength(recordInfo, recordLength); + + // The value length is the recordLength minus everything other than the value. + var valueLength = recordLength - RecordInfo.Size - headerLength - ExtendedNamespaceLength - keyLength - recordInfo.GetOptionalSize() - fillerLength; + + // Move past the key and value length bytes and the key data to the start of the value data + return (valueLength, keyAddress + keyLength); + } + + internal readonly int GetAllocatedRecordSize() => GetRecordLength(DeconstructKVByteLengths(out _ /*headerLength*/).numRecordLengthBytes); + + internal readonly int GetActualRecordSize(RecordInfo recordInfo) + { + var recordLength = GetRecordLength(DeconstructKVByteLengths(out _ /*headerLength*/).numRecordLengthBytes); + return recordLength - GetFillerLength(recordInfo, recordLength); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/OptionalFieldsShift.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/OptionalFieldsShift.cs new file mode 100644 index 00000000000..26ad43bfceb --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/OptionalFieldsShift.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Saves optional fields ETag and Expiration during a record-resizing operation and restores them when done. + /// + /// + /// We don't save ObjectLogPosition; that's only used during Serialization. The caller (TrySetValueLength) adjusts filler + /// address and length by the growth (positive or negative) of the object value, so no address adjustment or zeroing of + /// space is needed. + /// + internal unsafe struct OptionalFieldsShift + { + long eTag = LogRecord.NoETag; + long expiration = LogRecord.NoExpiration; + + public OptionalFieldsShift() { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Save(long address, RecordInfo recordInfo) + { + if (recordInfo.HasETag) + { + eTag = *(long*)address; + address += LogRecord.ETagSize; + } + if (recordInfo.HasExpiration) + { + expiration = *(long*)address; + address += LogRecord.ExpirationSize; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Restore(long address, in RecordSizeInfo sizeInfo, ref RecordInfo recordInfo) + { + if (sizeInfo.FieldInfo.HasETag) + { + *(long*)address = eTag; + address += LogRecord.ETagSize; + recordInfo.SetHasETag(); + } + else + recordInfo.ClearHasETag(); + + if (sizeInfo.FieldInfo.HasExpiration) + { + *(long*)address = expiration; + address += LogRecord.ExpirationSize; + recordInfo.SetHasExpiration(); + } + else + recordInfo.ClearHasExpiration(); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/OverflowByteArray.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/OverflowByteArray.cs new file mode 100644 index 00000000000..f34eed9a801 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/OverflowByteArray.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Tsavorite.core +{ + /// A byte[] wrapper that encodes start and end offsets of the actual data in the first sizeof() bytes in the array. + /// Used primarily for sector-aligned reads directly into the overflow byte[]. + public struct OverflowByteArray + { + /// Define the header of an overflow allocation. Overflow allocations are typically large so use a full int to allow greater than 64k offsets, + /// which makes it possible to read more information with a single IO and then copy it out to other destinations. Sector sizes may be up to 64k on NTFS systems, + /// which is sizeof(ushort) bytes, so the use of full ints removes boundary concerns (e.g. reading a value followed by optional bytes may cross a sector boundary, + /// in which case we need an end offset greater than a single sector). + struct OverflowHeader + { + internal const int Size = 2 * sizeof(int); + internal int startOffset, endOffset; + } + + internal readonly byte[] Array { get; init; } + + internal readonly bool IsEmpty => Array is null; + + internal readonly int StartOffset => Unsafe.As(ref Array[0]).startOffset + OverflowHeader.Size; + + /// The total size of the array allocated (includes space for offset values) + public readonly int TotalSize => Array.Length; + + /// The total heap size of the array (includes space for offset values and .net array overhead) + public readonly int HeapMemorySize => Array is null ? 0 : Array.Length + MemoryUtils.ByteArrayOverhead; + + readonly int EndOffset => Unsafe.As(ref Array[0]).endOffset; + + internal readonly int Length => Array.Length - StartOffset - EndOffset; + + /// + public override string ToString() => $"Len {Length}, IsEmpty {IsEmpty}, sOffset {StartOffset}, eOffset {EndOffset}, HeapMemSize {HeapMemorySize}, TotSize {TotalSize}"; + + /// ReadOnlySpan of data between offsets + internal readonly ReadOnlySpan ReadOnlySpan => Array.AsSpan(StartOffset, Length); + /// ReadOnlySpan of data between offsets + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ReadOnlySpan AsReadOnlySpan(int start) + { + var length = Length; + return start <= length ? Array.AsSpan(StartOffset + start, length - start) : throw new ArgumentOutOfRangeException(nameof(start)); + } + /// ReadOnlySpan of data between offsets + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ReadOnlySpan AsReadOnlySpan(int start, int len) + { + var length = Length; + return ((ulong)(uint)start + (uint)len <= (uint)length) ? Array.AsSpan(StartOffset + start, len) : throw new ArgumentOutOfRangeException($"start {nameof(start)} + len {len} exceeds length {length}"); + } + + /// Span of data between offsets + internal readonly Span Span => Array.AsSpan(StartOffset, Length); + /// Span of data between offsets + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Span AsSpan(int start) + { + var length = Length; + return start <= length ? Array.AsSpan(StartOffset + start, length - start) : throw new ArgumentOutOfRangeException(nameof(start)); + } + /// ReadOnlySpan of data between offsets + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Span AsSpan(int start, int len) + { + var length = Length; + return ((ulong)(uint)start + (uint)len <= (uint)length) ? Array.AsSpan(StartOffset + start, len) : throw new ArgumentOutOfRangeException($"start {nameof(start)} + len {len} exceeds length {length}"); + } + + /// Span of all data, including before and after offsets; this is for aligned Read from the device. + internal readonly Span AlignedReadSpan => Array.AsSpan(OverflowHeader.Size); + + /// + /// Get a view over the value bytes (between and the end-offset). + /// + /// + /// The returned aliases the underlying byte[]; consumers that need a stable native pointer + /// (e.g. for SIMD operations) should call for the duration of the operation, otherwise + /// GC compaction may relocate the array. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Memory AsMemory() => Array.AsMemory(StartOffset, Length); + + /// Construct an from a byte[] allocated by . + internal OverflowByteArray(byte[] data) => Array = data; + + internal OverflowByteArray(int length, int startOffset, int endOffset, bool zeroInit) + { + // Allocate with enough extra space for the metadata (offset from start and end) + Array = !zeroInit + ? GC.AllocateUninitializedArray(length + OverflowHeader.Size) + : (new byte[length + OverflowHeader.Size]); + ref var header = ref Unsafe.As(ref Array[0]); + header.startOffset = startOffset; + header.endOffset = endOffset; + } + + /// Increase the offset from the start, e.g. after having extracted the key that was read in the same IO operation as the value. + /// This is 'readonly' because it does not alter the array field, only its contents. + internal readonly void AdjustOffsetFromStart(int increment) => Unsafe.As(ref Array[0]).startOffset += increment; + /// Increase the offset from the end, e.g. after having extracted the optionals that were read in the same IO operation as the value. + /// This is 'readonly' because it does not alter the > array field, only its contents. + internal readonly void AdjustOffsetFromEnd(int increment) => Unsafe.As(ref Array[0]).endOffset += increment; + + internal readonly void SetOffsets(int offsetFromStart, int offsetFromEnd) + { + Debug.Assert(offsetFromStart > 0 && offsetFromStart < Array.Length - 1, "offsetFromStart is out of range"); + Debug.Assert(offsetFromEnd > 0 && offsetFromEnd < Array.Length - 1, "offsetFromEnd is out of range"); + Debug.Assert(offsetFromStart < offsetFromEnd, "offsetFromStart must be less than offsetFromEnd"); + ref var header = ref Unsafe.As(ref Array[0]); + header.startOffset = offsetFromStart; + header.endOffset = offsetFromEnd; + } + + /// Pin the underlying heap object. It is the caller's responsibility to release the returned . + public readonly GCHandle Pin() + => GCHandle.Alloc(Array, GCHandleType.Pinned); + + /// Get the of a byte[] allocated by constructor. + internal static ReadOnlySpan AsReadOnlySpan(object value) => new OverflowByteArray(Unsafe.As(value)).ReadOnlySpan; + + /// Get the of a byte[] allocated by constructor. + internal static Span AsSpan(object value) => new OverflowByteArray(Unsafe.As(value)).Span; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/PageHeader.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/PageHeader.cs new file mode 100644 index 00000000000..290d18efa95 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/PageHeader.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.InteropServices; + +namespace Tsavorite.core +{ + [StructLayout(LayoutKind.Explicit, Size = Size)] + public struct PageHeader + { + const ushort CurrentVersion = 1; + + /// The number of bits in the size of the struct. Currently set to make the size that the 0'th page offset was in earlier versions; 64 bytes + internal const int SizeBits = 6; + + /// The size of the struct. Must be a power of 2. Currently set to the size that the 0'th page offset was; 64 bytes + public const int Size = 1 << SizeBits; + + /// Version of this page header. + [FieldOffset(0)] + internal ushort version; + + [FieldOffset(sizeof(ushort))] + internal ushort unusedUshort1; + + [FieldOffset(sizeof(int))] + internal int unusedInt1; + + /// The lowest object-log position on this main-log page, if ObjectAllocator. Contains both segmentId and offset on segment + [FieldOffset(sizeof(long))] + internal ulong objectLogLowestPositionWord; + + // Unused; as they become used, start with higher # + [FieldOffset(sizeof(long) * 2)] + internal long unusedLong6; + [FieldOffset(sizeof(long) * 3)] + internal long unusedLong5; + [FieldOffset(sizeof(long) * 4)] + internal long unusedLong4; + [FieldOffset(sizeof(long) * 5)] + internal long unusedLong3; + [FieldOffset(sizeof(long) * 6)] + internal long unusedLong2; + [FieldOffset(sizeof(long) * 7)] + internal long unusedLong1; + + /// + /// Initializes the struct. + /// + /// + internal void Initialize() + { + this = default; + version = CurrentVersion; + objectLogLowestPositionWord = ObjectLogFilePositionInfo.NotSet; + } + + internal static unsafe void Initialize(long physicalAddressOfStartOfPage) => (*(PageHeader*)physicalAddressOfStartOfPage).Initialize(); + + /// + /// Set the lowest object-log position on this main-log page, if ObjectAllocator. + /// + /// The position in the object log. + internal void SetLowestObjectLogPosition(in ObjectLogFilePositionInfo position) + { + if (objectLogLowestPositionWord == ObjectLogFilePositionInfo.NotSet) + objectLogLowestPositionWord = position.word; + } + + /// + /// Set the lowest object-log position on this main-log page, if ObjectAllocator. + /// + /// The number of bits in the object log's segments. + internal ObjectLogFilePositionInfo GetLowestObjectLogPosition(int segmentBits) + => objectLogLowestPositionWord == ObjectLogFilePositionInfo.NotSet ? new() : new(objectLogLowestPositionWord, segmentBits); + + public override readonly string ToString() + => $"ver {version}, lowObjLogPos {objectLogLowestPositionWord}, us1 {unusedUshort1}, ui1 {unusedInt1}, ul1 {unusedLong1}, ul2 {unusedLong2}, ul3 {unusedLong3}, ul4 {unusedLong4}, ul5 {unusedLong5}, ul6 {unusedLong6}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/PageUnit.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/PageUnit.cs index 5d093c3fd55..60a78fb7819 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/PageUnit.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/PageUnit.cs @@ -5,10 +5,19 @@ namespace Tsavorite.core { - struct PageUnit + struct PageUnit { - public byte[] value; + /// The byte array of this circular buffer page + public byte[] array; + + /// The pinned pointer to this circular buffer page public long pointer; + + /// The specific allocator's associated value for this circular buffer page + public TValuePage value; + + /// + public override readonly string ToString() => $"Value {value}, Pointer {pointer}, Array.Length {array.Length}"; } [StructLayout(LayoutKind.Explicit)] @@ -16,8 +25,8 @@ internal struct FullPageStatus { [FieldOffset(0)] public long LastFlushedUntilAddress; - [FieldOffset(8)] - public long Dirty; + /// + public override readonly string ToString() => $"LastFUA {LastFlushedUntilAddress}"; } [StructLayout(LayoutKind.Explicit)] @@ -29,5 +38,8 @@ internal struct PageOffset public int Page; [FieldOffset(0)] public long PageAndOffset; + + /// + public override readonly string ToString() => $"Page {Page}, Offset {Offset}"; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ScanCursorState.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ScanCursorState.cs index eeddbebc452..3b74a8f4fa2 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/ScanCursorState.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ScanCursorState.cs @@ -3,15 +3,15 @@ namespace Tsavorite.core { - internal sealed class ScanCursorState + internal sealed class ScanCursorState { - internal IScanIteratorFunctions functions; + internal IScanIteratorFunctions functions; internal long acceptedCount; // Number of records pushed to and accepted by the caller internal bool endBatch; // End the batch (but return a valid cursor for the next batch, as if "count" records had been returned) internal bool retryLastRecord; // Retry the last record when returning a valid cursor internal bool stop; // Stop the operation (as if all records in the db had been returned) - internal void Initialize(IScanIteratorFunctions scanIteratorFunctions) + internal void Initialize(IScanIteratorFunctions scanIteratorFunctions) { functions = scanIteratorFunctions; acceptedCount = 0; diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/ScanIteratorBase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/ScanIteratorBase.cs index 10c04f4763a..24f846defd4 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/ScanIteratorBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/ScanIteratorBase.cs @@ -2,48 +2,65 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; using System.Threading; using Microsoft.Extensions.Logging; namespace Tsavorite.core { +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + /// /// Scan iterator for hybrid log /// public abstract class ScanIteratorBase { - /// - /// Frame size - /// + /// Frame size (1 or 2) protected readonly int frameSize; - /// - /// Begin address. Cannot be readonly due to SnapCursorToLogicalAddress - /// + /// Begin address of the scan. Cannot be readonly due to SnapCursorToLogicalAddress protected long beginAddress; - /// - /// End address - /// + /// End address of the scan protected readonly long endAddress; - /// - /// Epoch - /// + /// Epoch from the store protected readonly LightEpoch epoch; - /// - /// Current and next address for iteration - /// - protected long currentAddress, nextAddress; + /// Number of deferred DoReadPage drain callbacks that have been registered but not yet executed. + protected int pendingDrainCallbacks; + + /// Current address for iteration + protected long currentAddress; + /// Next address for iteration + protected long nextAddress; + + /// vector for waiting for frame-load completion. + /// This array is in parallel with , , and . + private CountdownEvent[] loadCompletionEvents; + + /// vector for canceling the wait for frame-load completion. + /// This array is in parallel with , , and . + private CancellationTokenSource[] loadCTSs; + + /// Vector of endAddresses for the currently loaded pages of the frames. + /// This array is in parallel with , , and . + private long[] loadedPages; + + /// Vector of endAddresses for the currently in-flight, and possibly completed, loading of pages of the frames. + /// This is updated atomically when we start the of a page. + /// This array is in parallel with , , and . + private long[] nextLoadedPages; + + /// The circular buffer we cycle through for object-log deserialization. + CircularDiskReadBuffer[] objectReadBuffers; - private CountdownEvent[] loaded; - private CancellationTokenSource[] loadedCancel; - private long[] loadedPage; - private long[] nextLoadedPage; + /// Number of bits in the size of the log page private readonly int logPageSizeBits; + + /// Whether to include closed records in the scan protected readonly bool includeClosedRecords; - protected readonly bool returnTombstoned; /// /// Current address @@ -70,20 +87,20 @@ public abstract class ScanIteratorBase /// protected ILogger logger; + /// + /// Buffering for holding copies of in-memory records + /// + protected InMemoryScanBufferingMode memScanBufferingMode; + /// /// Constructor /// - /// - /// - /// - /// - /// - /// - /// - /// - public unsafe ScanIteratorBase(long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords, LightEpoch epoch, int logPageSizeBits, bool initForReads = true, ILogger logger = null) + public unsafe ScanIteratorBase(long beginAddress, long endAddress, DiskScanBufferingMode diskScanBufferingMode, InMemoryScanBufferingMode memScanBufferingMode, + bool includeClosedRecords, LightEpoch epoch, int logPageSizeBits, bool initForReads = true, ILogger logger = null) { this.logger = logger; + this.memScanBufferingMode = memScanBufferingMode; + // If we are protected when creating the iterator, we do not need per-GetNext protection if (epoch != null && !epoch.ThisInstanceProtected()) this.epoch = epoch; @@ -96,139 +113,213 @@ public unsafe ScanIteratorBase(long beginAddress, long endAddress, ScanBuffering currentAddress = -1; nextAddress = beginAddress; - if (scanBufferingMode == ScanBufferingMode.SinglePageBuffering) + if (diskScanBufferingMode == DiskScanBufferingMode.SinglePageBuffering) frameSize = 1; - else if (scanBufferingMode == ScanBufferingMode.DoublePageBuffering) + else if (diskScanBufferingMode == DiskScanBufferingMode.DoublePageBuffering) frameSize = 2; - else if (scanBufferingMode == ScanBufferingMode.NoBuffering) + else if (diskScanBufferingMode == DiskScanBufferingMode.NoBuffering) { frameSize = 0; return; } - if (initForReads) InitializeForReads(); + if (initForReads) + InitializeForReads(); } - /// - /// Initialize for reads - /// + /// Initialize fields for read callback management public virtual void InitializeForReads() { - loaded = new CountdownEvent[frameSize]; - loadedCancel = new CancellationTokenSource[frameSize]; - loadedPage = new long[frameSize]; - nextLoadedPage = new long[frameSize]; - for (int i = 0; i < frameSize; i++) + loadCompletionEvents = new CountdownEvent[frameSize]; + loadCTSs = new CancellationTokenSource[frameSize]; + loadedPages = new long[frameSize]; + nextLoadedPages = new long[frameSize]; + for (var i = 0; i < frameSize; i++) { - loadedPage[i] = -1; - nextLoadedPage[i] = -1; - loadedCancel[i] = new CancellationTokenSource(); + loadedPages[i] = -1; + nextLoadedPages[i] = -1; + loadCTSs[i] = new CancellationTokenSource(); } currentAddress = -1; nextAddress = beginAddress; } + /// Initialize read buffers + public virtual void InitializeReadBuffers(AllocatorBase allocatorBase = default) + { + objectReadBuffers = new CircularDiskReadBuffer[frameSize]; + for (var i = 0; i < frameSize; i++) + objectReadBuffers[i] = allocatorBase?.CreateCircularReadBuffers(); + } + /// /// Buffer and load /// - /// - /// - /// - /// - /// + /// The current logical address + /// The page containing the current logical address + /// The frame index of the current page (the page modulo the number of frames) + /// Head address of the log + /// Address to stop the scan at + /// True we had to await the event here; /// - protected unsafe bool BufferAndLoad(long currentAddress, long currentPage, long currentFrame, long headAddress, long endAddress) + protected bool BufferAndLoad(long currentIterationAddress, long currentPage, long currentFrame, long headAddress, long endIterationAddress) { - for (int i = 0; i < frameSize; i++) + for (var i = 0; i < frameSize; i++) { + // Read the next page. If i == 0 this is the page we are about to iterate; if i > 0, then we are issuing read-ahead for efficiency. var nextPage = currentPage + i; - var pageStartAddress = nextPage << logPageSizeBits; - // Cannot load page if it is entirely in memory or beyond the end address - if (pageStartAddress >= headAddress || pageStartAddress >= endAddress) + // Cannot load nextPage if it is entirely in memory or beyond the end address + var pageStartAddress = GetLogicalAddressOfStartOfPage(nextPage, logPageSizeBits); + if (pageStartAddress >= headAddress || pageStartAddress >= endIterationAddress) continue; - var pageEndAddress = (nextPage + 1) << logPageSizeBits; - if (endAddress < pageEndAddress) - pageEndAddress = endAddress; - if (headAddress < pageEndAddress) - pageEndAddress = headAddress; + // Determine the endAddress on nextPage, which may be limited by endAddress or headAddress to be before end of page. + var pageEndAddress = GetLogicalAddressOfStartOfPage(nextPage + 1, logPageSizeBits); + if (endIterationAddress < pageEndAddress) + pageEndAddress = endIterationAddress; + // With HeadAddress now possibly in the middle of the page, we have to ensure we handle re-entering with the same currentFrame while + // a previous request on currentFrame is ongoing; this is ensured by CalculateReadOnlyAddress. So just read the entire page regardless + // of headAddress; the entire page will have been flushed to disk already. TODO Leaving this here in case we change to record-aligned ReadOnlyAddress. + //if (headAddress < pageEndAddress) + // pageEndAddress = headAddress; + + // Calculate the nextFrame we will load nextPage into var nextFrame = (currentFrame + i) % frameSize; - long val; - while ((val = nextLoadedPage[nextFrame]) < pageEndAddress || loadedPage[nextFrame] < pageEndAddress) + // Loop using CAS as a latch-free way to ensure only one thread issues the load for nextPage into nextFrame. + while (true) { - if (val < pageEndAddress && Interlocked.CompareExchange(ref nextLoadedPage[nextFrame], pageEndAddress, val) == val) + // Get the endAddress of the next page being loaded for this frame. If it is already loaded, as indicated by being >= the required endAddress, we're done. + var val = nextLoadedPages[nextFrame]; + if (val >= pageEndAddress && loadedPages[nextFrame] >= pageEndAddress) + break; + + // If the endAddress of the next page being loaded is less than the endAddress we need for the next page for this frame, + // try to atomically exchange it with the endAddress we need. If successful, issue the load. + if (val < pageEndAddress && Interlocked.CompareExchange(ref nextLoadedPages[nextFrame], pageEndAddress, val) == val) { - var tmp_i = i; + Debug.Assert(loadCompletionEvents[nextFrame] is null || loadCompletionEvents[nextFrame].IsSet, + $"i {i}, currentAddress {currentIterationAddress}, currentFrame {currentFrame}, nextFrame {nextFrame} overwriting unset completion event"); + var readBuffer = objectReadBuffers is not null ? objectReadBuffers[nextFrame] : default; + + var frameIndex = i; + _ = Interlocked.Increment(ref pendingDrainCallbacks); if (epoch != null) - { - epoch.BumpCurrentEpoch(() => - { - AsyncReadPagesFromDeviceToFrame(tmp_i + (currentAddress >> logPageSizeBits), 1, endAddress, Empty.Default, out loaded[nextFrame], 0, null, null, loadedCancel[nextFrame]); - loadedPage[nextFrame] = pageEndAddress; - }); - } + epoch.BumpCurrentEpoch(() => DoReadPage(frameIndex)); else + DoReadPage(frameIndex); + + void DoReadPage(int frameIndex) { - AsyncReadPagesFromDeviceToFrame(tmp_i + (currentAddress >> logPageSizeBits), 1, endAddress, Empty.Default, out loaded[nextFrame], 0, null, null, loadedCancel[nextFrame]); - loadedPage[nextFrame] = pageEndAddress; + try + { + AsyncReadPageFromDeviceToFrame(readBuffer, readPage: frameIndex + GetPageOfAddress(currentIterationAddress, logPageSizeBits), untilAddress: endIterationAddress, + context: Empty.Default, out loadCompletionEvents[nextFrame], devicePageOffset: 0, device: null, objectLogDevice: null, loadCTSs[nextFrame]); + } + catch + { + _ = Interlocked.Decrement(ref pendingDrainCallbacks); + throw; + } + loadedPages[nextFrame] = pageEndAddress; } } else + { + // Someone else incremented nextLoadedPage[nextFrame] or the BumpCE has not completed and set loadedPages, so give things a chance to work and try again. epoch?.ProtectAndDrain(); + } } } - return WaitForFrameLoad(currentAddress, currentFrame); + + // Wait only for currentFrame; nextFrame(s, if we ever have frameSize > 2) will process in the background until we actually need its data, + // in which case it will come in here as currentFrame, see that nextLoadedPage is already set, and then this line will wait for it. + // WaitForFrameLoad returns immediately if the wait has already been satisfied. + return WaitForFrameLoad(currentIterationAddress, currentFrame); } /// /// Whether we need to buffer new page from disk /// - protected unsafe bool NeedBufferAndLoad(long currentAddress, long currentPage, long currentFrame, long headAddress, long endAddress) + protected bool NeedBufferAndLoad(long currentAddress, long currentPage, long currentFrame, long headAddress, long endAddress) { - for (int i = 0; i < frameSize; i++) + for (var i = 0; i < frameSize; i++) { + // Read the next page. If i == 0 this is the page we are about to iterate; if i > 0, then we are issuing read-ahead for efficiency. var nextPage = currentPage + i; - var pageStartAddress = nextPage << logPageSizeBits; + var pageStartAddress = GetLogicalAddressOfStartOfPage(nextPage, logPageSizeBits); - // Cannot load page if it is entirely in memory or beyond the end address + // Cannot load nextPage if it is entirely in memory or beyond the end address if (pageStartAddress >= headAddress || pageStartAddress >= endAddress) continue; - var pageEndAddress = (nextPage + 1) << logPageSizeBits; + // Determine the endAddress on nextPage, which may be limited by endAddress or headAddress to be before end of page. + var pageEndAddress = GetLogicalAddressOfStartOfPage(nextPage + 1, logPageSizeBits); if (endAddress < pageEndAddress) pageEndAddress = endAddress; if (headAddress < pageEndAddress) pageEndAddress = headAddress; + // Calculate the nextFrame we will load nextPage into var nextFrame = (currentFrame + i) % frameSize; - if (nextLoadedPage[nextFrame] < pageEndAddress || loadedPage[nextFrame] < pageEndAddress) + // If the endAddress of the next page being loaded for this frame is already loaded, as indicated by being >= the required endAddress, + // we don't need to load. + if (nextLoadedPages[nextFrame] < pageEndAddress || loadedPages[nextFrame] < pageEndAddress) return true; } return false; } - internal abstract void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, + internal abstract void AsyncReadPageFromDeviceToFrame(CircularDiskReadBuffer readBuffers, long readPage, long untilAddress, TContext context, out CountdownEvent completed, long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null); + protected void AsyncReadPageFromDeviceToFrameCallback(uint errorCode, uint numBytes, object context) + { + try + { + var result = (PageAsyncReadResult)context; + + if (errorCode == 0) + _ = result.handle?.Signal(); + else + { + logger?.LogError($"{nameof(AsyncReadPageFromDeviceToFrameCallback)} error: {{errorCode}}", errorCode); + result.cts?.Cancel(); + } + } + finally + { + _ = Interlocked.Decrement(ref pendingDrainCallbacks); + } + } + + /// + /// Wait for the current frame to complete loading + /// + /// + /// + /// True if we had to wait for the current frame load to complete; else false + /// private bool WaitForFrameLoad(long currentAddress, long currentFrame) { - if (loaded[currentFrame].IsSet) + if (loadCompletionEvents[currentFrame].IsSet) return false; try { epoch?.Suspend(); - loaded[currentFrame].Wait(loadedCancel[currentFrame].Token); // Ensure we have completed ongoing load + loadCompletionEvents[currentFrame].Wait(loadCTSs[currentFrame].Token); // Ensure we have completed ongoing load } catch (Exception e) { - loadedPage[currentFrame] = -1; - loadedCancel[currentFrame] = new CancellationTokenSource(); - Utility.MonotonicUpdate(ref nextAddress, (1 + (currentAddress >> logPageSizeBits)) << logPageSizeBits, out _); + // Exception occurred so skip the page containing the currentAddress, and reinitialize the loaded page and cancellation token for the current frame. + // The exception may have been an OperationCanceledException. + loadedPages[currentFrame] = -1; + loadCTSs[currentFrame] = new CancellationTokenSource(); + _ = Utility.MonotonicUpdate(ref nextAddress, GetLogicalAddressOfStartOfPage(1 + GetPageOfAddress(currentAddress, logPageSizeBits), logPageSizeBits), out _); throw new TsavoriteException("Page read from storage failed, skipping page. Inner exception: " + e.ToString()); } finally @@ -243,21 +334,35 @@ private bool WaitForFrameLoad(long currentAddress, long currentFrame) /// public virtual void Dispose() { - if (loaded != null) + // Wait for all deferred DoReadPage callbacks and their async I/O to complete before freeing + // resources. The counter is incremented before BumpCurrentEpoch registration and decremented + // in AsyncReadPageFromDeviceToFrameCallback when I/O completes, so reaching zero guarantees + // no outstanding access to our state. The deferred callbacks will be drained by other threads' + // epoch operations (Resume, Suspend, ProtectAndDrain). + while (Volatile.Read(ref pendingDrainCallbacks) > 0) + Thread.Yield(); + + for (var i = 0; i < frameSize; i++) { - // Wait for ongoing reads to complete/fail - for (int i = 0; i < frameSize; i++) + // Wait for ongoing reads to complete/fail; if the wait throws (e.g. due to cancellation), we still + // need to dispose the event, CTS, and read buffers below. + try { - if (loadedPage[i] != -1) - { - try - { - loaded[i].Wait(loadedCancel[i].Token); - } - catch { } - } + if (loadCompletionEvents != null && loadedPages[i] != -1) + loadCompletionEvents[i]?.Wait(loadCTSs[i].Token); } + catch { } + + // Always dispose resources regardless of whether the wait succeeded. + loadCompletionEvents?[i]?.Dispose(); + loadCTSs?[i]?.Dispose(); + loadCTSs?[i] = null; + + // Do not null this; we didn't hold onto the hlogBase to recreate. CircularDiskReadBuffer.Dispose() clears + // things and leaves it in an "initialized" state. + objectReadBuffers?[i]?.Dispose(); } + loadCompletionEvents = default; } /// @@ -265,21 +370,24 @@ public virtual void Dispose() /// public void Reset() { - loaded = new CountdownEvent[frameSize]; - loadedCancel = new CancellationTokenSource[frameSize]; - loadedPage = new long[frameSize]; - nextLoadedPage = new long[frameSize]; - for (int i = 0; i < frameSize; i++) + Dispose(); + loadCompletionEvents = new CountdownEvent[frameSize]; + loadCTSs = new CancellationTokenSource[frameSize]; + loadedPages = new long[frameSize]; + nextLoadedPages = new long[frameSize]; + for (var i = 0; i < frameSize; i++) { - loadedPage[i] = -1; - nextLoadedPage[i] = -1; - loadedCancel[i] = new CancellationTokenSource(); + loadedPages[i] = -1; + nextLoadedPages[i] = -1; + loadCTSs[i] = new CancellationTokenSource(); + // readBuffers do not need to be reset because that is done in its Dispose, leaving it in an "initialized" state. + // Also, OnBeginReadRecords() will do reinitialization internally. } currentAddress = -1; nextAddress = beginAddress; } /// - public override string ToString() => $"BA {BeginAddress}, EA {EndAddress}, CA {CurrentAddress}, NA {NextAddress}"; + public override string ToString() => $"BA {AddressString(BeginAddress)}, EA {AddressString(EndAddress)}, CA {AddressString(CurrentAddress)}, NA {AddressString(NextAddress)}"; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SerializationPhase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SerializationPhase.cs new file mode 100644 index 00000000000..afd48a8f3f4 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/SerializationPhase.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + public enum SerializationPhase : int + { + /// + /// Serialization has not been started. + /// + REST, + + /// + /// Serialization is in progress. + /// + SERIALIZING, + + /// + /// Serialization has been completed. + /// + SERIALIZED + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocator.cs index 4949003aae2..893a1ef900d 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocator.cs @@ -1,13 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Runtime.CompilerServices; namespace Tsavorite.core { - // Allocator for SpanByte Keys and Values. - public struct SpanByteAllocator : IAllocator - where TStoreFunctions : IStoreFunctions + /// + /// Allocator for ReadOnlySpan{byte} Keys and Span{byte} Values. + /// + public struct SpanByteAllocator : IAllocator + where TStoreFunctions : IStoreFunctions { /// The wrapped class containing all data and most actual functionality. This must be the ONLY field in this structure so its size is sizeof(IntPtr). private readonly SpanByteAllocatorImpl _this; @@ -18,108 +21,87 @@ public SpanByteAllocator(AllocatorSettings settings, TStoreFunctions storeFuncti _this = new(settings, storeFunctions, @this => new SpanByteAllocator(@this)); } - public SpanByteAllocator(object @this) + internal SpanByteAllocator(object @this) { // Called by AllocatorBase via primary ctor wrapperCreator _this = (SpanByteAllocatorImpl)@this; } /// - public readonly AllocatorBase GetBase() - where TAllocator : IAllocator - => (AllocatorBase)(object)_this; - - /// - public readonly bool IsFixedLength => false; + public readonly AllocatorBase GetBase() + where TAllocator : IAllocator + => (AllocatorBase)(object)_this; /// public readonly bool HasObjectLog => false; /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetStartLogicalAddress(long page) => _this.GetStartLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetFirstValidLogicalAddress(long page) => _this.GetFirstValidLogicalAddress(page); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetPhysicalAddress(long logicalAddress) => _this.GetPhysicalAddress(logicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref RecordInfo GetInfo(long physicalAddress) - => ref SpanByteAllocatorImpl.GetInfo(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe ref RecordInfo GetInfoFromBytePointer(byte* ptr) - => ref SpanByteAllocatorImpl.GetInfoFromBytePointer(ptr); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref SpanByte GetKey(long physicalAddress) - => ref SpanByteAllocatorImpl.GetKey(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref SpanByte GetValue(long physicalAddress) => ref _this.GetValue(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref SpanByte GetAndInitializeValue(long physicalAddress, long endPhysicalAddress) => ref _this.GetAndInitializeValue(physicalAddress, endPhysicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) => _this.GetRecordSize(physicalAddress); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref SpanByte key, ref TInput input, ref SpanByte value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - where TVariableLengthInput : IVariableLengthInput - => _this.GetRMWCopyDestinationRecordSize(ref key, ref input, ref value, ref recordInfo, varlenInput); - - /// - public (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref SpanByte key) => _this.GetTombstoneRecordSize(ref key); + public readonly void InitializeRecord(TKey key, long logicalAddress, in RecordSizeInfo sizeInfo, ref LogRecord newLogRecord) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => newLogRecord.InitializeRecord(key, in sizeInfo); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetRequiredRecordSize(long physicalAddress, int availableBytes) => _this.GetRequiredRecordSize(physicalAddress, availableBytes); + public readonly RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => _this.GetRMWCopyRecordSize(in srcLogRecord, ref input, varlenInput); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetAverageRecordSize() => _this.GetAverageRecordSize(); + public readonly RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetRMWInitialRecordSize(key, ref input, varlenInput); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetFixedRecordSize() => _this.GetFixedRecordSize(); + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, value, ref input, varlenInput); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref SpanByte key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => _this.GetRMWInitialRecordSize(ref key, ref input, sessionFunctions); + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, value, ref input, varlenInput); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref SpanByte key, ref SpanByte value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput - => _this.GetUpsertRecordSize(ref key, ref value, ref input, sessionFunctions); + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => _this.GetUpsertRecordSize(key, in inputLogRecord, ref input, varlenInput); - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref SpanByte key, ref SpanByte value) => _this.GetRecordSize(ref key, ref value); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly int GetValueLength(ref SpanByte value) - => SpanByteAllocatorImpl.GetValueLength(ref value); + /// Get record size required for a new tombstone record + public readonly RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _this.GetDeleteRecordSize(key); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) - => SpanByteAllocatorImpl.RetrievedFullRecord(record, ref ctx); + public readonly void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) => _this.PopulateRecordSizeInfo(ref sizeInfo); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -127,56 +109,34 @@ public readonly unsafe bool RetrievedFullRecord(byte* record, ref AsyncIOContext /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly bool IsAllocated(int pageIndex) => _this.IsAllocated(pageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe void PopulatePage(byte* src, int required_bytes, long destinationPageIndex) => _this.PopulatePage(src, required_bytes, destinationPageIndex); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPage(long logicalAddress, long version) => _this.MarkPage(logicalAddress, version); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void MarkPageAtomic(long logicalAddress, long version) => _this.MarkPageAtomic(logicalAddress, version); + public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void ClearPage(long page, int offset = 0) => _this.ClearPage(page, offset); + public readonly int OverflowPageCount => _this.OverflowPageCount; /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); + public readonly LogRecord CreateLogRecord(long logicalAddress) => _this.CreateLogRecord(logicalAddress); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref SpanByte GetContextRecordKey(ref AsyncIOContext ctx) - => ref SpanByteAllocatorImpl.GetContextRecordKey(ref ctx); + public readonly LogRecord CreateLogRecord(long logicalAddress, long physicalAddress) => _this.CreateLogRecord(logicalAddress, physicalAddress); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ref SpanByte GetContextRecordValue(ref AsyncIOContext ctx) => ref _this.GetContextRecordValue(ref ctx); + public readonly LogRecord CreateRemappedLogRecordOverPinnedTransientMemory(long logicalAddress, long physicalAddress) => _this.CreateLogRecord(logicalAddress, physicalAddress); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetKeyContainer(ref SpanByte key) => _this.GetKeyContainer(ref key); + public readonly ObjectIdMap TransientObjectIdMap => default; /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly IHeapContainer GetValueContainer(ref SpanByte value) => _this.GetValueContainer(ref value); + public void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) => _this.OnDispose(ref logRecord, disposeReason); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long[] GetSegmentOffsets() - => SpanByteAllocatorImpl.GetSegmentOffsets(); + public void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) => _this.OnDisposeDiskRecord(ref logRecord, disposeReason); /// - public readonly int OverflowPageCount => _this.OverflowPageCount; - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void SerializeKey(ref SpanByte key, long physicalAddress) - => SpanByteAllocatorImpl.SerializeKey(ref key, physicalAddress); + public void EvictRecordsInRange(long startAddress, long endAddress, EvictionSource source) { } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs index 3ab9540e45d..490d94699e8 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteAllocatorImpl.cs @@ -5,460 +5,282 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; -using static Tsavorite.core.Utility; namespace Tsavorite.core { - // Allocator for SpanByte, possibly with a Blittable Key or Value. - internal sealed unsafe class SpanByteAllocatorImpl : AllocatorBase> - where TStoreFunctions : IStoreFunctions + // Allocator for ReadOnlySpan Key and Span Value. + internal sealed unsafe class SpanByteAllocatorImpl : AllocatorBase> + where TStoreFunctions : IStoreFunctions { - // Circular buffer definition - private readonly byte[][] values; - private readonly long[] pointers; - private readonly long* nativePointers; - - private readonly OverflowPool overflowPagePool; + private OverflowPool> freePagePool; public SpanByteAllocatorImpl(AllocatorSettings settings, TStoreFunctions storeFunctions, Func> wrapperCreator) - : base(settings.LogSettings, storeFunctions, wrapperCreator, settings.evictCallback, settings.epoch, settings.flushCallback, settings.logger) + : base(settings, storeFunctions, wrapperCreator, settings.logger) { - overflowPagePool = new OverflowPool(4, p => { }); - - if (BufferSize > 0) - { - values = new byte[BufferSize][]; - pointers = GC.AllocateArray(BufferSize, true); - nativePointers = (long*)Unsafe.AsPointer(ref pointers[0]); - } + freePagePool = new OverflowPool>(4, p => { }); + pageHeaderSize = PageHeader.Size; } - internal int OverflowPageCount => overflowPagePool.Count; + internal int OverflowPageCount => freePagePool.Count; - public override void Reset() + /// + protected override void FreeAllAllocatedPages() { - base.Reset(); for (int index = 0; index < BufferSize; index++) { if (IsAllocated(index)) FreePage(index); } - Initialize(); + } + + /// Allocate memory page, pinned in memory, and in sector aligned form, if possible + internal void AllocatePage(int index) + { + IncrementAllocatedPageCount(); + + if (freePagePool.TryGet(out var item)) + { + pageArrays[index] = item.array; + pagePointers[index] = item.pointer; + } + else + { + // No free pages are available so allocate new + AllocatePinnedPageArray(index); + } + PageHeader.Initialize(pagePointers[index]); } void ReturnPage(int index) { Debug.Assert(index < BufferSize); - if (values[index] != null) + if (pagePointers[index] != default) { - overflowPagePool.TryAdd(new PageUnit + _ = freePagePool.TryAdd(new() { - pointer = pointers[index], - value = values[index] + array = pageArrays[index], + pointer = pagePointers[index], + value = Empty.Default }); - values[index] = null; - pointers[index] = 0; - Interlocked.Decrement(ref AllocatedPageCount); + pageArrays[index] = default; + pagePointers[index] = default; + _ = Interlocked.Decrement(ref AllocatedPageCount); } } - public override void Initialize() => Initialize(Constants.kFirstValidAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref RecordInfo GetInfo(long physicalAddress) => ref Unsafe.AsRef((void*)physicalAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref RecordInfo GetInfoFromBytePointer(byte* ptr) => ref Unsafe.AsRef(ptr); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ref SpanByte GetKey(long physicalAddress) => ref Unsafe.AsRef((byte*)physicalAddress + RecordInfo.GetLength()); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref SpanByte GetValue(long physicalAddress) => ref Unsafe.AsRef((byte*)ValueOffset(physicalAddress)); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ref SpanByte GetAndInitializeValue(long physicalAddress, long endAddress) + internal void FreePage(long page) { - var src = (byte*)ValueOffset(physicalAddress); + ClearPage(page, 0); - // Initialize the SpanByte to the length of the entire value space, less the length of the int size prefix. - *(int*)src = (int)((byte*)endAddress - src) - sizeof(int); - return ref Unsafe.AsRef(src); + // If the logSizeTracker is not active, then all pages are used once allocated so there's nothing to add to the overflow pool. + if (logSizeTracker is not null) + ReturnPage((int)(page % BufferSize)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static long KeyOffset(long physicalAddress) => physicalAddress + RecordInfo.GetLength(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private long ValueOffset(long physicalAddress) => KeyOffset(physicalAddress) + AlignedKeySize(physicalAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int AlignedKeySize(long physicalAddress) => RoundUp(KeySize(physicalAddress), Constants.kRecordAlignment); + internal LogRecord CreateLogRecord(long logicalAddress) => CreateLogRecord(logicalAddress, GetPhysicalAddress(logicalAddress)); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int KeySize(long physicalAddress) => (*(SpanByte*)KeyOffset(physicalAddress)).TotalSize; + internal LogRecord CreateLogRecord(long logicalAddress, long physicalAddress) => new(physicalAddress); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private int ValueSize(long physicalAddress) => (*(SpanByte*)ValueOffset(physicalAddress)).TotalSize; + public RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + { + // Used by RMW to determine the length of copy destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetRMWModifiedFieldInfo(in srcLogRecord, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetValueLength(ref SpanByte value) => value.TotalSize; - - const int FieldInitialLength = sizeof(int); // The .Length field of a SpanByte is the initial length + public RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + { + // Used by RMW to determine the length of initial destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetRMWInitialFieldInfo(key, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (int actualSize, int allocatedSize) GetRecordSize(long physicalAddress) + public RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput { - ref var recordInfo = ref GetInfo(physicalAddress); - if (recordInfo.IsNull()) - return (RecordInfo.GetLength(), RecordInfo.GetLength()); - - var valueLen = ValueSize(physicalAddress); - if (recordInfo.HasFiller) // Get the extraValueLength - valueLen += *(int*)(ValueOffset(physicalAddress) + RoundUp(valueLen, sizeof(int))); - - var size = RecordInfo.GetLength() + AlignedKeySize(physicalAddress) + valueLen; - return (size, RoundUp(size, Constants.kRecordAlignment)); + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, value, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (int actualSize, int allocatedSize, int keySize) GetRMWCopyDestinationRecordSize(ref SpanByte key, ref TInput input, ref SpanByte value, ref RecordInfo recordInfo, TVariableLengthInput varlenInput) - where TVariableLengthInput : IVariableLengthInput + public RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput { - // Used by RMW to determine the length of copy destination (taking Input into account), so does not need to get filler length. - var keySize = key.TotalSize; - var size = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + varlenInput.GetRMWModifiedValueLength(ref value, ref input); - return (size, RoundUp(size, Constants.kRecordAlignment), keySize); + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, in inputLogRecord, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal (int actualSize, int allocatedSize, int keySize) GetTombstoneRecordSize(ref SpanByte key) + public RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput { - int keySize = key.TotalSize; - // Only metadata space needed since this is going to be used for tombstoning anyway. - int minAllocationForTombstone = sizeof(int); - int size = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + minAllocationForTombstone; - return (size, RoundUp(size, Constants.kRecordAlignment), keySize); + // Used by Upsert to determine the length of insert destination (client uses Input to fill in whether ETag and Expiration are inluded); Filler information is not needed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = varlenInput.GetUpsertFieldInfo(key, value, ref input) }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; } - public int GetRequiredRecordSize(long physicalAddress, int availableBytes) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - // We need at least [average record size]... - var reqBytes = GetAverageRecordSize(); - if (availableBytes < reqBytes) - return reqBytes; - - // We need at least [RecordInfo size] + [actual key size]... - reqBytes = RecordInfo.GetLength() + AlignedKeySize(physicalAddress) + FieldInitialLength; - if (availableBytes < reqBytes) - return reqBytes; - - // We need at least [RecordInfo size] + [actual key size] + [actual value size] - var recordInfo = GetInfo(physicalAddress); - var valueLen = ValueSize(physicalAddress); - if (recordInfo.HasFiller) + // Used by Delete to determine the length of a new tombstone record. Does not require an ISessionFunctions method. + var sizeInfo = new RecordSizeInfo() { - // We have a filler, so the valueLen we have now is the usedValueLength; we need to offset to where the extraValueLength is and read that int - var alignedUsedValueLength = RoundUp(valueLen, sizeof(int)); - reqBytes = RecordInfo.GetLength() + AlignedKeySize(physicalAddress) + alignedUsedValueLength + sizeof(int); - if (availableBytes < reqBytes) - return reqBytes; - valueLen += *(int*)(ValueOffset(physicalAddress) + alignedUsedValueLength); - } - - // Now we know the full record length. - reqBytes = RecordInfo.GetLength() + AlignedKeySize(physicalAddress) + valueLen; - reqBytes = RoundUp(reqBytes, Constants.kRecordAlignment); - return reqBytes; + FieldInfo = new() + { + KeySize = key.KeyBytes.Length, + ValueSize = 0, // No payload for the default value + HasETag = false, + HasExpiration = false + } + }; + PopulateRecordSizeInfo(ref sizeInfo); + return sizeInfo; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetAverageRecordSize() => RecordInfo.GetLength() + (RoundUp(FieldInitialLength, Constants.kRecordAlignment) * 2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetFixedRecordSize() => GetAverageRecordSize(); - - public (int actualSize, int allocatedSize, int keySize) GetRMWInitialRecordSize(ref SpanByte key, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput + public void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) { - int keySize = key.TotalSize; - var actualSize = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + sessionFunctions.GetRMWInitialValueLength(ref input); - return (actualSize, RoundUp(actualSize, Constants.kRecordAlignment), keySize); + Debug.Assert(sizeInfo.word == 0, "RecordSizeInfo should not be reused"); + + // For SpanByteAllocator, we are always inline. + // Key + sizeInfo.SetKeyIsInline(); + var keySize = sizeInfo.FieldInfo.KeySize; + if (keySize > 1 << LogSettings.kMaxStringSizeBits) + throw new TsavoriteException($"Max inline key size is {1 << LogSettings.kMaxStringSizeBits}"); + + // Value + sizeInfo.MaxInlineValueSize = int.MaxValue; // Not currently doing out-of-line for SpanByteAllocator + sizeInfo.SetValueIsInline(); + var valueSize = sizeInfo.FieldInfo.ValueSize; + + // Record + sizeInfo.CalculateSizes(keySize, valueSize); } - public (int actualSize, int allocatedSize, int keySize) GetUpsertRecordSize(ref SpanByte key, ref SpanByte value, ref TInput input, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : IVariableLengthInput + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) { - int keySize = key.TotalSize; - var actualSize = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + sessionFunctions.GetUpsertValueLength(ref value, ref input); - return (actualSize, RoundUp(actualSize, Constants.kRecordAlignment), keySize); - } + if (logRecord.IsSet) + { + storeFunctions.OnDispose(ref logRecord, disposeReason); - public (int actualSize, int allocatedSize, int keySize) GetRecordSize(ref SpanByte key, ref SpanByte value) - { - int keySize = key.TotalSize; - var actualSize = RecordInfo.GetLength() + RoundUp(keySize, Constants.kRecordAlignment) + value.TotalSize; - return (actualSize, RoundUp(actualSize, Constants.kRecordAlignment), keySize); + logRecord.ClearOptionals(); + } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void SerializeKey(ref SpanByte src, long physicalAddress) => src.CopyTo((byte*)KeyOffset(physicalAddress)); + internal void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) { /* This allocator has no IHeapObject */ } /// /// Dispose memory allocator /// public override void Dispose() { - base.Dispose(); - overflowPagePool.Dispose(); - } - - /// - /// Allocate memory page, pinned in memory, and in sector aligned form, if possible - /// - /// - internal void AllocatePage(int index) - { - IncrementAllocatedPageCount(); - - if (overflowPagePool.TryGet(out var item)) + var localFreePagePool = Interlocked.Exchange(ref freePagePool, null); + if (localFreePagePool != null) { - pointers[index] = item.pointer; - values[index] = item.value; - return; + localFreePagePool.Dispose(); + base.Dispose(); } - - var adjustedSize = PageSize + 2 * sectorSize; - - byte[] tmp = GC.AllocateArray(adjustedSize, true); - long p = (long)Unsafe.AsPointer(ref tmp[0]); - pointers[index] = (p + (sectorSize - 1)) & ~((long)sectorSize - 1); - values[index] = tmp; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetPhysicalAddress(long logicalAddress) - { - // Offset within page - var offset = (int)(logicalAddress & ((1L << LogPageSizeBits) - 1)); - - // Index of page within the circular buffer - var pageIndex = (int)((logicalAddress >> LogPageSizeBits) & (BufferSize - 1)); - return *(nativePointers + pageIndex) + offset; } - internal bool IsAllocated(int pageIndex) => values[pageIndex] != null; - protected override void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult) - { - WriteAsync((IntPtr)pointers[flushPage % BufferSize], - (ulong)(AlignedPageSizeBytes * flushPage), - (uint)AlignedPageSizeBytes, - callback, - asyncResult, device); - } + => WriteInlinePageAsync((IntPtr)pagePointers[flushPage % BufferSize], (ulong)(AlignedPageSizeBytes * flushPage), + (uint)AlignedPageSizeBytes, callback, asyncResult, device); - protected override void WriteAsyncToDevice - (long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, - PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long[] localSegmentOffsets, long fuzzyStartLogicalAddress) + protected override void WriteAsyncToDeviceForSnapshot(long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, + PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long fuzzyStartLogicalAddress) { VerifyCompatibleSectorSize(device); - var alignedPageSize = (pageSize + (sectorSize - 1)) & ~(sectorSize - 1); - - WriteAsync((IntPtr)pointers[flushPage % BufferSize], - (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), - (uint)alignedPageSize, callback, asyncResult, - device); - } - - public long GetStartLogicalAddress(long page) => page << LogPageSizeBits; - - public long GetFirstValidLogicalAddress(long page) - { - if (page == 0) - return (page << LogPageSizeBits) + Constants.kFirstValidAddress; - return page << LogPageSizeBits; - } - - internal void ClearPage(long page, int offset) - { - if (offset == 0) - Array.Clear(values[page % BufferSize], offset, values[page % BufferSize].Length - offset); - else - { - // Adjust array offset for cache alignment - offset += (int)(pointers[page % BufferSize] - (long)Unsafe.AsPointer(ref values[page % BufferSize][0])); - Array.Clear(values[page % BufferSize], offset, values[page % BufferSize].Length - offset); - } - } - - internal void FreePage(long page) - { - ClearPage(page, 0); - if (EmptyPageCount > 0) - ReturnPage((int)(page % BufferSize)); - } - - /// - /// Delete in-memory portion of the log - /// - internal override void DeleteFromMemory() - { - for (int i = 0; i < values.Length; i++) - values[i] = null; - } - - protected override void ReadAsync( - ulong alignedSourceAddress, int destinationPageIndex, uint aligned_read_length, - DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device, IDevice objlogDevice) - { - device.ReadAsync(alignedSourceAddress, (IntPtr)pointers[destinationPageIndex], - aligned_read_length, callback, asyncResult); - } - - /// - /// Invoked by users to obtain a record from disk. It uses sector aligned memory to read - /// the record efficiently into memory. - /// - /// - /// - /// - /// - /// - protected override void AsyncReadRecordObjectsToMemory(long fromLogical, int numBytes, DeviceIOCompletionCallback callback, AsyncIOContext context, SectorAlignedMemory result = default) - { - throw new InvalidOperationException("AsyncReadRecordObjectsToMemory invalid for SpanByteAllocator"); + WriteInlinePageAsync((IntPtr)pagePointers[flushPage % BufferSize], (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), + (uint)AlignedPageSizeBytes, callback, asyncResult, device); } - internal static bool RetrievedFullRecord(byte* record, ref AsyncIOContext ctx) => true; - - internal static ref SpanByte GetContextRecordKey(ref AsyncIOContext ctx) => ref GetKey((long)ctx.record.GetValidPointer()); - - internal ref SpanByte GetContextRecordValue(ref AsyncIOContext ctx) => ref GetValue((long)ctx.record.GetValidPointer()); - - internal IHeapContainer GetKeyContainer(ref SpanByte key) => new SpanByteHeapContainer(ref key, bufferPool); - - internal IHeapContainer GetValueContainer(ref SpanByte value) => new SpanByteHeapContainer(ref value, bufferPool); - - internal static long[] GetSegmentOffsets() => null; - - internal void PopulatePage(byte* src, int required_bytes, long destinationPage) - { - throw new TsavoriteException("SpanByteAllocator memory pages are sector aligned - use direct copy"); - // Buffer.MemoryCopy(src, (void*)pointers[destinationPage % BufferSize], required_bytes, required_bytes); - } + protected override void ReadAsync(ulong alignedSourceAddress, IntPtr destinationPtr, uint aligned_read_length, + DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device) + => device.ReadAsync(alignedSourceAddress, destinationPtr, aligned_read_length, callback, asyncResult); /// /// Iterator interface for pull-scanning Tsavorite log /// - public override ITsavoriteScanIterator Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords) - => new SpanByteScanIterator(store, this, beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch, logger: logger); + public override ITsavoriteScanIterator Scan(TsavoriteKV> store, + long beginAddress, long endAddress, DiskScanBufferingMode diskScanBufferingMode, bool includeClosedRecords) + => new SpanByteScanIterator>(store, this, beginAddress, endAddress, epoch, diskScanBufferingMode, includeClosedRecords: includeClosedRecords, logger: logger); /// /// Implementation for push-scanning Tsavorite log, called from LogAccessor /// - internal override bool Scan(TsavoriteKV> store, - long beginAddress, long endAddress, ref TScanFunctions scanFunctions, ScanBufferingMode scanBufferingMode) + internal override bool Scan(TsavoriteKV> store, + long beginAddress, long endAddress, ref TScanFunctions scanFunctions, DiskScanBufferingMode diskScanBufferingMode) { - using SpanByteScanIterator iter = new(store, this, beginAddress, endAddress, scanBufferingMode, false, epoch, logger: logger); + using SpanByteScanIterator> iter = new(store, this, beginAddress, endAddress, epoch, diskScanBufferingMode, logger: logger); return PushScanImpl(beginAddress, endAddress, ref scanFunctions, iter); } /// /// Implementation for push-scanning Tsavorite log with a cursor, called from LogAccessor /// - internal override bool ScanCursor(TsavoriteKV> store, - ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) + internal override bool ScanCursor(TsavoriteKV> store, + ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, bool resetCursor = true, bool includeTombstones = false) { - using SpanByteScanIterator iter = new(store, this, cursor, endAddress, ScanBufferingMode.SinglePageBuffering, includeClosedRecords: maxAddress < long.MaxValue, epoch, logger: logger); - return ScanLookup>(store, scanCursorState, ref cursor, count, scanFunctions, iter, validateCursor, maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); + using SpanByteScanIterator> iter = new(store, this, cursor, endAddress, epoch, DiskScanBufferingMode.SinglePageBuffering, + includeClosedRecords: maxAddress < long.MaxValue, logger: logger); + return ScanLookup>>(store, scanCursorState, + ref cursor, count, scanFunctions, iter, validateCursor, maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); } /// /// Implementation for push-iterating key versions, called from LogAccessor /// - internal override bool IterateKeyVersions(TsavoriteKV> store, - ref SpanByte key, long beginAddress, ref TScanFunctions scanFunctions) + internal override bool IterateKeyVersions(TsavoriteKV> store, + TKey key, long beginAddress, ref TScanFunctions scanFunctions) { - using SpanByteScanIterator iter = new(store, this, beginAddress, epoch, logger: logger); - return IterateKeyVersionsImpl(store, ref key, beginAddress, ref scanFunctions, iter); + using SpanByteScanIterator> iter = new(store, this, beginAddress, epoch, logger: logger); + return IterateHashChain(store, key, beginAddress, ref scanFunctions, iter); } /// - internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver> observer) + internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver observer) { - using var iter = new SpanByteScanIterator(store: null, this, beginAddress, endAddress, ScanBufferingMode.NoBuffering, false, epoch, true, logger: logger); + using var iter = new SpanByteScanIterator>(store: null, this, beginAddress, endAddress, epoch, DiskScanBufferingMode.NoBuffering, InMemoryScanBufferingMode.NoBuffering, + includeClosedRecords: false, assumeInMemory: true, logger: logger); observer?.OnNext(iter); } - - /// - /// Read pages from specified device - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - internal void AsyncReadPagesFromDeviceToFrame( - long readPageStart, - int numPages, - long untilAddress, - DeviceIOCompletionCallback callback, - TContext context, - BlittableFrame frame, - out CountdownEvent completed, - long devicePageOffset = 0, - IDevice device = null, IDevice objectLogDevice = null) - { - var usedDevice = device; - if (device == null) - { - usedDevice = this.device; - } - - completed = new CountdownEvent(numPages); - for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++) - { - int pageIndex = (int)(readPage % frame.frameSize); - if (frame.frame[pageIndex] == null) - { - frame.Allocate(pageIndex); - } - else - { - frame.Clear(pageIndex); - } - var asyncResult = new PageAsyncReadResult() - { - page = readPage, - context = context, - handle = completed, - frame = frame - }; - - ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); - - uint readLength = (uint)AlignedPageSizeBytes; - long adjustedUntilAddress = (AlignedPageSizeBytes * (untilAddress >> LogPageSizeBits) + (untilAddress & PageSizeMask)); - - if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) - { - readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); - readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); - } - - if (device != null) - offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); - - usedDevice.ReadAsync(offsetInFile, (IntPtr)frame.pointers[pageIndex], readLength, callback, asyncResult); - } - } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs index ed7a1ce5dfd..c0eafac6d5d 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/SpanByteScanIterator.cs @@ -12,104 +12,138 @@ namespace Tsavorite.core /// /// Scan iterator for hybrid log /// - public sealed class SpanByteScanIterator : ScanIteratorBase, ITsavoriteScanIterator, IPushScanIterator - where TStoreFunctions : IStoreFunctions + public sealed unsafe class SpanByteScanIterator : ScanIteratorBase, ITsavoriteScanIterator, IPushScanIterator + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - private readonly TsavoriteKV> store; - private readonly SpanByteAllocatorImpl hlog; + private readonly TsavoriteKV store; + private readonly AllocatorBase hlogBase; private readonly BlittableFrame frame; - private SectorAlignedMemory memory; - private readonly bool forceInMemory; + private SectorAlignedMemory recordBuffer; + private readonly bool assumeInMemory; - private long currentPhysicalAddress; + private DiskLogRecord diskLogRecord; /// /// Constructor /// /// - /// The fully derived log implementation + /// The fully derived log implementation /// /// - /// + /// + /// /// - /// Epoch to use for protection; may be null if is true. - /// Provided address range is known by caller to be in memory, even if less than HeadAddress + /// Epoch to use for protection; may be null if is true. + /// Provided address range is known by caller to be in memory, even if less than HeadAddress /// - internal SpanByteScanIterator(TsavoriteKV> store, SpanByteAllocatorImpl hlog, - long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode, bool includeClosedRecords, LightEpoch epoch, bool forceInMemory = false, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, endAddress, scanBufferingMode, includeClosedRecords, epoch, hlog.LogPageSizeBits, logger: logger) + internal SpanByteScanIterator(TsavoriteKV store, AllocatorBase hlogBase, + long beginAddress, long endAddress, LightEpoch epoch, + DiskScanBufferingMode diskScanBufferingMode, InMemoryScanBufferingMode memScanBufferingMode = InMemoryScanBufferingMode.NoBuffering, + bool includeClosedRecords = false, bool assumeInMemory = false, ILogger logger = null) + : base(beginAddress == 0 ? hlogBase.GetFirstValidLogicalAddressOnPage(0) : beginAddress, endAddress, + diskScanBufferingMode, memScanBufferingMode, includeClosedRecords, epoch, hlogBase.LogPageSizeBits, logger: logger) { this.store = store; - this.hlog = hlog; - this.forceInMemory = forceInMemory; + this.hlogBase = hlogBase; + this.assumeInMemory = assumeInMemory; if (frameSize > 0) - frame = new BlittableFrame(frameSize, hlog.PageSize, hlog.GetDeviceSectorSize()); + frame = new BlittableFrame(frameSize, hlogBase.PageSize, hlogBase.GetDeviceSectorSize()); } /// /// Constructor for use with tail-to-head push iteration of the passed key's record versions /// - internal SpanByteScanIterator(TsavoriteKV> store, SpanByteAllocatorImpl hlog, + internal SpanByteScanIterator(TsavoriteKV store, AllocatorBase hlogBase, long beginAddress, LightEpoch epoch, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, hlog.GetTailAddress(), ScanBufferingMode.SinglePageBuffering, false, epoch, hlog.LogPageSizeBits, logger: logger) + : base(beginAddress == 0 ? hlogBase.GetFirstValidLogicalAddressOnPage(0) : beginAddress, hlogBase.GetTailAddress(), + DiskScanBufferingMode.SinglePageBuffering, InMemoryScanBufferingMode.NoBuffering, false, epoch, hlogBase.LogPageSizeBits, logger: logger) { this.store = store; - this.hlog = hlog; - forceInMemory = false; + this.hlogBase = hlogBase; + assumeInMemory = false; if (frameSize > 0) - frame = new BlittableFrame(frameSize, hlog.PageSize, hlog.GetDeviceSectorSize()); + frame = new BlittableFrame(frameSize, hlogBase.PageSize, hlogBase.GetDeviceSectorSize()); } - /// - /// Gets reference to current key - /// - public ref SpanByte GetKey() => ref hlog._wrapper.GetKey(currentPhysicalAddress); - - /// - /// Gets reference to current value - /// - public ref SpanByte GetValue() => ref hlog.GetValue(currentPhysicalAddress); - + #region TODO Unify with ObjectScanIterator /// public bool SnapCursorToLogicalAddress(ref long cursor) { Debug.Assert(currentAddress == -1, "SnapCursorToLogicalAddress must be called before GetNext()"); Debug.Assert(nextAddress == cursor, "SnapCursorToLogicalAddress should have nextAddress == cursor"); - if (!InitializeGetNext(out long headAddress, out long currentPage)) + if (!InitializeGetNextAndAcquireEpoch(out var stopAddress)) return false; - epoch?.Suspend(); - - beginAddress = nextAddress = SnapToLogicalAddressBoundary(ref cursor, headAddress, currentPage); + try + { + if (!LoadPageIfNeeded(out var headAddress, out var currentPage, stopAddress)) + return false; + beginAddress = nextAddress = SnapToLogicalAddressBoundary(ref cursor, headAddress, currentPage); + } + finally + { + epoch?.Suspend(); + } return true; } - private bool InitializeGetNext(out long headAddress, out long currentPage) + private bool InitializeGetNextAndAcquireEpoch(out long stopAddress) { + if (diskLogRecord.IsSet) + { + hlogBase._wrapper.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + diskLogRecord.Dispose(); + } + diskLogRecord = default; currentAddress = nextAddress; - var stopAddress = endAddress < hlog.GetTailAddress() ? endAddress : hlog.GetTailAddress(); + + // Acquire the epoch BEFORE sampling Initializing / TailAddress / HeadAddress / + // pagePointers, so that any allocator state we read is consistent with the epoch + // we hold. + epoch?.Resume(); + + // If a concurrent Reset is rebuilding the allocator, terminate the iteration + // cleanly — Reset is a wholesale wipe, the records we were iterating are gone, + // and the address range we were stepping through no longer maps to live data. + // Also avoids dereferencing the non-monotonic mid-Initialize state (HeadAddress + // already rewound to FirstValidAddress while TailPageOffset still holds the + // pre-Reset tail and pagePointers[i] are mostly 0). + if (hlogBase.Initializing) + { + epoch?.Suspend(); + stopAddress = 0; + return false; + } + + stopAddress = endAddress < hlogBase.GetTailAddress() ? endAddress : hlogBase.GetTailAddress(); if (currentAddress >= stopAddress) { - headAddress = currentPage = 0; + epoch?.Suspend(); return false; } - epoch?.Resume(); - headAddress = hlog.HeadAddress; + // Success; caller will suspend the epoch as needed. + return true; + } - if (currentAddress < hlog.BeginAddress && !forceInMemory) - currentAddress = hlog.BeginAddress; + private bool LoadPageIfNeeded(out long headAddress, out long currentPage, long stopAddress) + { + headAddress = hlogBase.HeadAddress; + + if (currentAddress < hlogBase.BeginAddress && !assumeInMemory) + currentAddress = hlogBase.BeginAddress; // If currentAddress < headAddress and we're not buffering and not guaranteeing the records are in memory, fail. - if (frameSize == 0 && currentAddress < headAddress && !forceInMemory) + if (frameSize == 0 && currentAddress < headAddress && !assumeInMemory) { - epoch?.Suspend(); + // Caller will suspend the epoch. throw new TsavoriteException("Iterator address is less than log HeadAddress in memory-scan mode"); } - currentPage = currentAddress >> hlog.LogPageSizeBits; - if (currentAddress < headAddress && !forceInMemory) + currentPage = hlogBase.GetPage(currentAddress); + if (currentAddress < headAddress && !assumeInMemory) _ = BufferAndLoad(currentAddress, currentPage, currentPage % frameSize, headAddress, stopAddress); // Success; keep the epoch held for GetNext (SnapCursorToLogicalAddress will Suspend()). @@ -119,20 +153,21 @@ private bool InitializeGetNext(out long headAddress, out long currentPage) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal long SnapToLogicalAddressBoundary(ref long logicalAddress, long headAddress, long currentPage) { - long offset = logicalAddress & hlog.PageSizeMask; - long physicalAddress = GetPhysicalAddress(logicalAddress, headAddress, currentPage, offset) - offset; - long totalSizes = 0; + var offset = hlogBase.GetOffsetOnPage(logicalAddress); + + // Subtracting offset means this physicalAddress is at the start of the page. Adjust for PageHeader. + long totalSizes = PageHeader.Size; if (currentPage == 0) { - if (logicalAddress < hlog.BeginAddress) - return logicalAddress = hlog.BeginAddress; - physicalAddress += hlog.BeginAddress; - totalSizes = (int)hlog.BeginAddress; + if (logicalAddress < hlogBase.BeginAddress) + return logicalAddress = hlogBase.BeginAddress; + totalSizes = (int)hlogBase.BeginAddress; } + var physicalAddress = GetPhysicalAddress(logicalAddress, headAddress, currentPage, offset) - offset + totalSizes; while (totalSizes <= offset) { - var (_, allocatedSize) = hlog.GetRecordSize(physicalAddress); + var allocatedSize = new LogRecord(physicalAddress).AllocatedSize; if (totalSizes + allocatedSize > offset) break; totalSizes += allocatedSize; @@ -142,84 +177,110 @@ internal long SnapToLogicalAddressBoundary(ref long logicalAddress, long headAdd return logicalAddress += totalSizes - offset; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + long GetPhysicalAddress(long currentAddress, long headAddress, long currentPage, long offset) + => currentAddress >= headAddress || assumeInMemory + ? hlogBase.GetPhysicalAddress(currentAddress) + : frame.GetPhysicalAddress(currentPage, offset); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + long GetPhysicalAddressAndAllocatedSize(long currentAddress, long headAddress, long currentPage, long offset, out long allocatedSize) + { + var physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); + + // We are just getting sizes so no need for ObjectIdMap + var logRecord = new LogRecord(physicalAddress); + allocatedSize = logRecord.AllocatedSize; + return logRecord.physicalAddress; + } + #endregion TODO Unify with ObjectScanIterator + /// /// Get next record in iterator /// /// True if record found, false if end of scan - public unsafe bool GetNext(out RecordInfo recordInfo) + public bool GetNext() { - recordInfo = default; - while (true) { - if (!InitializeGetNext(out long headAddress, out long currentPage)) + if (!InitializeGetNextAndAcquireEpoch(out var stopAddress)) return false; - var offset = currentAddress & hlog.PageSizeMask; - long physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); - int recordSize = hlog.GetRecordSize(physicalAddress).Item2; - - // If record does not fit on page, skip to the next page. - if ((currentAddress & hlog.PageSizeMask) + recordSize > hlog.PageSize) + try { - nextAddress = (1 + (currentAddress >> hlog.LogPageSizeBits)) << hlog.LogPageSizeBits; - epoch?.Suspend(); - continue; - } + if (!LoadPageIfNeeded(out var headAddress, out var currentPage, stopAddress)) + return false; - nextAddress = currentAddress + recordSize; + var offset = hlogBase.GetOffsetOnPage(currentAddress); + var physicalAddress = GetPhysicalAddressAndAllocatedSize(currentAddress, headAddress, currentPage, offset, out var allocatedSize); + var recordInfo = LogRecord.GetInfo(physicalAddress); - recordInfo = hlog._wrapper.GetInfo(physicalAddress); + // If record does not fit on page, skip to the next page. Offset should always be at least PageHeader.Size; if it's zero, it means + // our record size aligned perfectly with end of page, so we must move to the next page (skipping its PageHeader). + if (offset == 0 || offset + allocatedSize > hlogBase.PageSize) + { + var nextPage = hlogBase.GetPage(currentAddress); + nextAddress = hlogBase.GetFirstValidLogicalAddressOnPage(offset == 0 ? nextPage : nextPage + 1); + continue; + } - var skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (skipOnScan || recordInfo.IsNull()) - { - epoch?.Suspend(); - continue; - } + nextAddress = currentAddress + allocatedSize; - currentPhysicalAddress = physicalAddress; + var skipOnScan = !includeClosedRecords && recordInfo.SkipOnScan; + if (skipOnScan || recordInfo.IsNull) + continue; - // We will return control to the caller, which means releasing epoch protection, and we don't want the caller to lock. - // Copy the entire record into bufferPool memory, so we do not have a ref to log data outside epoch protection. - // Lock to ensure no value tearing while copying to temp storage. - if (currentAddress >= headAddress || forceInMemory) - { - OperationStackContext> stackCtx = default; - try + if (currentAddress >= headAddress || assumeInMemory) { - if (memory == null) - { - memory = hlog.bufferPool.Get(recordSize); - } - else + // TODO: for this PR we always buffer the in-memory records; pull iterators require it, and currently push iterators are implemented on top of pull. + // Copy the entire record into bufferPool memory so we don't have a ref to log data outside epoch protection. + OperationStackContext stackCtx = default; + try { - if (memory.AlignedTotalCapacity < recordSize) + // Lock to ensure no value tearing while copying to temp storage. + if (currentAddress >= headAddress && store is not null) { - memory.Return(); - memory = hlog.bufferPool.Get(recordSize); + var logRecord = hlogBase._wrapper.CreateLogRecord(currentAddress, physicalAddress); + store.LockForScan(ref stackCtx, logRecord); } - } - // GetKey() should work but for safety and consistency with other allocators use physicalAddress. - if (currentAddress >= headAddress && store is not null) - store.LockForScan(ref stackCtx, ref hlog._wrapper.GetKey(physicalAddress)); + if (recordBuffer == null) + recordBuffer = hlogBase.bufferPool.Get((int)allocatedSize); + else if (recordBuffer.AlignedTotalCapacity < (int)allocatedSize) + { + recordBuffer.Return(); + recordBuffer = hlogBase.bufferPool.Get((int)allocatedSize); + } - unsafe + // These objects are still alive in the log, so do not dispose the value object if any (SpanByteAllocator has none). + // Don't pass the recordBuffer to diskLogRecord; we reuse that here. + var remapPtr = recordBuffer.GetValidPointer(); + Buffer.MemoryCopy((byte*)physicalAddress, remapPtr, allocatedSize, allocatedSize); + var memoryLogRecord = hlogBase._wrapper.CreateRemappedLogRecordOverPinnedTransientMemory(currentAddress, (long)remapPtr); + diskLogRecord = new DiskLogRecord(in memoryLogRecord); + } + finally { - Buffer.MemoryCopy((byte*)currentPhysicalAddress, memory.aligned_pointer, recordSize, recordSize); - currentPhysicalAddress = (long)memory.aligned_pointer; + if (stackCtx.recSrc.HasLock) + store.UnlockForScan(ref stackCtx); } } - finally + else { - if (stackCtx.recSrc.HasLock) - store.UnlockForScan(ref stackCtx); + // We advance a record at a time in the IO frame so set the diskLogRecord to the current frame offset and advance nextAddress. + // SpanByteAllocator has no objects, so no value-object disposal is required. + diskLogRecord = new(new LogRecord(physicalAddress, hlogBase._wrapper.TransientObjectIdMap)); + // Fire OnDiskRead so app can invalidate stale TreeHandles, etc., on records loaded from disk. + if (hlogBase.storeFunctions.CallOnDiskRead) + hlogBase.storeFunctions.OnDiskRead(ref diskLogRecord.logRecord); } } + finally + { + // Success + epoch?.Suspend(); + } - // Success - epoch?.Suspend(); return true; } } @@ -228,67 +289,145 @@ public unsafe bool GetNext(out RecordInfo recordInfo) /// Get previous record and keep the epoch held while we call the user's scan functions /// /// True if record found, false if end of scan - bool IPushScanIterator.BeginGetPrevInMemory(ref SpanByte key, out RecordInfo recordInfo, out bool continueOnDisk) + bool IPushScanIterator.BeginGetPrevInMemory(TKey key, out LogRecord logRecord, out bool continueOnDisk) { - recordInfo = default; - continueOnDisk = false; - while (true) { // "nextAddress" is reused as "previous address" for this operation. currentAddress = nextAddress; - if (currentAddress < hlog.HeadAddress) + var headAddress = hlogBase.HeadAddress; + if (currentAddress < headAddress) { - continueOnDisk = currentAddress >= hlog.BeginAddress; + logRecord = default; + continueOnDisk = currentAddress >= hlogBase.BeginAddress; return false; } epoch?.Resume(); - var headAddress = hlog.HeadAddress; - - var currentPage = currentAddress >> hlog.LogPageSizeBits; - var offset = currentAddress & hlog.PageSizeMask; - long physicalAddress = GetPhysicalAddress(currentAddress, headAddress, currentPage, offset); + logRecord = hlogBase._wrapper.CreateLogRecord(currentAddress); + nextAddress = logRecord.Info.PreviousAddress; - recordInfo = hlog._wrapper.GetInfo(physicalAddress); - nextAddress = recordInfo.PreviousAddress; - bool skipOnScan = includeClosedRecords ? false : recordInfo.SkipOnScan; - if (skipOnScan || recordInfo.IsNull() || !hlog._storeFunctions.KeysEqual(ref hlog._wrapper.GetKey(physicalAddress), ref key)) + // Do not SkipOnScan here; we Seal previous versions. + if (logRecord.Info.IsNull || !hlogBase.storeFunctions.KeysEqual(logRecord, key)) { epoch?.Suspend(); continue; } // Success; defer epoch?.Suspend(); to EndGet - currentPhysicalAddress = physicalAddress; + continueOnDisk = false; return true; } } - bool IPushScanIterator.EndGetPrevInMemory() + void IPushScanIterator.EndGetPrevInMemory() => epoch?.Suspend(); + + #region ISourceLogRecord + /// + public ref RecordInfo InfoRef => ref diskLogRecord.InfoRef; + /// + public RecordInfo Info => diskLogRecord.Info; + + /// + public byte RecordType => diskLogRecord.RecordType; + + /// + public ReadOnlySpan Namespace => diskLogRecord.Namespace; + + /// + public ObjectIdMap ObjectIdMap => diskLogRecord.ObjectIdMap; + + /// + public bool IsSet => diskLogRecord.IsSet; + + /// + public ReadOnlySpan Key => diskLogRecord.Key; + + /// + public bool IsPinnedKey => diskLogRecord.IsPinnedKey; + + /// + public byte* PinnedKeyPointer => diskLogRecord.PinnedKeyPointer; + + /// + public OverflowByteArray KeyOverflow { - epoch?.Suspend(); - return true; + get => diskLogRecord.KeyOverflow; + set => diskLogRecord.KeyOverflow = value; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - long GetPhysicalAddress(long currentAddress, long headAddress, long currentPage, long offset) + /// + public Span ValueSpan => diskLogRecord.ValueSpan; + + /// + public IHeapObject ValueObject => diskLogRecord.ValueObject; + + /// + public bool IsPinnedValue => diskLogRecord.IsPinnedValue; + + /// + public byte* PinnedValuePointer => diskLogRecord.PinnedValuePointer; + + /// + public OverflowByteArray ValueOverflow { - long physicalAddress; - if (currentAddress >= headAddress || forceInMemory) - physicalAddress = hlog.GetPhysicalAddress(currentAddress); - else - physicalAddress = frame.GetPhysicalAddress(currentPage % frameSize, offset); - return physicalAddress; + get => diskLogRecord.ValueOverflow; + set => diskLogRecord.ValueOverflow = value; } - /// - /// Get next record in iterator - /// - /// - public bool GetNext(out RecordInfo recordInfo, out SpanByte key, out SpanByte value) - => throw new NotSupportedException("Use GetNext(out RecordInfo) to retrieve references to key/value"); + /// + public SpanByteAndMemory ValueSpanByteAndMemory => diskLogRecord.ValueSpanByteAndMemory; + + /// + public long ETag => diskLogRecord.ETag; + + /// + public long Expiration => diskLogRecord.Expiration; + + /// + public void ClearValueIfHeap() { } // Not relevant for "iterator as logrecord" + + /// + public bool IsMemoryLogRecord => false; + + /// + public unsafe ref LogRecord AsMemoryLogRecordRef() => throw new InvalidOperationException("Cannot cast a DiskLogRecord to a memory LogRecord."); + + /// + public bool IsDiskLogRecord => true; + + /// + public unsafe ref DiskLogRecord AsDiskLogRecordRef() => ref Unsafe.AsRef(in diskLogRecord); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordFieldInfo GetRecordFieldInfo() => diskLogRecord.GetRecordFieldInfo(); + + /// + public int AllocatedSize => diskLogRecord.AllocatedSize; + + /// + public int ActualSize => diskLogRecord.ActualSize; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long CalculateHeapMemorySize() => diskLogRecord.CalculateHeapMemorySize(); + #endregion // ISourceLogRecord + + #region IKey + /// + public bool IsPinned => false; + + /// + public ReadOnlySpan KeyBytes => Key; + + /// + public bool HasNamespace => diskLogRecord.HasNamespace; + + /// + public ReadOnlySpan NamespaceBytes => diskLogRecord.NamespaceBytes; + #endregion /// /// Dispose iterator @@ -296,36 +435,15 @@ public bool GetNext(out RecordInfo recordInfo, out SpanByte key, out SpanByte va public override void Dispose() { base.Dispose(); - memory?.Return(); - memory = null; + if (diskLogRecord.IsSet) + hlogBase._wrapper.OnDisposeDiskRecord(ref diskLogRecord, DisposeReason.DeserializedFromDisk); + recordBuffer?.Return(); + recordBuffer = null; frame?.Dispose(); } - internal override void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, + internal override void AsyncReadPageFromDeviceToFrame(CircularDiskReadBuffer readBuffers, long readPage, long untilAddress, TContext context, out CountdownEvent completed, long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) - => hlog.AsyncReadPagesFromDeviceToFrame(readPageStart, numPages, untilAddress, AsyncReadPagesCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice); - - private unsafe void AsyncReadPagesCallback(uint errorCode, uint numBytes, object context) - { - var result = (PageAsyncReadResult)context; - - if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncReadPagesCallback)} error: {{errorCode}}", errorCode); - result.cts?.Cancel(); - } - - if (result.freeBuffer1 != null) - { - hlog.PopulatePage(result.freeBuffer1.GetValidPointer(), result.freeBuffer1.required_bytes, result.page); - result.freeBuffer1.Return(); - result.freeBuffer1 = null; - } - - if (errorCode == 0) - result.handle?.Signal(); - - Interlocked.MemoryBarrier(); - } + => hlogBase.AsyncReadPageFromDeviceToFrame(readBuffers, readPage, untilAddress, AsyncReadPageFromDeviceToFrameCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice, cts); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocator.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocator.cs new file mode 100644 index 00000000000..e69b4ba783c --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocator.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + // This is unused; just allows things to build. TsavoriteLog does not do key comparisons or value operations; it is just a memory allocator +#pragma warning disable IDE0065 // Misplaced using directive + using TsavoriteLogStoreFunctions = StoreFunctions; + + /// + /// Struct wrapper (for inlining) around the TsavoriteLogAllocator used by TsavoriteLog. + /// + public struct TsavoriteLogAllocator : IAllocator + { + /// The wrapped class containing all data and most actual functionality. This must be the ONLY field in this structure so its size is sizeof(IntPtr). + private readonly TsavoriteLogAllocatorImpl _this; + + public TsavoriteLogAllocator(object @this) + { + // Called by AllocatorBase via primary ctor wrapperCreator + _this = (TsavoriteLogAllocatorImpl)@this; + } + + /// + public readonly AllocatorBase GetBase() + where TAllocator : IAllocator + => (AllocatorBase)(object)_this; + + /// + public readonly bool HasObjectLog => false; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void InitializeRecord(TKey key, long logicalAddress, in RecordSizeInfo _, ref LogRecord newLogRecord) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetRMWCopyRecordSize(in TSourceLogRecord srcLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetRMWInitialRecordSize(TKey key, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, ReadOnlySpan value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, IHeapObject value, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TVariableLengthInput : IVariableLengthInput + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetUpsertRecordSize(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordSizeInfo GetDeleteRecordSize(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void AllocatePage(int pageIndex) => _this.AllocatePage(pageIndex); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void FreePage(long pageIndex) => _this.FreePage(pageIndex); + + /// + public readonly int OverflowPageCount => _this.OverflowPageCount; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly LogRecord CreateLogRecord(long logicalAddress) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly LogRecord CreateLogRecord(long logicalAddress, long physicalAddress) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + public readonly LogRecord CreateRemappedLogRecordOverPinnedTransientMemory(long logicalAddress, long physicalAddress) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + public readonly ObjectIdMap TransientObjectIdMap => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) => throw new NotImplementedException("Not implemented for TsavoriteLogAllocator"); + + /// + public void EvictRecordsInRange(long startAddress, long endAddress, EvictionSource source) { } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocatorImpl.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocatorImpl.cs new file mode 100644 index 00000000000..5b40281e563 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/TsavoriteLogAllocatorImpl.cs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Threading; + +namespace Tsavorite.core +{ + // This is unused; just allows things to build. TsavoriteLog does not do key comparisons or value operations; it is just a memory allocator. + using TsavoriteLogStoreFunctions = StoreFunctions; + + /// Simple log allocator used by TsavoriteLog + public sealed unsafe class TsavoriteLogAllocatorImpl : AllocatorBase + { + private readonly OverflowPool> freePagePool; + + /// Constructor +#pragma warning disable IDE0290 // Use primary constructor + public TsavoriteLogAllocatorImpl(AllocatorSettings settings) + : base(settings, new TsavoriteLogStoreFunctions(), @this => new TsavoriteLogAllocator(@this), settings.logger) + { + freePagePool = new OverflowPool>(4, p => { }); + } + + /// + protected override void FreeAllAllocatedPages() + { + for (var index = 0; index < BufferSize; index++) + { + if (IsAllocated(index)) + FreePage(index); + } + } + + /// + /// Allocate memory page, pinned in memory + /// + /// + internal void AllocatePage(int index) + { + IncrementAllocatedPageCount(); + + if (freePagePool.TryGet(out var item)) + { + pageArrays[index] = item.array; + pagePointers[index] = item.pointer; + } + else + { + // No free pages are available so allocate new + AllocatePinnedPageArray(index); + } + PageHeader.Initialize(pagePointers[index]); + } + + void ReturnPage(int index) + { + Debug.Assert(index < BufferSize); + if (pagePointers[index] != default) + { + _ = freePagePool.TryAdd(new() + { + array = pageArrays[index], + pointer = pagePointers[index], + value = Empty.Default + }); + pageArrays[index] = default; + pagePointers[index] = default; + _ = Interlocked.Decrement(ref AllocatedPageCount); + } + } + + internal void FreePage(long page) + { + ClearPage(page, 0); + + // If the logSizeTracker is not active, then all pages are used once allocated so there's nothing to add to the overflow pool. + if (logSizeTracker is not null) + ReturnPage((int)(page % BufferSize)); + } + + /// + /// Dispose memory allocator + /// + public override void Dispose() + { + base.Dispose(); + freePagePool.Dispose(); + } + + internal int OverflowPageCount => freePagePool.Count; + + /// + protected override void WriteAsync(long flushPage, DeviceIOCompletionCallback callback, PageAsyncFlushResult asyncResult) + { + WriteInlinePageAsync((IntPtr)pagePointers[flushPage % BufferSize], + (ulong)(AlignedPageSizeBytes * flushPage), + (uint)AlignedPageSizeBytes, + callback, + asyncResult, device); + } + + /// + protected override void WriteAsyncToDeviceForSnapshot(long startPage, long flushPage, int pageSize, DeviceIOCompletionCallback callback, + PageAsyncFlushResult asyncResult, IDevice device, IDevice objectLogDevice, long fuzzyStartLogicalAddress) + { + VerifyCompatibleSectorSize(device); + var alignedPageSize = (pageSize + (sectorSize - 1)) & ~(sectorSize - 1); + + WriteInlinePageAsync((IntPtr)pagePointers[flushPage % BufferSize], + (ulong)(AlignedPageSizeBytes * (flushPage - startPage)), + (uint)alignedPageSize, callback, asyncResult, + device); + } + + protected override void ReadAsync(ulong alignedSourceAddress, IntPtr destinationPtr, uint aligned_read_length, + DeviceIOCompletionCallback callback, PageAsyncReadResult asyncResult, IDevice device) + => device.ReadAsync(alignedSourceAddress, destinationPtr, aligned_read_length, callback, asyncResult); + + private protected override bool VerifyRecordFromDiskCallback(ref AsyncIOContext ctx, out long prevAddressToRead, out int prevLengthToRead) + => throw new TsavoriteException("TsavoriteLogAllocator does not support VerifyRecordFromDiskCallback"); + + /// + /// Iterator interface for pull-scanning Tsavorite log + /// + public override ITsavoriteScanIterator Scan(TsavoriteKV store, + long beginAddress, long endAddress, DiskScanBufferingMode diskScanBufferingMode, bool includeSealedRecords) + => throw new TsavoriteException("TsavoriteLogAllocator Scan methods should not be used"); + + /// + /// Implementation for push-scanning Tsavorite log, called from LogAccessor + /// + internal override bool Scan(TsavoriteKV store, + long beginAddress, long endAddress, ref TScanFunctions scanFunctions, DiskScanBufferingMode diskScanBufferingMode) + => throw new TsavoriteException("TsavoriteLogAllocator Scan methods should not be used"); + + /// + /// Implementation for push-scanning Tsavorite log with a cursor, called from LogAccessor + /// + internal override bool ScanCursor(TsavoriteKV store, + ScanCursorState scanCursorState, ref long cursor, long count, TScanFunctions scanFunctions, long endAddress, bool validateCursor, long maxAddress, + bool resetCursor = true, bool includeTombstones = false) + => throw new TsavoriteException("TsavoriteLogAllocator Scan methods should not be used"); + + /// + /// Implementation for push-iterating key versions, called from LogAccessor + /// + internal override bool IterateKeyVersions(TsavoriteKV store, TKey key, + long beginAddress, ref TScanFunctions scanFunctions) + => throw new TsavoriteException("TsavoriteLogAllocator Scan methods should not be used"); + + /// + internal override void MemoryPageScan(long beginAddress, long endAddress, IObserver observer) + => throw new TsavoriteException("TsavoriteLogAllocator Scan methods should not be used"); + + /// + /// Read pages from specified device + /// + internal void AsyncReadPageFromDeviceToFrame( + long readPage, + long untilAddress, + DeviceIOCompletionCallback callback, + TContext context, + BlittableFrame frame, + out CountdownEvent completed, + long devicePageOffset = 0, + IDevice device = null, + IDevice objectLogDevice = null, + CancellationTokenSource cts = null) + { + var usedDevice = device ?? this.device; + + completed = new CountdownEvent(1); + + int pageIndex = (int)(readPage % frame.frameSize); + if (frame.frame[pageIndex] == null) + frame.Allocate(pageIndex); + else + frame.Clear(pageIndex); + + var asyncResult = new PageAsyncReadResult() + { + page = readPage, + context = context, + handle = completed, + cts = cts + }; + + ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); + + uint readLength = (uint)AlignedPageSizeBytes; + long adjustedUntilAddress = AlignedPageSizeBytes * GetPage(untilAddress) + GetOffsetOnPage(untilAddress); + + if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) + { + readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); + readLength = (uint)((readLength + (sectorSize - 1)) & ~(sectorSize - 1)); + } + + if (device != null) + offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); + + usedDevice.ReadAsync(offsetInFile, (IntPtr)frame.pointers[pageIndex], readLength, callback, asyncResult); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Async/CompletePendingAsync.cs b/libs/storage/Tsavorite/cs/src/core/Async/CompletePendingAsync.cs index 167d5733b26..098ab349d86 100644 --- a/libs/storage/Tsavorite/cs/src/core/Async/CompletePendingAsync.cs +++ b/libs/storage/Tsavorite/cs/src/core/Async/CompletePendingAsync.cs @@ -9,9 +9,9 @@ namespace Tsavorite.core /// /// The Tsavorite key-value store /// - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Check if at least one (sync) request is ready for CompletePending to operate on @@ -28,8 +28,8 @@ internal static ValueTask ReadyToCompletePendingAsync /// /// internal async ValueTask CompletePendingAsync(TSessionFunctionsWrapper sessionFunctions, - CancellationToken token, CompletedOutputIterator completedOutputs) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + CancellationToken token, CompletedOutputIterator completedOutputs) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { while (true) { @@ -44,8 +44,8 @@ internal async ValueTask CompletePendingAsync /// Basic Tsavorite Context implementation. /// - public readonly struct BasicContext - : ITsavoriteContext - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public readonly struct BasicContext + : ITsavoriteContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - readonly ClientSession clientSession; - internal readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; + readonly ClientSession clientSession; + internal readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; /// public bool IsNull => clientSession is null; - private TsavoriteKV store => clientSession.store; + private TsavoriteKV store => clientSession.store; - internal BasicContext(ClientSession clientSession) + internal BasicContext(ClientSession clientSession) { this.clientSession = clientSession; sessionFunctions = new(clientSession); @@ -43,20 +48,22 @@ public void UnsafeSuspendThread() #region ITsavoriteContext /// - public ClientSession Session => clientSession; + public ClientSession Session => clientSession; /// - public long GetKeyHash(TKey key) => clientSession.store.GetKeyHash(ref key); - - /// - public long GetKeyHash(ref TKey key) => clientSession.store.GetKeyHash(ref key); + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => clientSession.store.GetKeyHash(key); /// public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) => clientSession.CompletePending(sessionFunctions, wait, spinWaitForCommit); /// - public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) => clientSession.CompletePendingWithOutputs(sessionFunctions, out completedOutputs, wait, spinWaitForCommit); /// @@ -64,17 +71,17 @@ public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationTo => clientSession.CompletePendingAsync(sessionFunctions, waitForCommit, token); /// - public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) => clientSession.CompletePendingWithOutputsAsync(sessionFunctions, waitForCommit, token); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) { UnsafeResumeThread(); try { - return clientSession.store.ContextRead(ref key, ref input, ref output, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, userContext, sessionFunctions); } finally { @@ -93,7 +100,7 @@ public void ReadWithPrefetch(ref TBatch batch, TContext userContext = de UnsafeResumeThread(); try { - clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); + clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); } finally { @@ -103,57 +110,23 @@ public void ReadWithPrefetch(ref TBatch batch, TContext userContext = de /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - => Read(ref key, ref input, ref output, ref readOptions, out _, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + => Read(key, ref input, ref output, ref readOptions, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, TContext userContext = default) { TInput input = default; - output = default; - return Read(ref key, ref input, ref output, userContext); + return Read(key, ref input, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) { TInput input = default; - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); + return Read(key, ref input, ref output, ref readOptions, userContext); } /// @@ -162,7 +135,7 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, userContext), output); + return (Read(key, ref input, ref output, userContext), output); } /// @@ -171,17 +144,17 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, ref readOptions, userContext), output); + return (Read(key, ref input, ref output, ref readOptions, userContext), output); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { UnsafeResumeThread(); try { - return store.ContextRead(ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return store.ContextRead(key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -196,7 +169,7 @@ public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, UnsafeResumeThread(); try { - return store.ContextReadAtAddress(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return store.ContextReadAtAddress, TStoreFunctions, TAllocator>>(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -206,12 +179,12 @@ public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { UnsafeResumeThread(); try { - return store.ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return store.ContextReadAtAddress(address, key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -221,40 +194,40 @@ public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TO /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + return Upsert(key, store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + return Upsert(key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) + private Status Upsert(TKey key, long keyHash, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) { UnsafeResumeThread(); try { - return store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, userContext, sessionFunctions); + return store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out _, userContext, sessionFunctions); } finally { @@ -264,22 +237,13 @@ private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue d /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) { + var keyHash = upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key); UnsafeResumeThread(); try { - return store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + return store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); } finally { @@ -289,53 +253,56 @@ private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue d /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, TContext userContext = default) - => Upsert(ref key, ref desiredValue, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref desiredValue, ref upsertOptions, userContext); + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, ref upsertOptions, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) - => RMW(ref key, store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out _, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out _, userContext); - - /// - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); - - /// - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + private Status Upsert(TKey key, long keyHash, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) { - var keyHash = rmwOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key); - return RMW(ref key, keyHash, ref input, ref output, out recordMetadata, userContext); + UnsafeResumeThread(); + try + { + return store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out _, userContext, sessionFunctions); + } + finally + { + UnsafeSuspendThread(); + } } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) { + var keyHash = upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key); UnsafeResumeThread(); try { - return store.ContextRMW(ref key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); + return store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); } finally { @@ -345,64 +312,96 @@ private Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput out /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, userContext); - } + public Status Upsert(in TSourceLogRecord diskLogRecord) + where TSourceLogRecord : ISourceLogRecord + => Upsert(diskLogRecord, in diskLogRecord); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord { - output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + TInput input = default; + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, TContext userContext = default) + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord { TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + public Status Upsert(ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TSourceLogRecord : ISourceLogRecord + => Upsert(inputLogRecord, ref input, in inputLogRecord, ref output, ref upsertOptions, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord { - TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + var keyHash = upsertOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key); + + UnsafeResumeThread(); + try + { + return store.ContextUpsert(key, keyHash, ref input, inputLogRecord: in inputLogRecord, ref output, out _, userContext, sessionFunctions); + } + finally + { + UnsafeSuspendThread(); + } } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, TContext userContext = default) - => RMW(ref key, ref input, userContext); + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => RMW(key, store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, ref RMWOptions rmwOptions, TContext userContext = default) - => RMW(ref key, ref input, ref rmwOptions, userContext); + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out _, userContext); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, TContext userContext = default) - => Delete(ref key, store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, deleteOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var keyHash = rmwOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key); + return RMW(key, keyHash, ref input, ref output, out recordMetadata, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Delete(ref TKey key, long keyHash, TContext userContext = default) + private Status RMW(TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) { UnsafeResumeThread(); try { - return store.ContextDelete, TStoreFunctions, TAllocator>>(ref key, keyHash, userContext, sessionFunctions); + return store.ContextRMW(key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); } finally { @@ -410,54 +409,76 @@ private Status Delete(ref TKey key, long keyHash, TContext userContext = default } } + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, TContext userContext = default) + { + TOutput output = default; + return RMW(key, ref input, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + { + TOutput output = default; + return RMW(key, ref input, ref output, ref rmwOptions, userContext); + } + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public Status Delete(TKey key, TContext userContext = default) - => Delete(ref key, userContext); + => Delete(key, store.storeFunctions.GetKeyHashCode64(key), userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, ref deleteOptions, userContext); + => Delete(key, deleteOptions.KeyHash ?? store.storeFunctions.GetKeyHashCode64(key), userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetModified(TKey key) - => clientSession.ResetModified(sessionFunctions, ref key); + private Status Delete(TKey key, long keyHash, TContext userContext = default) + { + UnsafeResumeThread(); + try + { + return store.ContextDelete, TStoreFunctions, TAllocator>>(key, keyHash, userContext, sessionFunctions); + } + finally + { + UnsafeSuspendThread(); + } + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetModified(ref TKey key) - => clientSession.ResetModified(sessionFunctions, ref key); + public void ResetModified(TKey key) => clientSession.ResetModified(sessionFunctions, key); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool IsModified(TKey key) - => clientSession.IsModified(sessionFunctions, ref key); + internal bool IsModified(TKey key) => clientSession.IsModified(sessionFunctions, key); /// - public void Refresh() - => clientSession.Refresh(sessionFunctions); + public void Refresh() => clientSession.Refresh(sessionFunctions); #endregion ITsavoriteContext /// /// Copy key and value to tail, succeed only if key is known to not exist in between expectedLogicalAddress and tail. /// - /// - /// - /// - /// + /// /// LogicalAddress of the record to be copied /// Lower-bound address (addresses are searched from tail (high) to head (low); do not search for "future records" earlier than this) [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status CompactionCopyToTail(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, long currentAddress, long untilAddress) + internal Status CompactionCopyToTail(in TSourceLogRecord srcLogRecord, long currentAddress, long untilAddress) + where TSourceLogRecord : ISourceLogRecord { UnsafeResumeThread(); try { - return store.CompactionConditionalCopyToTail, TStoreFunctions, TAllocator>>( - sessionFunctions, ref key, ref input, ref value, ref output, currentAddress, untilAddress); + return store.CompactionConditionalCopyToTail, TStoreFunctions, TAllocator>, TSourceLogRecord>( + sessionFunctions, in srcLogRecord, currentAddress, untilAddress); } finally { @@ -469,19 +490,19 @@ internal Status CompactionCopyToTail(ref TKey key, ref TInput input, ref TValue /// Push a scan record to client if key is known to not exist in between expectedLogicalAddress and tail. /// /// Scan cursor tracking state, from the session on which this scan was initiated - /// - /// - /// + /// /// LogicalAddress of the record to be copied /// Lower-bound address (addresses are searched from tail (high) to head (low); do not search for "future records" earlier than this) + /// Maximum address for determining liveness, records after this address are not considered when checking validity. [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ConditionalScanPush(ScanCursorState scanCursorState, RecordInfo recordInfo, ref TKey key, ref TValue value, long currentAddress, long untilAddress, long maxAddress) + internal Status ConditionalScanPush(ScanCursorState scanCursorState, in TSourceLogRecord srcLogRecord, long currentAddress, long untilAddress, long maxAddress) + where TSourceLogRecord : ISourceLogRecord { UnsafeResumeThread(); try { - return store.hlogBase.ConditionalScanPush, TStoreFunctions, TAllocator>>( - sessionFunctions, scanCursorState, recordInfo, ref key, ref value, currentAddress, untilAddress, maxAddress); + return store.hlogBase.ConditionalScanPush, TStoreFunctions, TAllocator>, TSourceLogRecord>( + sessionFunctions, scanCursorState, in srcLogRecord, currentAddress, currentAddress, untilAddress, maxAddress); } finally { @@ -496,13 +517,13 @@ internal Status ConditionalScanPush(ScanCursorState scanCursorStat /// Logical address of record, if found /// Look until this address; if less than HeadAddress, then HeadAddress is used /// Status - public Status ContainsKeyInMemory(ref TKey key, out long logicalAddress, long fromAddress = -1) + public Status ContainsKeyInMemory(TKey key, out long logicalAddress, long fromAddress = -1) // TODO: remove when we remove tempkv/tempdb in iterators { UnsafeResumeThread(); try { - return store.InternalContainsKeyInMemory, TStoreFunctions, TAllocator>>( - ref key, sessionFunctions, out logicalAddress, fromAddress); + return store.InternalContainsKeyInMemory, TStoreFunctions, TAllocator>>( + key, sessionFunctions, out logicalAddress, fromAddress); } finally { diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/ClientSession.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/ClientSession.cs index 448c4821426..ea56612472c 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/ClientSession.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/ClientSession.cs @@ -13,23 +13,29 @@ namespace Tsavorite.core /// /// Thread-independent session interface to Tsavorite /// - public sealed class ClientSession : IClientSession, IDisposable - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public sealed class ClientSession : IClientSession, IDisposable + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - internal readonly TsavoriteKV store; + internal readonly TsavoriteKV store; - internal readonly TsavoriteKV.TsavoriteExecutionContext ctx; + internal readonly TsavoriteKV.TsavoriteExecutionContext ctx; internal readonly TFunctions functions; - internal CompletedOutputIterator completedOutputs; + internal CompletedOutputIterator completedOutputs; - readonly UnsafeContext uContext; - readonly LockableUnsafeContext luContext; - readonly LockableContext lContext; - readonly BasicContext bContext; + readonly UnsafeContext uContext; + readonly TransactionalUnsafeContext luContext; + readonly TransactionalContext lContext; + readonly BasicContext bContext; + readonly ConsistentReadContext crContext; + readonly TransactionalConsistentReadContext tcrContext; internal const string NotAsyncSessionErr = "Session does not support async operations"; @@ -40,56 +46,69 @@ public sealed class ClientSession scanCursorState; + ScanCursorState scanCursorState; - internal void AcquireLockable(TSessionFunctions sessionFunctions) - where TSessionFunctions : ISessionFunctionsWrapper + internal void AcquireTransactional(TSessionFunctions sessionFunctions) + where TSessionFunctions : ISessionFunctionsWrapper { - CheckIsNotAcquiredLockable(sessionFunctions); - sessionFunctions.Ctx.isAcquiredLockable = true; + CheckIsNotAcquiredTransactional(sessionFunctions); + sessionFunctions.Ctx.isAcquiredTransactional = true; } internal void LocksAcquired(TSessionFunctions sessionFunctions, long txnVersion) - where TSessionFunctions : ISessionFunctionsWrapper + where TSessionFunctions : ISessionFunctionsWrapper { - CheckIsAcquiredLockable(sessionFunctions); + CheckIsAcquiredTransactional(sessionFunctions); sessionFunctions.Ctx.txnVersion = txnVersion; } - internal void ReleaseLockable(TSessionFunctions sessionFunctions) - where TSessionFunctions : ISessionFunctionsWrapper + internal void ReleaseTransactional(TSessionFunctions sessionFunctions) + where TSessionFunctions : ISessionFunctionsWrapper { - CheckIsAcquiredLockable(sessionFunctions); + CheckIsAcquiredTransactional(sessionFunctions); if (TotalLockCount > 0) - throw new TsavoriteException($"EndLockable called with locks held: {sharedLockCount} shared locks, {exclusiveLockCount} exclusive locks"); - sessionFunctions.Ctx.isAcquiredLockable = false; + throw new TsavoriteException($"EndTransactional called with locks held: {sharedLockCount} shared locks, {exclusiveLockCount} exclusive locks"); + sessionFunctions.Ctx.isAcquiredTransactional = false; sessionFunctions.Ctx.txnVersion = 0; } - internal void CheckIsAcquiredLockable(TSessionFunctions sessionFunctions) - where TSessionFunctions : ISessionFunctionsWrapper + internal void CheckIsAcquiredTransactional(TSessionFunctions sessionFunctions) + where TSessionFunctions : ISessionFunctionsWrapper { - if (!sessionFunctions.Ctx.isAcquiredLockable) - throw new TsavoriteException("Lockable method call when BeginLockable has not been called"); + if (!sessionFunctions.Ctx.isAcquiredTransactional) + throw new TsavoriteException("Transactional method call when BeginTransactional has not been called"); } - void CheckIsNotAcquiredLockable(TSessionFunctions sessionFunctions) - where TSessionFunctions : ISessionFunctionsWrapper + void CheckIsNotAcquiredTransactional(TSessionFunctions sessionFunctions) + where TSessionFunctions : ISessionFunctionsWrapper { - if (sessionFunctions.Ctx.isAcquiredLockable) - throw new TsavoriteException("BeginLockable cannot be called twice (call EndLockable first)"); + if (sessionFunctions.Ctx.isAcquiredTransactional) + throw new TsavoriteException("BeginTransactional cannot be called twice (call EndTransactional first)"); } internal ClientSession( - TsavoriteKV store, - TsavoriteKV.TsavoriteExecutionContext ctx, + TsavoriteKV store, + TsavoriteKV.TsavoriteExecutionContext ctx, TFunctions functions, + bool enableConsistentRead = false, ILoggerFactory loggerFactory = null) { - bContext = new(this); - uContext = new(this); - lContext = new(this); - luContext = new(this); + if (enableConsistentRead) + { + crContext = new(this); + tcrContext = new(this); + bContext = crContext.BasicContext; + uContext = new(this); + lContext = tcrContext.TransactionalContext; + luContext = new(this); + } + else + { + bContext = new(this); + uContext = new(this); + lContext = new(this); + luContext = new(this); + } this.loggerFactory = loggerFactory; logger = loggerFactory?.CreateLogger($"ClientSession-{GetHashCode():X8}"); @@ -141,34 +160,42 @@ public void Dispose() /// /// Return a new interface to Tsavorite operations that supports manual epoch control. /// - public UnsafeContext UnsafeContext => uContext; + public UnsafeContext UnsafeContext => uContext; /// - /// Return a new interface to Tsavorite operations that supports manual locking and epoch control. + /// Return a new interface to Tsavorite operations that supports Transactional locking and manual epoch control. /// - public LockableUnsafeContext LockableUnsafeContext => luContext; + public TransactionalUnsafeContext TransactionalUnsafeContext => luContext; /// - /// Return a session wrapper that supports manual locking. + /// Return a session wrapper that supports Transactional locking. /// - public LockableContext LockableContext => lContext; + public TransactionalContext TransactionalContext => lContext; /// /// Return a session wrapper struct that passes through to client session /// - public BasicContext BasicContext => bContext; + public BasicContext BasicContext => bContext; - #region ITsavoriteContext + /// + /// Return the consistent read context; + /// + public ConsistentReadContext ConsistentReadContext => crContext; - /// - public long GetKeyHash(TKey key) => store.GetKeyHash(ref key); + /// + /// Return the transactional consistent read context + /// + public TransactionalConsistentReadContext TransactionalConsistentReadContext => tcrContext; + + #region ITsavoriteContext /// - public long GetKeyHash(ref TKey key) => store.GetKeyHash(ref key); + public long GetKeyHash(TKey key) + => store.GetKeyHash(key); /// internal void Refresh(TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { UnsafeResumeThread(sessionFunctions); try @@ -182,13 +209,13 @@ internal void Refresh(TSessionFunctionsWrapper session } /// - internal void ResetModified(TSessionFunctionsWrapper sessionFunctions, ref TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal void ResetModified(TSessionFunctionsWrapper sessionFunctions, TKey key) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { UnsafeResumeThread(sessionFunctions); try { - UnsafeResetModified(sessionFunctions, ref key); + UnsafeResetModified(sessionFunctions, key); } finally { @@ -197,13 +224,13 @@ internal void ResetModified(TSessionFunctionsWrapper s } /// - public int CompareKeyHashes(TLockableKey key1, TLockableKey key2) where TLockableKey : ILockableKey => store.LockTable.CompareKeyHashes(key1, key2); + public int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => store.LockTable.CompareKeyHashes(key1, key2); /// - public int CompareKeyHashes(ref TLockableKey key1, ref TLockableKey key2) where TLockableKey : ILockableKey => store.LockTable.CompareKeyHashes(ref key1, ref key2); + public int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => store.LockTable.CompareKeyHashes(ref key1, ref key2); /// - public void SortKeyHashes(Span keys) where TLockableKey : ILockableKey => store.LockTable.SortKeyHashes(keys); + public void SortKeyHashes(Span keys) where TTransactionalKey : ITransactionalKey => store.LockTable.SortKeyHashes(keys); #endregion ITsavoriteContext @@ -211,12 +238,12 @@ internal void ResetModified(TSessionFunctionsWrapper s /// internal bool CompletePending(TSessionFunctionsWrapper sessionFunctions, bool wait = false, bool spinWaitForCommit = false) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper => CompletePending(sessionFunctions, getOutputs: false, wait, spinWaitForCommit); /// - internal bool CompletePendingWithOutputs(TSessionFunctionsWrapper sessionFunctions, out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool CompletePendingWithOutputs(TSessionFunctionsWrapper sessionFunctions, out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { InitializeCompletedOutputs(); var result = CompletePending(sessionFunctions, getOutputs: true, wait, spinWaitForCommit); @@ -228,8 +255,8 @@ internal bool CompletePendingWithOutputs(TSessionFunct /// Synchronously complete outstanding pending synchronous operations, returning outputs for the completed operations. /// Assumes epoch protection is managed by user. Async operations must be completed individually. /// - internal bool UnsafeCompletePendingWithOutputs(TSessionFunctionsWrapper sessionFunctions, out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool UnsafeCompletePendingWithOutputs(TSessionFunctionsWrapper sessionFunctions, out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { InitializeCompletedOutputs(); var result = UnsafeCompletePending(sessionFunctions, true, wait, spinWaitForCommit); @@ -240,13 +267,13 @@ internal bool UnsafeCompletePendingWithOutputs(TSessio private void InitializeCompletedOutputs() { if (completedOutputs is null) - completedOutputs = new CompletedOutputIterator(); + completedOutputs = new CompletedOutputIterator(); else completedOutputs.Dispose(); } internal bool CompletePending(TSessionFunctionsWrapper sessionFunctions, bool getOutputs, bool wait, bool spinWaitForCommit) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { UnsafeResumeThread(sessionFunctions); try @@ -260,7 +287,7 @@ internal bool CompletePending(TSessionFunctionsWrapper } internal bool UnsafeCompletePending(TSessionFunctionsWrapper sessionFunctions, bool getOutputs, bool wait, bool spinWaitForCommit) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var requestedOutputs = getOutputs ? completedOutputs : default; var result = store.InternalCompletePending(sessionFunctions, wait, requestedOutputs); @@ -283,13 +310,13 @@ internal bool UnsafeCompletePending(TSessionFunctionsW /// internal ValueTask CompletePendingAsync(TSessionFunctionsWrapper sessionFunctions, bool waitForCommit = false, CancellationToken token = default) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper => CompletePendingAsync(sessionFunctions, getOutputs: false, waitForCommit, token); /// - internal async ValueTask> CompletePendingWithOutputsAsync(TSessionFunctionsWrapper sessionFunctions, + internal async ValueTask> CompletePendingWithOutputsAsync(TSessionFunctionsWrapper sessionFunctions, bool waitForCommit = false, CancellationToken token = default) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { InitializeCompletedOutputs(); await CompletePendingAsync(sessionFunctions, getOutputs: true, waitForCommit, token).ConfigureAwait(false); @@ -297,7 +324,7 @@ internal async ValueTask(TSessionFunctionsWrapper sessionFunctions, bool getOutputs, bool waitForCommit = false, CancellationToken token = default) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { token.ThrowIfCancellationRequested(); @@ -325,35 +352,30 @@ public async ValueTask ReadyToCompletePendingAsync(CancellationToken token = def if (store.epoch.ThisInstanceProtected()) throw new NotSupportedException("Async operations not supported over protected epoch"); - await TsavoriteKV.ReadyToCompletePendingAsync(ctx, token).ConfigureAwait(false); + await TsavoriteKV.ReadyToCompletePendingAsync(ctx, token).ConfigureAwait(false); } #endregion Pending Operations #region Other Operations - internal void UnsafeResetModified(TSessionFunctionsWrapper sessionFunctions, ref TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal void UnsafeResetModified(TSessionFunctionsWrapper sessionFunctions, TKey key) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { OperationStatus status; do - status = store.InternalModifiedBitOperation(ref key, out _); + status = store.InternalModifiedBitOperation(key, out _); while (store.HandleImmediateNonPendingRetryStatus(status, sessionFunctions)); } /// - internal unsafe void ResetModified(TSessionFunctionsWrapper sessionFunctions, TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - => ResetModified(sessionFunctions, ref key); - - /// - internal bool IsModified(TSessionFunctionsWrapper sessionFunctions, ref TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool IsModified(TSessionFunctionsWrapper sessionFunctions, TKey key) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { UnsafeResumeThread(sessionFunctions); try { - return UnsafeIsModified(sessionFunctions, ref key); + return UnsafeIsModified(sessionFunctions, key); } finally { @@ -361,29 +383,24 @@ internal bool IsModified(TSessionFunctionsWrapper sess } } - internal bool UnsafeIsModified(TSessionFunctionsWrapper sessionFunctions, ref TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool UnsafeIsModified(TSessionFunctionsWrapper sessionFunctions, TKey key) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { RecordInfo modifiedInfo; OperationStatus status; do - status = store.InternalModifiedBitOperation(ref key, out modifiedInfo, false); + status = store.InternalModifiedBitOperation(key, out modifiedInfo, false); while (store.HandleImmediateNonPendingRetryStatus(status, sessionFunctions)); return modifiedInfo.Modified; } - /// - internal unsafe bool IsModified(TSessionFunctionsWrapper sessionFunctions, TKey key) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - => IsModified(sessionFunctions, ref key); - /// /// Wait for commit of all operations completed until the current point in session. /// Does not itself issue checkpoint/commits. /// /// private async ValueTask WaitForCommitAsync(TSessionFunctionsWrapper sessionFunctions, CancellationToken token = default) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { token.ThrowIfCancellationRequested(); @@ -411,19 +428,7 @@ private async ValueTask WaitForCommitAsync(TSessionFun /// Compaction type (whether we lookup records or scan log for liveness checking) /// Address until which compaction was done public long Compact(long compactUntilAddress, CompactionType compactionType = CompactionType.Scan) - => Compact(compactUntilAddress, compactionType, default(DefaultCompactionFunctions)); - - /// - /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log - /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate - /// - /// Input for SingleWriter - /// Output from SingleWriter; it will be called all records that are moved, before Compact() returns, so the user must supply buffering or process each output completely - /// Compact log until this address - /// Compaction type (whether we lookup records or scan log for liveness checking) - /// Address until which compaction was done - public long Compact(ref TInput input, ref TOutput output, long compactUntilAddress, CompactionType compactionType = CompactionType.Scan) - => Compact(ref input, ref output, compactUntilAddress, compactionType, default(DefaultCompactionFunctions)); + => Compact(compactUntilAddress, compactionType, default(DefaultCompactionFunctions)); /// /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log @@ -431,38 +436,18 @@ public long Compact(ref TInput input, ref TOutput output, long compactUntilAddre /// /// Compact log until this address /// Compaction type (whether we lookup records or scan log for liveness checking) - /// User provided compaction functions (see ). + /// User provided compaction functions (see ). /// Address until which compaction was done public long Compact(long untilAddress, CompactionType compactionType, CompactionFunctions compactionFunctions) - where CompactionFunctions : ICompactionFunctions - { - TInput input = default; - TOutput output = default; - return store.Compact(functions, compactionFunctions, ref input, ref output, untilAddress, compactionType); - } - - /// - /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log - /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate - /// - /// Input for SingleWriter - /// Output from SingleWriter; it will be called all records that are moved, before Compact() returns, so the user must supply buffering or process each output completely - /// Compact log until this address - /// Compaction type (whether we lookup records or scan log for liveness checking) - /// User provided compaction functions (see ). - /// Address until which compaction was done - public long Compact(ref TInput input, ref TOutput output, long untilAddress, CompactionType compactionType, CompactionFunctions compactionFunctions) - where CompactionFunctions : ICompactionFunctions - { - return store.Compact(functions, compactionFunctions, ref input, ref output, untilAddress, compactionType); - } + where CompactionFunctions : ICompactionFunctions + => store.Compact(compactionFunctions, untilAddress, compactionType); /// /// Pull iterator for all (distinct) live key-values stored in Tsavorite /// /// Report records until this address (tail by default) /// Tsavorite iterator - public ITsavoriteScanIterator Iterate(long untilAddress = -1) + public ITsavoriteScanIterator Iterate(long untilAddress = -1) => store.Iterate(functions, untilAddress); /// @@ -472,7 +457,7 @@ public ITsavoriteScanIterator Iterate(long untilAddress = -1) /// Report records until this address (tail by default) /// True if Iteration completed; false if Iteration ended early due to one of the TScanIterator reader functions returning false public bool Iterate(ref TScanFunctions scanFunctions, long untilAddress = -1) - where TScanFunctions : IScanIteratorFunctions + where TScanFunctions : IScanIteratorFunctions => store.Iterate(functions, ref scanFunctions, untilAddress); /// @@ -486,12 +471,40 @@ public bool Iterate(ref TScanFunctions scanFunctions, long until /// Whether to reset cursor at the end of the iteration /// Whether to include tombstoned record when iterating /// True if Iteration completed; false if Iteration ended early due to one of the TScanIterator reader functions returning false - public bool IterateLookup(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, bool validateCursor = false, long maxAddress = long.MaxValue, bool resetCursor = true, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions + public bool IterateLookup(ref TScanFunctions scanFunctions, ref long cursor, long untilAddress = -1, bool validateCursor = false, long maxAddress = long.MaxValue, + bool resetCursor = true, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions { if (untilAddress == -1) untilAddress = store.Log.TailAddress; - return ScanCursor(ref cursor, count: long.MaxValue, scanFunctions, endAddress: untilAddress, validateCursor: validateCursor, maxAddress: maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); + return ScanCursor(ref cursor, count: long.MaxValue, scanFunctions, endAddress: untilAddress, validateCursor: validateCursor, maxAddress: maxAddress, + resetCursor: resetCursor, includeTombstones: includeTombstones); + } + + /// + /// Push iteration with snapshot semantics: emits each unique live key exactly once, + /// using its latest version that lives in [BeginAddress, TailAddress(at call time)). + /// Concurrent RCUs that move a key's tail above the captured TailAddress during + /// the scan do not suppress the in-range version — the scan returns a consistent + /// point-in-time view of every key that existed when the scan started. + /// + /// Internally captures Log.TailAddress once and passes it as both + /// endAddress and maxAddress to . This is the + /// lookup-based equivalent of the legacy + /// (which builds a parallel tempKv sized like the keyspace and copies every + /// in-range record into it). The snapshot variant avoids that O(N) memory cost; per-call + /// allocations match those of . + /// + /// Functions receiving pushed records. + /// Whether to include tombstoned records while iterating. Default false. + /// True if iteration completed; false if Reader returned false or iteration ended early. + public bool IterateLookupSnapshot(ref TScanFunctions scanFunctions, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + { + var untilAddress = store.Log.TailAddress; + long cursor = 0; + return ScanCursor(ref cursor, count: long.MaxValue, scanFunctions, endAddress: untilAddress, + validateCursor: false, maxAddress: untilAddress, resetCursor: true, includeTombstones: includeTombstones); } /// @@ -511,20 +524,21 @@ public bool IterateLookup(ref TScanFunctions scanFunctions, ref /// Maximum address for determining liveness, records after this address are not considered when checking validity. /// Whether to set cursor to zero at the end of iteration. /// Whether to include tombstoned records while iterating. - /// True if Scan completed and pushed records and there may be more records; false if Scan ended early due to finding less than records /// or one of the TScanIterator reader functions returning false, or if we determined that there are no records remaining. In other words, if this returns true, /// there may be more records satisfying the iteration criteria beyond . - public bool ScanCursor(ref long cursor, long count, TScanFunctions scanFunctions, long endAddress = long.MaxValue, bool validateCursor = false, long maxAddress = long.MaxValue, bool resetCursor = true, bool includeTombstones = false) - where TScanFunctions : IScanIteratorFunctions - => store.hlogBase.ScanCursor(store, scanCursorState ??= new(), ref cursor, count, scanFunctions, endAddress, validateCursor, maxAddress, resetCursor: resetCursor, includeTombstones: includeTombstones); + public bool ScanCursor(ref long cursor, long count, TScanFunctions scanFunctions, long endAddress = long.MaxValue, bool validateCursor = false, + long maxAddress = long.MaxValue, bool resetCursor = true, bool includeTombstones = false) + where TScanFunctions : IScanIteratorFunctions + => store.hlogBase.ScanCursor(store, scanCursorState ??= new(), ref cursor, count, scanFunctions, endAddress, validateCursor, maxAddress, + resetCursor: resetCursor, includeTombstones: includeTombstones); /// /// Resume session on current thread. IMPORTANT: Call SuspendThread before any async op. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void UnsafeResumeThread(TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // We do not track any "acquired" state here; if someone mixes calls between safe and unsafe contexts, they will // get the "trying to acquire already-acquired epoch" error. diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/ConsistentReadContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/ConsistentReadContext.cs new file mode 100644 index 00000000000..d3a62bf0ef3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/ConsistentReadContext.cs @@ -0,0 +1,289 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + /// + /// Consistent read context that extends basicContext functionality with consistent read protocols. + /// + public readonly struct ConsistentReadContext + : ITsavoriteContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + public readonly BasicContext BasicContext { get; } + + /// + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Session.store.GetKeyHash(key); + + internal ConsistentReadContext(ClientSession clientSession) + { + BasicContext = new BasicContext(clientSession); + } + + /// + public bool IsNull => BasicContext.IsNull; + + /// + public ClientSession Session => BasicContext.Session; + + #region ITsavoriteContext/Read + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + { + var hash = GetKeyHash(key); + Session.functions.BeforeConsistentReadCallback(hash); + var status = BasicContext.Read(key, ref input, ref output, userContext); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + => Read(key, ref input, ref output, ref readOptions, out _, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, ref readOptions, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, ref readOptions, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var hash = GetKeyHash(key); + Session.functions.BeforeConsistentReadCallback(hash); + var status = BasicContext.Read(key, ref input, ref output, ref readOptions, out recordMetadata, userContext); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow reads from address!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow reads from address!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) + where TBatch : IReadArgBatch +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + do + { + Thread.Yield(); + Session.functions.BeforeConsistentReadKeyBatchCallback(batch.Parameters); + BasicContext.ReadWithPrefetch(ref batch, userContext); + } while (!Session.functions.AfterConsistentReadKeyBatchCallback(batch.Count)); + } + + #endregion + + #region ITsavoriteContext + + /// + public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) + { + var status = BasicContext.CompletePending(wait, spinWaitForCommit); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + { + var status = BasicContext.CompletePendingWithOutputs(out completedOutputs, wait, spinWaitForCommit); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + public async ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationToken token = default) + { + await BasicContext.CompletePendingAsync(waitForCommit, token).ConfigureAwait(false); + Session.functions.AfterConsistentReadKeyCallback(); + } + + /// + public async ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + { + var status = BasicContext.CompletePendingWithOutputsAsync(waitForCommit, token); + Session.functions.AfterConsistentReadKeyCallback(); + return await status.ConfigureAwait(false); + } + + /// + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => BasicContext.Upsert(diskLogRecord); + + /// + public Status Upsert(TKey key, in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Upsert(TKey key, ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Delete(TKey key, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + public void ResetModified(TKey key) + => throw new TsavoriteException("Consistent read context does not reset ResetModified!"); + + /// + public void Refresh() + => throw new TsavoriteException("Consistent read context does not reset Refresh!"); + #endregion + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/ILockableContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/ITransactionalContext.cs similarity index 51% rename from libs/storage/Tsavorite/cs/src/core/ClientSession/ILockableContext.cs rename to libs/storage/Tsavorite/cs/src/core/ClientSession/ITransactionalContext.cs index a43dc189994..8df46423738 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/ILockableContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/ITransactionalContext.cs @@ -7,17 +7,16 @@ namespace Tsavorite.core { /// - /// Lockable context functions. Useful when doing generic locking across diverse - /// and - /// specializations. + /// Transactional context functions. Useful when doing generic locking across diverse + /// and + /// specializations. /// - /// - public interface ILockableContext + public interface ITransactionalContext { /// /// Begins a series of lock operations on possibly multiple keys; call before any locks are taken. /// - void BeginLockable(); + void BeginTransaction(); /// /// Call after all locks are acquired. Provide transaction version @@ -28,116 +27,115 @@ public interface ILockableContext /// /// Ends a series of lock operations on possibly multiple keys; call after all locks are released. /// - void EndLockable(); + void EndTransaction(); /// - /// Compare two structures that implement ILockableKey. + /// Compare two structures that implement . /// - /// The type of the app data struct or class containing key info + /// The type of the app data struct or class containing key info /// The first key to compare /// The first key to compare /// The result of key1.CompareTo(key2) - int CompareKeyHashes(TLockableKey key1, TLockableKey key2) - where TLockableKey : ILockableKey; + int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) + where TTransactionalKey : ITransactionalKey; /// - /// Compare two structures that implement ILockableKey. + /// Compare two structures that implement . /// - /// The type of the app data struct or class containing key info + /// The type of the app data struct or class containing key info /// The first key to compare /// The first key to compare /// The result of key1.CompareTo(key2) - int CompareKeyHashes(ref TLockableKey key1, ref TLockableKey key2) - where TLockableKey : ILockableKey; + int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) + where TTransactionalKey : ITransactionalKey; /// - /// Sort an array of app data structures (or classes) by lock code and lock type; these will be passed to Lockable*Session.Lock + /// Sort an array of app data structures (or classes) by lock code and lock type; these will be passed to Transactional*Session.Lock /// - /// The type of the app data struct or class containing key info + /// The type of the app data struct or class containing key info /// The array of app key data - void SortKeyHashes(Span keys) - where TLockableKey : ILockableKey; + void SortKeyHashes(Span keys) + where TTransactionalKey : ITransactionalKey; /// /// Locks the keys identified in the passed array. /// - /// + /// /// keys to be locked, and whether that locking is shared or exclusive; must be sorted by . - void Lock(ReadOnlySpan keys) - where TLockableKey : ILockableKey; + void Lock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey; /// - /// Locks the keys identified in the passed array, with retry limits or cancellation. + /// Locks the keys identified in the passed array. /// - /// + /// /// keys to be locked, and whether that locking is shared or exclusive; must be sorted by . - bool TryLock(ReadOnlySpan keys) - where TLockableKey : ILockableKey; + bool TryLock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey; /// - /// Locks the keys identified in the passed array, with retry limits or cancellation. + /// Locks the keys identified in the passed array, with retry limits. /// - /// + /// /// keys to be locked, and whether that locking is shared or exclusive; must be sorted by . /// TimeSpan limiting the duration of the TryLock() call over all keys. - bool TryLock(ReadOnlySpan keys, TimeSpan timeout) - where TLockableKey : ILockableKey; + bool TryLock(ReadOnlySpan keys, TimeSpan timeout) + where TTransactionalKey : ITransactionalKey; /// /// Locks the keys identified in the passed array, with retry limits or cancellation. /// - /// + /// /// keys to be locked, and whether that locking is shared or exclusive; must be sorted by . /// The cancellation token - bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) - where TLockableKey : ILockableKey; + bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey; /// - /// Locks the keys identified in the passed array, with retry limits or cancellation. + /// /// - /// /// keys to be locked, and whether that locking is shared or exclusive; must be sorted by . - /// TimeSpan limiting the duration of the TryLock() call over all keys. + /// TimeSpan limiting the duration of the TryLock() call. /// The cancellation token - bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey; + bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey; /// - /// Tries to promote a shared lock the key to an exclusive lock, with retry limits or cancellation. + /// Promotes a shared lock on the key to an exclusive lock, with retry limits or cancellation. /// - /// + /// /// key whose lock is to be promoted. /// On success, the caller must update the ILockableKey.LockType so the unlock has the right type - bool TryPromoteLock(TLockableKey key) - where TLockableKey : ILockableKey; + bool TryPromoteLock(TTransactionalKey key) + where TTransactionalKey : ITransactionalKey; /// /// Promotes a shared lock on the key to an exclusive lock, with retry limits or cancellation. /// - /// + /// /// key whose lock is to be promoted. /// The cancellation token - /// On success, the caller must update the ILockableKey.LockType so the unlock has the right type - bool TryPromoteLock(TLockableKey key, CancellationToken cancellationToken) - where TLockableKey : ILockableKey; + /// On success, the caller must update the ITransactionalKey.LockType so the unlock has the right type + bool TryPromoteLock(TTransactionalKey key, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey; /// /// Promotes a shared lock on the key to an exclusive lock, with retry limits or cancellation. /// - /// + /// /// key whose lock is to be promoted. /// TimeSpan limiting the duration of the TryPromoteLock() call. /// The cancellation token, if any - /// On success, the caller must update the ILockableKey.LockType so the unlock has the right type - bool TryPromoteLock(TLockableKey key, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey; + /// On success, the caller must update the ITransactionalKey.LockType so the unlock has the right type + bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey; /// /// Unlocks the keys identified in the passed array. /// - /// + /// /// key hashCodes to be unlocked, and whether that unlocking is shared or exclusive; must be sorted by . - void Unlock(ReadOnlySpan keys) - where TLockableKey : ILockableKey; + void Unlock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/ITsavoriteContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/ITsavoriteContext.cs index e0a9d4c1281..4a116cde277 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/ITsavoriteContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/ITsavoriteContext.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Threading; using System.Threading.Tasks; @@ -9,30 +10,32 @@ namespace Tsavorite.core /// /// Interface for Key-only Tsavorite operations /// - public interface ITsavoriteContext + public interface ITsavoriteContext { /// - /// Obtain a code by which groups of keys will be sorted for manual locking, to avoid deadlocks. + /// Obtain a code by which groups of keys will be sorted for Transactional locking, to avoid deadlocks. /// The key to obtain a code for /// - /// The hashcode of the key; created and returned by - long GetKeyHash(TKey key); - - /// - /// Obtain a code by which groups of keys will be sorted for manual locking, to avoid deadlocks. - /// The key to obtain a code for - /// - /// The hashcode of the key; created and returned by - long GetKeyHash(ref TKey key); + /// The hashcode of the key; created and returned by + long GetKeyHash(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; } /// /// Interface for Tsavorite operations /// - public interface ITsavoriteContext : ITsavoriteContext - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public interface ITsavoriteContext : ITsavoriteContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Indicates whether this context has been initialized. @@ -40,9 +43,9 @@ public interface ITsavoriteContext - /// Obtain the underlying + /// Obtain the underlying /// - ClientSession Session { get; } + ClientSession Session { get; } /// /// Synchronously complete outstanding pending synchronous operations. @@ -61,7 +64,7 @@ public interface ITsavoriteContextWait for all pending operations on session to complete /// Spin-wait until ongoing commit/checkpoint, if any, completes /// True if all pending operations have completed, false otherwise - bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false); + bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false); /// /// Complete all pending synchronous Tsavorite operations. @@ -75,28 +78,7 @@ public interface ITsavoriteContext /// Outputs completed by this operation - ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default); - - /// - /// Read operation - /// - /// The key to look up - /// Input to help extract the retrieved value into - /// The location to place the retrieved value - /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default); - - /// - /// Read operation - /// - /// The key to look up - /// Input to help extract the retrieved value into - /// The location to place the retrieved value - /// Contains options controlling the Read operation - /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default); + ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default); /// /// Read operation @@ -105,8 +87,8 @@ public interface ITsavoriteContextInput to help extract the retrieved value into /// The location to place the retrieved value /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(TKey key, TInput input, out TOutput output, TContext userContext = default); + /// is populated by the implementation + Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default); /// /// Read operation @@ -116,8 +98,8 @@ public interface ITsavoriteContextThe location to place the retrieved value /// Contains options controlling the Read operation /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(TKey key, TInput input, out TOutput output, ref ReadOptions readOptions, TContext userContext = default); + /// is populated by the implementation + Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default); /// /// Read operation @@ -125,8 +107,8 @@ public interface ITsavoriteContextThe key to look up /// The location to place the retrieved value /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(ref TKey key, ref TOutput output, TContext userContext = default); + /// is populated by the implementation + Status Read(TKey key, ref TOutput output, TContext userContext = default); /// /// Read operation @@ -135,27 +117,8 @@ public interface ITsavoriteContextThe location to place the retrieved value /// Contains options controlling the Read operation /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(ref TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default); - - /// - /// Read operation - /// - /// - /// - /// - /// - Status Read(TKey key, out TOutput output, TContext userContext = default); - - /// - /// Read operation - /// - /// - /// - /// Contains options controlling the Read operation - /// - /// - Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TContext userContext = default); + /// is populated by the implementation + Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default); /// /// Read operation @@ -184,15 +147,15 @@ public interface ITsavoriteContextContains options controlling the Read operation /// On output, receives: /// - ///
  • The address of the found record. This may be different from the passed on the call, due to - /// tracing back over hash collisions until we arrive at the key match
    • - ///
  • A copy of the record's header in ; can be passed - /// in a subsequent call, thereby enumerating all records in a key's hash chain.
    • + /// The address of the found record. This may be different from the passed on the call, due to + /// tracing back over hash collisions until we arrive at the key match + /// A copy of the record's header in ; can be passed + /// in a subsequent call, thereby enumerating all records in a key's hash chain. ///     /// /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation - Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default); + /// is populated by the implementation + Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// /// Read operation that accepts an address to lookup at, instead of a key. @@ -203,7 +166,7 @@ public interface ITsavoriteContextContains options controlling the Read operation, including the address to read at in StartAddress /// On output, receives metadata about the record /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation; this should store the key if it needs it + /// is populated by the implementation; this should store the key if it needs it Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// @@ -216,8 +179,8 @@ public interface ITsavoriteContextContains options controlling the Read operation, including the address to read at in StartAddress /// On output, receives metadata about the record /// User application context passed in case the read goes pending due to IO - /// is populated by the implementation; this should store the key if it needs it - Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default); + /// is populated by the implementation; this should store the key if it needs it + Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// /// Read batch operation, which attempts to prefetch as an optimization. @@ -236,7 +199,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status Upsert(ref TKey key, ref TValue desiredValue, TContext userContext = default); + Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default); /// /// Upsert operation @@ -246,7 +209,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status Upsert(ref TKey key, ref TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default); + Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default); /// /// Upsert operation @@ -257,7 +220,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default); /// /// Upsert operation @@ -269,7 +232,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default); /// /// Upsert operation @@ -278,42 +241,41 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// + /// /// /// /// - Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// /// Upsert operation /// /// - /// /// - /// - /// - /// /// /// - Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default); + Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default); /// /// Upsert operation /// /// /// + /// /// /// - Status Upsert(TKey key, TValue desiredValue, TContext userContext = default); + Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default); /// /// Upsert operation /// /// + /// /// - /// + /// /// /// - Status Upsert(TKey key, TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default); /// /// Upsert operation @@ -322,9 +284,10 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// + /// /// /// - Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default); /// /// Upsert operation @@ -334,53 +297,69 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// + /// /// /// - Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default); + Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// - /// RMW operation + /// Upsert operation with a disk log record /// - /// - /// - /// - /// + /// Log record that was read from disk /// - Status RMW(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default); + Status Upsert(in TSourceLogRecord diskLogRecord) + where TSourceLogRecord : ISourceLogRecord; /// - /// RMW operation + /// Upsert operation with a disk log record /// - /// - /// - /// - /// - /// + /// Key, which may be from or may be a modified key (e.g. prepending a prefix) + /// Log record that was read from disk /// - Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default); + Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord; /// - /// RMW operation + /// Upsert operation with a disk log record /// - /// + /// Key, which may be from or may be a modified key (e.g. prepending a prefix) /// - /// - /// - /// + /// Log record that was read from disk /// - Status RMW(ref TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default); + Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord; /// - /// RMW operation + /// Upsert operation with a disk log record and user-supplied key /// - /// /// + /// Log record that was read from disk /// - /// - /// + /// /// /// - Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default); + Status Upsert(ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TSourceLogRecord : ISourceLogRecord; + + /// + /// Upsert operation with a disk log record and user-supplied key + /// + /// Log record that was read from disk + /// + Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord; /// /// RMW operation @@ -390,7 +369,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status RMW(TKey key, TInput input, out TOutput output, TContext userContext = default); + Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default); /// /// RMW operation @@ -401,26 +380,30 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status RMW(TKey key, TInput input, out TOutput output, ref RMWOptions rmwOptions, TContext userContext = default); + Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default); /// /// RMW operation /// /// /// + /// + /// /// /// - Status RMW(ref TKey key, ref TInput input, TContext userContext = default); + Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default); /// /// RMW operation /// /// /// + /// /// + /// /// /// - Status RMW(ref TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default); + Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default); /// /// RMW operation @@ -429,7 +412,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status RMW(TKey key, TInput input, TContext userContext = default); + Status RMW(TKey key, ref TInput input, TContext userContext = default); /// /// RMW operation @@ -439,24 +422,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// /// /// - Status RMW(TKey key, TInput input, ref RMWOptions rmwOptions, TContext userContext = default); - - /// - /// Delete operation - /// - /// - /// - /// - Status Delete(ref TKey key, TContext userContext = default); - - /// - /// Delete operation - /// - /// - /// - /// - /// - Status Delete(ref TKey key, ref DeleteOptions deleteOptions, TContext userContext = default); + Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default); /// /// Delete operation @@ -479,7 +445,7 @@ void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) /// Reset the modified bit of a record (for in memory records) /// /// - void ResetModified(ref TKey key); + void ResetModified(TKey key); /// /// Refresh session epoch and handle checkpointing phases. Used only diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/IUnsafeContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/IUnsafeContext.cs index 8e1ef26f327..9b4402561f7 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/IUnsafeContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/IUnsafeContext.cs @@ -5,8 +5,8 @@ namespace Tsavorite.core { /// /// Manual epoch control functions. Useful when doing generic operations across diverse - /// and - /// specializations. + /// and + /// specializations. /// public interface IUnsafeContext { diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/LockableUnsafeContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/LockableUnsafeContext.cs deleted file mode 100644 index 638ceca9ddc..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/LockableUnsafeContext.cs +++ /dev/null @@ -1,505 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; - -namespace Tsavorite.core -{ - /// - /// Tsavorite Context implementation that allows manual control of record locking and epoch management. For advanced use only. - /// - public readonly struct LockableUnsafeContext - : ITsavoriteContext, ILockableContext, IUnsafeContext - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - { - readonly ClientSession clientSession; - readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; - - /// - public bool IsNull => clientSession is null; - - internal LockableUnsafeContext(ClientSession clientSession) - { - this.clientSession = clientSession; - sessionFunctions = new(clientSession); - } - - #region Begin/EndUnsafe - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void BeginUnsafe() => clientSession.UnsafeResumeThread(sessionFunctions); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void EndUnsafe() => clientSession.UnsafeSuspendThread(); - - #endregion Begin/EndUnsafe - - #region Begin/EndLockable - - /// - public void BeginLockable() => clientSession.AcquireLockable(sessionFunctions); - - /// - public void LocksAcquired(long txnVersion) => clientSession.LocksAcquired(sessionFunctions, txnVersion); - - /// - public void EndLockable() => clientSession.ReleaseLockable(sessionFunctions); - #endregion Begin/EndLockable - - #region Key Locking - - /// - public int CompareKeyHashes(TLockableKey key1, TLockableKey key2) where TLockableKey : ILockableKey => clientSession.CompareKeyHashes(key1, key2); - - /// - public int CompareKeyHashes(ref TLockableKey key1, ref TLockableKey key2) where TLockableKey : ILockableKey => clientSession.CompareKeyHashes(ref key1, ref key2); - - /// - public void SortKeyHashes(Span keys) where TLockableKey : ILockableKey => clientSession.SortKeyHashes(keys); - - /// - public void Lock(ReadOnlySpan keys) - where TLockableKey : ILockableKey - { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for LockableUnsafeContext.Lock()"); - while (true) - { - if (LockableContext.DoManualLock(sessionFunctions, clientSession, keys)) - { - break; - } - // Suspend and resume epoch protection to give others a fair chance to progress - clientSession.store.epoch.Suspend(); - clientSession.store.epoch.Resume(); - } - } - - /// - public bool TryLock(ReadOnlySpan keys) - where TLockableKey : ILockableKey - => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken: default); - - /// - public bool TryLock(ReadOnlySpan keys, TimeSpan timeout) - where TLockableKey : ILockableKey - => TryLock(keys, timeout, cancellationToken: default); - - /// - public bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) - where TLockableKey : ILockableKey - => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken); - - /// - public bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey - { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for LockableUnsafeContext.Lock()"); - - return LockableContext.DoManualTryLock(sessionFunctions, clientSession, keys, timeout, cancellationToken); - } - - /// - public bool TryPromoteLock(TLockableKey key) - where TLockableKey : ILockableKey - => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken: default); - - /// - public bool TryPromoteLock(TLockableKey key, TimeSpan timeout) - where TLockableKey : ILockableKey - => TryPromoteLock(key, timeout, cancellationToken: default); - - /// - public bool TryPromoteLock(TLockableKey key, CancellationToken cancellationToken) - where TLockableKey : ILockableKey - => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken); - - /// - public bool TryPromoteLock(TLockableKey key, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey - { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for LockableUnsafeContext.Lock()"); - - return LockableContext.DoManualTryPromoteLock(sessionFunctions, clientSession, key, timeout, cancellationToken); - } - - /// - public void Unlock(ReadOnlySpan keys) - where TLockableKey : ILockableKey - { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for LockableUnsafeContext.Unlock()"); - - LockableContext.DoManualUnlock(clientSession, keys); - } - - /// - /// The id of the current Tsavorite Session - /// - public int SessionID { get { return clientSession.ctx.sessionID; } } - - #endregion Key Locking - - #region ITsavoriteContext - - /// - public ClientSession Session => clientSession; - - /// - public long GetKeyHash(TKey key) => clientSession.store.GetKeyHash(ref key); - - /// - public long GetKeyHash(ref TKey key) => clientSession.store.GetKeyHash(ref key); - - /// - public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.UnsafeCompletePending(sessionFunctions, false, wait, spinWaitForCommit); - } - - /// - public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.UnsafeCompletePendingWithOutputs(sessionFunctions, out completedOutputs, wait, spinWaitForCommit); - } - - /// - public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationToken token = default) - => clientSession.CompletePendingAsync(sessionFunctions, waitForCommit, token); - - /// - public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) - => clientSession.CompletePendingWithOutputsAsync(sessionFunctions, waitForCommit, token); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, ref readOptions, out _, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, TContext userContext = default) - { - TInput input = default; - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (Status status, TOutput output) Read(TKey key, TContext userContext = default) - { - TInput input = default; - TOutput output = default; - return (Read(ref key, ref input, ref output, userContext), output); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public (Status status, TOutput output) Read(TKey key, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - TOutput output = default; - return (Read(ref key, ref input, ref output, ref readOptions, userContext), output); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ReadWithPrefetch(ref TBatch batch, TContext userContext) - where TBatch : IReadArgBatch -#if NET9_0_OR_GREATER - , allows ref struct -#endif - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextReadAtAddress(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, TContext userContext = default) - { - TInput input = default; - TOutput output = default; - return Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) - { - TInput input = default; - TOutput output = default; - return Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, TContext userContext = default) - => Upsert(ref key, ref desiredValue, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref desiredValue, ref upsertOptions, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, ref upsertOptions, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) - => RMW(ref key, ref input, ref output, out _, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out _, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRMW(ref key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, ref RMWOptions rmwOptions, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, TContext userContext = default) - => Delete(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); - - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, long keyHash, TContext userContext = default) - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( - ref key, keyHash, userContext, sessionFunctions); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, TContext userContext = default) - => Delete(ref key, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, ref deleteOptions, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetModified(ref TKey key) - => clientSession.UnsafeResetModified(sessionFunctions, ref key); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool IsModified(TKey key) - => clientSession.UnsafeIsModified(sessionFunctions, ref key); - - /// - public void Refresh() - { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); - } - - #endregion ITsavoriteContext - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/ManageClientSessions.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/ManageClientSessions.cs index c1f38ab769e..c12726c7e1d 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/ManageClientSessions.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/ManageClientSessions.cs @@ -7,20 +7,28 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - internal Dictionary _activeSessions = new(); + internal Dictionary _activeSessions = []; /// /// Start a new client session with Tsavorite. /// /// Callback functions + /// Enable consistent read context /// for this session; override those specified at TsavoriteKV level, and may be overridden on individual Read operations /// Session instance - public ClientSession NewSession(TFunctions functions, ReadCopyOptions readCopyOptions = default) - where TFunctions : ISessionFunctions + public ClientSession NewSession( + TFunctions functions, + bool enableConsistentRead = false, + ReadCopyOptions readCopyOptions = default) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions { if (functions == null) throw new ArgumentNullException(nameof(functions)); @@ -29,15 +37,14 @@ public ClientSession(sessionID); ctx.MergeReadCopyOptions(ReadCopyOptions, readCopyOptions); - var session = new ClientSession(this, ctx, functions); if (RevivificationManager.IsEnabled) { if (_activeSessions == null) _ = Interlocked.CompareExchange(ref _activeSessions, [], null); - - lock (_activeSessions) - _activeSessions.Add(sessionID, new SessionInfo { session = session, isActive = true }); } + var session = new ClientSession(this, ctx, functions, enableConsistentRead); + lock (_activeSessions) + _activeSessions.Add(sessionID, new SessionInfo { session = session, isActive = true }); return session; } @@ -74,7 +81,6 @@ public string DumpRevivificationStats() // Merge the session-level stats into the global stats, clear the session-level stats, and keep the cumulative stats. foreach (var sessionInfo in _activeSessions.Values) sessionInfo.session.MergeRevivificationStatsTo(ref RevivificationManager.stats, reset: true); - } } return RevivificationManager.stats.Dump(); diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/NoOpSessionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/NoOpSessionFunctions.cs new file mode 100644 index 00000000000..39827d92fd7 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/NoOpSessionFunctions.cs @@ -0,0 +1,174 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Tsavorite.core +{ + /// + /// This implementation of is used during compaction, iteration, and any other + /// operations that require a session for ContinuePending but do operations directly on the rather than calling + /// methods for record operations (Delete methods simply return true to let + /// Tsavorite proceed with the delete). + /// + /// + /// Because this is used for copy operations, the , + /// , and + /// , and + /// methods are implemented to allow for copy of log records via Upsert, but no other methods are implemented. + /// + /// + /// + /// + internal struct NoOpSessionFunctions : ISessionFunctions + { + public readonly bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) => true; + + public readonly void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { } + + public readonly bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) => true; + + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + => throw new NotImplementedException("InPlaceWriter(ReadOnlySpan value) is not supported in this ISessionFunctions implementation"); + + public readonly bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + => throw new NotImplementedException("InPlaceWriter(IHeapObject value) is not supported in this ISessionFunctions implementation"); + + public readonly bool InPlaceWriter(ref LogRecord dstLogRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + // This includes ETag and Expiration + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(key: dstLogRecord, inputLogRecord, ref input) }; + dstLogRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + + public readonly bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + public readonly bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + public readonly bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; + public readonly void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) { } + + public readonly bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; + + public readonly bool NeedInitialUpdate(TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => true; + + public readonly bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + public readonly void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } + + public readonly void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } + + public readonly RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TInput input) + where TSourceLogRecord : ISourceLogRecord + => throw new NotImplementedException("GetRMWModifiedFieldInfo is not supported in this ISessionFunctions implementation"); + public readonly RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("GetRMWInitialFieldInfo is not supported in this ISessionFunctions implementation"); + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("GetUpsertFieldInfo(ReadOnlySpan value) is not supported in this ISessionFunctions implementation"); + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("IHeapObject value) is not supported in this ISessionFunctions implementation"); + public readonly RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + // TODO: Namespace! + => new() { KeySize = key.KeyBytes.Length, ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, ValueIsObject = inputLogRecord.Info.ValueIsObject }; + + /// + /// No reads during compaction + /// + public readonly bool Reader(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput dst, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + + /// + /// Write compacted live value to store + /// + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) => true; + + /// + /// Write compacted live value to store + /// + public readonly bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) => true; + + public readonly bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + // This includes ETag and Expiration + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) { } + public readonly void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) { } + public readonly void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { } + + public readonly void PostUpsertOperation(TKey key, ref TInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostUpsertOperation(TKey key, ref TInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostRMWOperation(TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public readonly void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + + public readonly void ConvertOutputToHeap(ref TInput input, ref TOutput output) { } + + public readonly void BeforeConsistentReadCallback(long hash) { } + + public readonly void AfterConsistentReadKeyCallback() { } + + /// + public readonly void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) { } + + public readonly bool AfterConsistentReadKeyBatchCallback(int keyCount) => true; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs index 1ea05abf472..1a97c80981b 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/SessionFunctionsWrapper.cs @@ -1,120 +1,180 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Runtime.CompilerServices; namespace Tsavorite.core { - internal readonly struct SessionFunctionsWrapper - : ISessionFunctionsWrapper - where TFunctions : ISessionFunctions - where TSessionLocker : struct, ISessionLocker - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal readonly struct SessionFunctionsWrapper + : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TSessionLocker : struct, ISessionLocker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - private readonly ClientSession _clientSession; + private readonly ClientSession _clientSession; private readonly TSessionLocker _sessionLocker; // Has no data members - public SessionFunctionsWrapper(ClientSession clientSession) + public SessionFunctionsWrapper(ClientSession clientSession) { _clientSession = clientSession; _sessionLocker = new TSessionLocker(); } - public TsavoriteKV Store => _clientSession.store; - public OverflowBucketLockTable LockTable => _clientSession.store.LockTable; + public TsavoriteKV Store => _clientSession.store; + public OverflowBucketLockTable LockTable => _clientSession.store.LockTable; #region Reads [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo) - => _clientSession.functions.SingleReader(ref key, ref input, ref value, ref dst, ref readInfo); + public bool Reader(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput dst, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.Reader(in srcLogRecord, ref input, ref dst, ref readInfo); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => _clientSession.functions.ConcurrentReader(ref key, ref input, ref value, ref dst, ref readInfo, ref recordInfo); - - public void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) - => _clientSession.functions.ReadCompletionCallback(ref key, ref input, ref output, ctx, status, recordMetadata); + public void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) + => _clientSession.functions.ReadCompletionCallback(ref diskLogRecord, ref input, ref output, ctx, status, recordMetadata); #endregion Reads #region Upserts [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => _clientSession.functions.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); + public bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + => _clientSession.functions.InitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + => _clientSession.functions.InitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, in inputLogRecord, ref output, ref upsertInfo); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + logRecord.InfoRef.SetModified(); + _clientSession.functions.PostInitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) { - recordInfo.SetDirtyAndModified(); - _clientSession.functions.PostSingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason); + logRecord.InfoRef.SetModified(); + _clientSession.functions.PostInitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool ConcurrentWriter(long physicalAddress, ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public readonly void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord { - (upsertInfo.UsedValueLength, upsertInfo.FullValueLength, _) = _clientSession.store.GetRecordLengths(physicalAddress, ref dst, ref recordInfo); - if (!_clientSession.functions.ConcurrentWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, ref recordInfo)) + dstLogRecord.InfoRef.SetModified(); + _clientSession.functions.PostInitialWriter(ref dstLogRecord, in sizeInfo, ref input, in inputLogRecord, ref output, ref upsertInfo); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo)) return false; - _clientSession.store.SetExtraValueLength(ref dst, ref recordInfo, upsertInfo.UsedValueLength, upsertInfo.FullValueLength); - recordInfo.SetDirtyAndModified(); + logRecord.InfoRef.SetModified(); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + public bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo)) + return false; + logRecord.InfoRef.SetModified(); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + if (!_clientSession.functions.InPlaceWriter(ref logRecord, ref input, in inputLogRecord, ref output, ref upsertInfo)) + return false; + logRecord.InfoRef.SetModified(); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostUpsertOperation(TOpKey key, ref TInput input, ReadOnlySpan srcValueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + => _clientSession.functions.PostUpsertOperation(key, ref input, srcValueSpan, ref upsertInfo, epochAccessor); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void PostUpsertOperation(TOpKey key, ref TInput input, IHeapObject srcValueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor - => _clientSession.functions.PostUpsertOperation(ref key, ref input, ref src, ref upsertInfo, epochAccessor); + => _clientSession.functions.PostUpsertOperation(key, ref input, srcValueObject, ref upsertInfo, epochAccessor); #endregion Upserts #region RMWs #region InitialUpdater [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) - => _clientSession.functions.NeedInitialUpdate(ref key, ref input, ref output, ref rmwInfo); + public bool NeedInitialUpdate(TOpKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _clientSession.functions.NeedInitialUpdate(key, ref input, ref output, ref rmwInfo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => _clientSession.functions.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + public bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + => _clientSession.functions.InitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) { - recordInfo.SetDirtyAndModified(); - _clientSession.functions.PostInitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo); + logRecord.InfoRef.SetModified(); + _clientSession.functions.PostInitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); } #endregion InitialUpdater #region CopyUpdater [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo) - => _clientSession.functions.NeedCopyUpdate(ref key, ref input, ref oldValue, ref output, ref rmwInfo); + public bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.NeedCopyUpdate(in srcLogRecord, ref input, ref output, ref rmwInfo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => _clientSession.functions.CopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo, ref recordInfo); + public bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.CopyUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord { - recordInfo.SetDirtyAndModified(); - return _clientSession.functions.PostCopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo); + dstLogRecord.InfoRef.SetModified(); + return _clientSession.functions.PostCopyUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); } #endregion CopyUpdater #region InPlaceUpdater [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool InPlaceUpdater(long physicalAddress, ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status, ref RecordInfo recordInfo) + public bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status) { - (rmwInfo.UsedValueLength, rmwInfo.FullValueLength, rmwInfo.FullRecordLength) = _clientSession.store.GetRecordLengths(physicalAddress, ref value, ref recordInfo); - - if (_clientSession.functions.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo)) + // This wraps the ISessionFunctions call to provide expiration logic. + if (_clientSession.functions.InPlaceUpdater(ref logRecord, ref input, ref output, ref rmwInfo)) { rmwInfo.Action = RMWAction.Default; - _clientSession.store.SetExtraValueLength(ref value, ref recordInfo, rmwInfo.UsedValueLength, rmwInfo.FullValueLength); - recordInfo.SetDirtyAndModified(); + logRecord.InfoRef.SetModified(); - // MarkPage is done in InternalRMW status = OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.InPlaceUpdatedRecord); return true; } @@ -122,8 +182,8 @@ public bool InPlaceUpdater(long physicalAddress, ref TKey key, ref TInput input, if (rmwInfo.Action == RMWAction.ExpireAndResume) { // This inserts the tombstone if appropriate - return _clientSession.store.ReinitializeExpiredRecord>( - ref key, ref input, ref value, ref output, ref recordInfo, ref rmwInfo, rmwInfo.Address, this, isIpu: true, out status); + return _clientSession.store.ReinitializeExpiredRecord>( + ref logRecord, ref input, ref output, ref rmwInfo, rmwInfo.Address, this, isIpu: true, out status); } if (rmwInfo.Action == RMWAction.CancelOperation) @@ -132,9 +192,16 @@ public bool InPlaceUpdater(long physicalAddress, ref TKey key, ref TInput input, } else if (rmwInfo.Action == RMWAction.ExpireAndStop) { - recordInfo.SetTombstone(); + // Tombstone is set by the caller (InternalRMW) AFTER OnDispose, + // so that internal heap accounting reads the pre-tombstone value size. status = OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.InPlaceUpdatedRecord | StatusCode.Expired); } + else if (rmwInfo.Action == RMWAction.WrongType) + { + // WrongType means the operation was rejected — the record must NOT be modified. + // Do not set Tombstone; the key should remain intact for correct-type operations. + status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.WrongType); + } else status = OperationStatus.SUCCESS; return false; @@ -142,42 +209,48 @@ public bool InPlaceUpdater(long physicalAddress, ref TKey key, ref TInput input, #endregion InPlaceUpdater [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + public void PostRMWOperation(TOpKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor - => _clientSession.functions.PostRMWOperation(ref key, ref input, ref rmwInfo, epochAccessor); + => _clientSession.functions.PostRMWOperation(key, ref input, ref rmwInfo, epochAccessor); - public void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) - => _clientSession.functions.RMWCompletionCallback(ref key, ref input, ref output, ctx, status, recordMetadata); + public void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) + => _clientSession.functions.RMWCompletionCallback(ref diskLogRecord, ref input, ref output, ctx, status, recordMetadata); #endregion RMWs #region Deletes [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) - => _clientSession.functions.SingleDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + public bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + => _clientSession.functions.InitialDeleter(ref logRecord, ref deleteInfo); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - recordInfo.SetDirtyAndModified(); - _clientSession.functions.PostSingleDeleter(ref key, ref deleteInfo); + logRecord.InfoRef.SetModified(); + _clientSession.functions.PostInitialDeleter(ref logRecord, ref deleteInfo); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool ConcurrentDeleter(long physicalAddress, ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo, out int allocatedSize) + public bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { - (deleteInfo.UsedValueLength, deleteInfo.FullValueLength, allocatedSize) = _clientSession.store.GetRecordLengths(physicalAddress, ref value, ref recordInfo); - if (!_clientSession.functions.ConcurrentDeleter(ref key, ref value, ref deleteInfo, ref recordInfo)) + if (!_clientSession.functions.InPlaceDeleter(ref logRecord, ref deleteInfo)) return false; - _clientSession.store.SetTombstoneAndExtraValueLength(ref value, ref recordInfo, deleteInfo.UsedValueLength, deleteInfo.FullValueLength); - recordInfo.SetDirtyAndModified(); + // Tombstone and Modified are set by the caller (InternalDelete) AFTER + // OnDispose, so that internal heap accounting reads the pre-tombstone value size. return true; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + public void PostDeleteOperation(TOpKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor - => _clientSession.functions.PostDeleteOperation(ref key, ref deleteInfo, epochAccessor); + => _clientSession.functions.PostDeleteOperation(key, ref deleteInfo, epochAccessor); #endregion Deletes #region Utilities @@ -185,43 +258,64 @@ public void PostDeleteOperation(ref TKey key, ref DeleteInfo del public void ConvertOutputToHeap(ref TInput input, ref TOutput output) => _clientSession.functions.ConvertOutputToHeap(ref input, ref output); #endregion Utilities - #region Transient locking - public bool IsManualLocking => _sessionLocker.IsManualLocking; + #region Ephemeral locking + public bool IsTransactionalLocking => _sessionLocker.IsTransactionalLocking; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientExclusive(ref TKey key, ref OperationStackContext stackCtx) => - _sessionLocker.TryLockTransientExclusive(Store, ref stackCtx); + public bool TryLockEphemeralExclusive(ref OperationStackContext stackCtx) => + _sessionLocker.TryLockEphemeralExclusive(Store, ref stackCtx); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientShared(ref TKey key, ref OperationStackContext stackCtx) - => _sessionLocker.TryLockTransientShared(Store, ref stackCtx); + public bool TryLockEphemeralShared(ref OperationStackContext stackCtx) + => _sessionLocker.TryLockEphemeralShared(Store, ref stackCtx); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientExclusive(ref TKey key, ref OperationStackContext stackCtx) - => _sessionLocker.UnlockTransientExclusive(Store, ref stackCtx); + public void UnlockEphemeralExclusive(ref OperationStackContext stackCtx) + => _sessionLocker.UnlockEphemeralExclusive(Store, ref stackCtx); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientShared(ref TKey key, ref OperationStackContext stackCtx) - => _sessionLocker.UnlockTransientShared(Store, ref stackCtx); - #endregion Transient locking + public void UnlockEphemeralShared(ref OperationStackContext stackCtx) + => _sessionLocker.UnlockEphemeralShared(Store, ref stackCtx); + #endregion Ephemeral locking #region Internal utilities [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetRMWInitialValueLength(ref TInput input) => _clientSession.functions.GetRMWInitialValueLength(ref input); + public RecordFieldInfo GetRMWInitialFieldInfo(TOpKey key, ref TInput input) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _clientSession.functions.GetRMWInitialFieldInfo(key, ref input); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetRMWModifiedValueLength(ref TValue t, ref TInput input) => _clientSession.functions.GetRMWModifiedValueLength(ref t, ref input); + public RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TInput input) + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.GetRMWModifiedFieldInfo(in srcLogRecord, ref input); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int GetUpsertValueLength(ref TValue t, ref TInput input) => _clientSession.functions.GetUpsertValueLength(ref t, ref input); + public RecordFieldInfo GetUpsertFieldInfo(TOpKey key, ReadOnlySpan value, ref TInput input) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _clientSession.functions.GetUpsertFieldInfo(key, value, ref input); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public IHeapContainer GetHeapContainer(ref TInput input) - { - if (typeof(TInput) == typeof(SpanByte)) - return new SpanByteHeapContainer(ref Unsafe.As(ref input), _clientSession.store.hlogBase.bufferPool) as IHeapContainer; - return new StandardHeapContainer(ref input); - } + public RecordFieldInfo GetUpsertFieldInfo(TOpKey key, IHeapObject value, ref TInput input) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => _clientSession.functions.GetUpsertFieldInfo(key, value, ref input); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly RecordFieldInfo GetUpsertFieldInfo(TOpKey key, in TSourceLogRecord inputLogRecord, ref TInput input) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + => _clientSession.functions.GetUpsertFieldInfo(key, in inputLogRecord, ref input); [MethodImpl(MethodImplOptions.AggressiveInlining)] public void UnsafeResumeThread() => _clientSession.UnsafeResumeThread(this); @@ -230,10 +324,10 @@ public IHeapContainer GetHeapContainer(ref TInput input) public void UnsafeSuspendThread() => _clientSession.UnsafeSuspendThread(); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) => _clientSession.CompletePendingWithOutputs(this, out completedOutputs, wait, spinWaitForCommit); - public TsavoriteKV.TsavoriteExecutionContext Ctx => _clientSession.ctx; + public TsavoriteKV.TsavoriteExecutionContext Ctx => _clientSession.ctx; #endregion Internal utilities } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalConsistentReadContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalConsistentReadContext.cs new file mode 100644 index 00000000000..1899cb26b8d --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalConsistentReadContext.cs @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + /// + /// Transactional consistent read context that extends transactionalContext functionality with consistent read protocols. + /// + public readonly struct TransactionalConsistentReadContext + : ITsavoriteContext, ITransactionalContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + public readonly TransactionalContext TransactionalContext { get; } + + /// + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Session.store.GetKeyHash(key); + + /// + public bool IsNull => TransactionalContext.IsNull; + + internal TransactionalConsistentReadContext(ClientSession clientSession) + { + TransactionalContext = new TransactionalContext(clientSession); + } + + #region Begin/EndTransaction + + /// + public void BeginTransaction() => TransactionalContext.BeginTransaction(); + + /// + public void LocksAcquired(long txnVersion) => TransactionalContext.LocksAcquired(txnVersion); + + /// + public void EndTransaction() => TransactionalContext.EndTransaction(); + + #endregion Begin/EndTransaction + + #region Key Locking + + /// + public int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) where TTransactionalKey : ITransactionalKey + => TransactionalContext.CompareKeyHashes(key1, key2); + + /// + public int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) where TTransactionalKey : ITransactionalKey + => TransactionalContext.CompareKeyHashes(ref key1, ref key2); + + /// + public void SortKeyHashes(Span keys) where TTransactionalKey : ITransactionalKey + => TransactionalContext.SortKeyHashes(keys); + + /// + public void Lock(ReadOnlySpan keys) where TTransactionalKey : ITransactionalKey + => TransactionalContext.Lock(keys); + + /// + public bool TryLock(ReadOnlySpan keys) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryLock(keys); + + /// + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryLock(keys, timeout); + + /// + public bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryLock(keys, cancellationToken); + + /// + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryLock(keys, timeout, cancellationToken); + + /// + public bool TryPromoteLock(TTransactionalKey key) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryPromoteLock(key); + + /// + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryPromoteLock(key, timeout); + + /// + public bool TryPromoteLock(TTransactionalKey key, CancellationToken cancellationToken) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryPromoteLock(key, cancellationToken); + + /// + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout, CancellationToken cancellationToken) where TTransactionalKey : ITransactionalKey + => TransactionalContext.TryPromoteLock(key, timeout, cancellationToken); + + /// + public void Unlock(ReadOnlySpan keys) where TTransactionalKey : ITransactionalKey + => TransactionalContext.Unlock(keys); + + /// + /// The id of the current Tsavorite Session + /// + public int SessionID => TransactionalContext.SessionID; + + #endregion Key Locking + + #region ITsavoriteContext/Read + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + { + var hash = GetKeyHash(key); + Session.functions.BeforeConsistentReadCallback(hash); + var status = TransactionalContext.Read(key, ref input, ref output, userContext); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + => Read(key, ref input, ref output, ref readOptions, out _, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, ref readOptions, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, ref readOptions, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var hash = GetKeyHash(key); + Session.functions.BeforeConsistentReadCallback(hash); + var status = TransactionalContext.Read(key, ref input, ref output, ref readOptions, out recordMetadata, userContext); + Session.functions.AfterConsistentReadKeyCallback(); + return status; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow reads from address!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow reads from address!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReadWithPrefetch(ref TBatch batch, TContext userContext = default) + where TBatch : IReadArgBatch +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + do + { + Thread.Yield(); + Session.functions.BeforeConsistentReadKeyBatchCallback(batch.Parameters); + TransactionalContext.ReadWithPrefetch(ref batch, userContext); + } while (!Session.functions.AfterConsistentReadKeyBatchCallback(batch.Count)); + } + + #endregion Read Methods (To be overridden with custom logic) + + #region ITsavoriteContext + + /// + public ClientSession Session => TransactionalContext.Session; + + /// + public long GetKeyHash(TKey key) => TransactionalContext.GetKeyHash(key); + + /// + public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) + => TransactionalContext.CompletePending(wait, spinWaitForCommit); + + /// + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + => TransactionalContext.CompletePendingWithOutputs(out completedOutputs, wait, spinWaitForCommit); + + /// + public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationToken token = default) + => TransactionalContext.CompletePendingAsync(waitForCommit, token); + + /// + public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + => TransactionalContext.CompletePendingWithOutputsAsync(waitForCommit, token); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) + => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord => throw new TsavoriteException("Consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, in TSourceLogRecord diskLogRecord) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) where TSourceLogRecord : ISourceLogRecord + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Delete(TKey key, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) + => throw new TsavoriteException("Transactional consistent read context does not allow writes!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ResetModified(TKey key) + => throw new TsavoriteException("Transactional consistent read context does not reset ResetModified!"); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsModified(TKey key) + => throw new TsavoriteException("Transactional consistent read context does not allow IsModified!"); + + /// + public void Refresh() + => throw new TsavoriteException("Transactional consistent read context does not Refresh!"); + + #endregion ITsavoriteContext + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/LockableContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalContext.cs similarity index 51% rename from libs/storage/Tsavorite/cs/src/core/ClientSession/LockableContext.cs rename to libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalContext.cs index 43368ad5c07..a5daa347a51 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/LockableContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalContext.cs @@ -10,56 +10,60 @@ namespace Tsavorite.core { /// - /// Tsavorite Context implementation that allows manual control of record locking and epoch management. For advanced use only. + /// Tsavorite Context implementation that allows Transactional control of locking and automatic epoch management. For advanced use only. /// - public readonly struct LockableContext : ITsavoriteContext, ILockableContext - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public readonly struct TransactionalContext : ITsavoriteContext, ITransactionalContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - readonly ClientSession clientSession; - readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; + readonly ClientSession clientSession; + readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; /// public bool IsNull => clientSession is null; const int KeyLockMaxRetryAttempts = 1000; - internal LockableContext(ClientSession clientSession) + internal TransactionalContext(ClientSession clientSession) { this.clientSession = clientSession; sessionFunctions = new(clientSession); } - #region Begin/EndLockable + #region Begin/EndTransaction /// - public void BeginLockable() => clientSession.AcquireLockable(sessionFunctions); + public void BeginTransaction() => clientSession.AcquireTransactional(sessionFunctions); /// public void LocksAcquired(long txnVersion) => clientSession.LocksAcquired(sessionFunctions, txnVersion); /// - public void EndLockable() => clientSession.ReleaseLockable(sessionFunctions); + public void EndTransaction() => clientSession.ReleaseTransactional(sessionFunctions); - #endregion Begin/EndLockable + #endregion Begin/EndTransaction #region Key Locking /// - public int CompareKeyHashes(TLockableKey key1, TLockableKey key2) where TLockableKey : ILockableKey => clientSession.CompareKeyHashes(ref key1, ref key2); + public int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => clientSession.CompareKeyHashes(ref key1, ref key2); /// - public int CompareKeyHashes(ref TLockableKey key1, ref TLockableKey key2) where TLockableKey : ILockableKey => clientSession.CompareKeyHashes(ref key1, ref key2); + public int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => clientSession.CompareKeyHashes(ref key1, ref key2); /// - public void SortKeyHashes(Span keys) where TLockableKey : ILockableKey => clientSession.SortKeyHashes(keys); + public void SortKeyHashes(Span keys) where TTransactionalKey : ITransactionalKey => clientSession.SortKeyHashes(keys); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool DoManualLock(TSessionFunctionsWrapper sessionFunctions, ClientSession clientSession, - ReadOnlySpan keys) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - where TLockableKey : ILockableKey + internal static bool DoTransactionalLock(TSessionFunctionsWrapper sessionFunctions, + ClientSession clientSession, ReadOnlySpan keys) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TTransactionalKey : ITransactionalKey { // The key codes are sorted, but there may be duplicates; the sorting is such that exclusive locks come first for each key code, // which of course allows the session to do shared operations as well, so we take the first occurrence of each key code. @@ -76,12 +80,12 @@ internal static bool DoManualLock(TSessi if (currBucketIndex != prevBucketIndex) { prevBucketIndex = currBucketIndex; - OperationStatus status = DoManualLock(clientSession, key); + var status = DoTransactionalLock(clientSession, key); if (status == OperationStatus.SUCCESS) continue; // Success; continue to the next key. // Lock failure before we've completed all keys, and we did not lock the current key. Unlock anything we've locked. - DoManualUnlock(clientSession, keys[..keyIdx]); + DoTransactionalUnlock(clientSession, keys[..keyIdx]); // We've released our locks so this refresh will let other threads advance and release their locks, and we will retry with a full timeout. _ = clientSession.store.HandleImmediateNonPendingRetryStatus(status, sessionFunctions); @@ -97,19 +101,20 @@ internal static bool DoManualLock(TSessi } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool DoManualTryLock(TSessionFunctionsWrapper sessionFunctions, ClientSession clientSession, - ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - where TLockableKey : ILockableKey + internal static bool DoTransactionalTryLock(TSessionFunctionsWrapper sessionFunctions, + ClientSession clientSession, + ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TTransactionalKey : ITransactionalKey { // The key codes are sorted, but there may be duplicates; the sorting is such that exclusive locks come first for each key code, // which of course allows the session to do shared operations as well, so we take the first occurrence of each key code. - // This is the same as DoManualLock but with timeout. + // This is the same as DoTransactionalLock but with timeout. // We can't start each retry with a full timeout because we might always fail if someone is not unlocking (e.g. another thread hangs // somehow while holding a lock, or the current thread has issued two lock calls on two key sets and the second tries to lock one in // the first, and so on). So set the timeout high enough to accommodate as many retries as you want. - var startTime = DateTime.UtcNow; + var startTimestamp = Stopwatch.GetTimestamp(); Retry: var prevBucketIndex = -1L; @@ -127,16 +132,17 @@ internal static bool DoManualTryLock(TSe status = OperationStatus.CANCELED; else { - status = DoManualLock(clientSession, key); + status = DoTransactionalLock(clientSession, key); if (status == OperationStatus.SUCCESS) continue; // Success; continue to the next key. } // Cancellation or lock failure before we've completed all keys; we have not locked the current key. Unlock anything we've locked. - DoManualUnlock(clientSession, keys[..keyIdx]); + DoTransactionalUnlock(clientSession, keys[..keyIdx]); // Lock failure is the only place we check the timeout. If we've exceeded that, or if we've had a cancellation, return false. - if (cancellationToken.IsCancellationRequested || DateTime.UtcNow.Ticks - startTime.Ticks > timeout.Ticks) + // A negative timeout (e.g. Timeout.InfiniteTimeSpan) means wait indefinitely until cancellation. + if (cancellationToken.IsCancellationRequested || (timeout >= TimeSpan.Zero && Stopwatch.GetElapsedTime(startTimestamp) > timeout)) return false; // No cancellation and we're within the timeout. We've released our locks so this refresh will let other threads advance @@ -151,12 +157,12 @@ internal static bool DoManualTryLock(TSe } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool DoManualTryPromoteLock(TSessionFunctionsWrapper sessionFunctions, ClientSession clientSession, - TLockableKey key, TimeSpan timeout, CancellationToken cancellationToken) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - where TLockableKey : ILockableKey + internal static bool DoManualTryPromoteLock(TSessionFunctionsWrapper sessionFunctions, ClientSession clientSession, + TTransactionalKey key, TimeSpan timeout, CancellationToken cancellationToken) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TTransactionalKey : ITransactionalKey { - var startTime = DateTime.UtcNow; + var startTimestamp = Stopwatch.GetTimestamp(); while (true) { if (clientSession.store.InternalPromoteLock(key.KeyHash)) @@ -164,12 +170,13 @@ internal static bool DoManualTryPromoteLock timeout.Ticks) + // CancellationToken can accompany either of the other two mechanisms. + // A negative timeout (e.g. Timeout.InfiniteTimeSpan) means wait indefinitely until cancellation. + if (cancellationToken.IsCancellationRequested || (timeout >= TimeSpan.Zero && Stopwatch.GetElapsedTime(startTimestamp) > timeout)) break; // out of the retry loop // Lock failed, must retry @@ -182,8 +189,8 @@ internal static bool DoManualTryPromoteLock(ClientSession clientSession, TLockableKey key) - where TLockableKey : ILockableKey + internal static OperationStatus DoTransactionalLock(ClientSession clientSession, TTransactionalKey key) + where TTransactionalKey : ITransactionalKey { if (key.LockType == LockType.Shared) { @@ -201,9 +208,9 @@ internal static OperationStatus DoManualLock(ClientSession(ClientSession clientSession, - ReadOnlySpan keys) - where TLockableKey : ILockableKey + internal static void DoTransactionalUnlock(ClientSession clientSession, + ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey { // The key codes are sorted, but there may be duplicates; the sorting is such that exclusive locks come first for each key code. // Unlock has to be done in the reverse order of locking, so we take the *last* occurrence of each key there, and keyIdx moves backward. @@ -227,18 +234,18 @@ internal static void DoManualUnlock(ClientSession - public void Lock(ReadOnlySpan keys) - where TLockableKey : ILockableKey + public void Lock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for LockableUnsafeContext.Lock()"); + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for TransactionalUnsafeContext.Lock()"); var lockAcquired = false; while (!lockAcquired) { clientSession.UnsafeResumeThread(sessionFunctions); try { - lockAcquired = DoManualLock(sessionFunctions, clientSession, keys); + lockAcquired = DoTransactionalLock(sessionFunctions, clientSession, keys); } finally { @@ -248,31 +255,31 @@ public void Lock(ReadOnlySpan keys) } /// - public bool TryLock(ReadOnlySpan keys) - where TLockableKey : ILockableKey + public bool TryLock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken: default); /// - public bool TryLock(ReadOnlySpan keys, TimeSpan timeout) - where TLockableKey : ILockableKey + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout) + where TTransactionalKey : ITransactionalKey => TryLock(keys, timeout, cancellationToken: default); /// - public bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) - where TLockableKey : ILockableKey + public bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken); /// - public bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for LockableUnsafeContext.Lock()"); + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for TransactionalUnsafeContext.TryLock()"); clientSession.UnsafeResumeThread(sessionFunctions); try { - return DoManualTryLock(sessionFunctions, clientSession, keys, timeout, cancellationToken); + return DoTransactionalTryLock(sessionFunctions, clientSession, keys, timeout, cancellationToken); } finally { @@ -281,26 +288,26 @@ public bool TryLock(ReadOnlySpan keys, TimeSpan time } /// - public bool TryPromoteLock(TLockableKey key) - where TLockableKey : ILockableKey + public bool TryPromoteLock(TTransactionalKey key) + where TTransactionalKey : ITransactionalKey => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken: default); /// - public bool TryPromoteLock(TLockableKey key, TimeSpan timeout) - where TLockableKey : ILockableKey + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout) + where TTransactionalKey : ITransactionalKey => TryPromoteLock(key, timeout, cancellationToken: default); /// - public bool TryPromoteLock(TLockableKey key, CancellationToken cancellationToken) - where TLockableKey : ILockableKey + public bool TryPromoteLock(TTransactionalKey key, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken); /// - public bool TryPromoteLock(TLockableKey key, TimeSpan timeout, CancellationToken cancellationToken) - where TLockableKey : ILockableKey + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for LockableUnsafeContext.Lock()"); + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for TransactionalUnsafeContext.TryPromoteLock()"); clientSession.UnsafeResumeThread(sessionFunctions); try @@ -314,16 +321,16 @@ public bool TryPromoteLock(TLockableKey key, TimeSpan timeout, Can } /// - public void Unlock(ReadOnlySpan keys) - where TLockableKey : ILockableKey + public void Unlock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey { - clientSession.CheckIsAcquiredLockable(sessionFunctions); - Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for LockableUnsafeContext.Unlock()"); + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected(), "Trying to protect an already-protected epoch for TransactionalUnsafeContext.Unlock()"); clientSession.UnsafeResumeThread(sessionFunctions); try { - DoManualUnlock(clientSession, keys); + DoTransactionalUnlock(clientSession, keys); } finally { @@ -341,13 +348,15 @@ public void Unlock(ReadOnlySpan keys) #region ITsavoriteContext /// - public ClientSession Session => clientSession; - - /// - public long GetKeyHash(TKey key) => clientSession.store.GetKeyHash(ref key); + public ClientSession Session => clientSession; /// - public long GetKeyHash(ref TKey key) => clientSession.store.GetKeyHash(ref key); + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => clientSession.store.GetKeyHash(key); /// public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) @@ -365,7 +374,7 @@ public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) } /// - public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); @@ -384,18 +393,18 @@ public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationTo => clientSession.CompletePendingAsync(sessionFunctions, waitForCommit, token); /// - public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) => clientSession.CompletePendingWithOutputsAsync(sessionFunctions, waitForCommit, token); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextRead(ref key, ref input, ref output, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, userContext, sessionFunctions); } finally { @@ -405,57 +414,23 @@ public Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - => Read(ref key, ref input, ref output, ref readOptions, out _, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + => Read(key, ref input, ref output, ref readOptions, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, TContext userContext = default) { TInput input = default; - output = default; - return Read(ref key, ref input, ref output, userContext); + return Read(key, ref input, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) { TInput input = default; - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); + return Read(key, ref input, ref output, ref readOptions, userContext); } /// @@ -464,7 +439,7 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, userContext), output); + return (Read(key, ref input, ref output, userContext), output); } /// @@ -473,18 +448,18 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, ref readOptions, userContext), output); + return (Read(key, ref input, ref output, ref readOptions, userContext), output); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextRead(ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -503,7 +478,7 @@ public void ReadWithPrefetch(ref TBatch batch, TContext userContext = de clientSession.UnsafeResumeThread(sessionFunctions); try { - clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); + clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); } finally { @@ -519,7 +494,7 @@ public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextReadAtAddress(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextReadAtAddress, TStoreFunctions, TAllocator>>(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -529,13 +504,13 @@ public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextReadAtAddress(address, key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } finally { @@ -545,41 +520,41 @@ public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TO /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) + private Status Upsert(TKey key, long keyHash, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, userContext, sessionFunctions); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out _, userContext, sessionFunctions); } finally { @@ -589,23 +564,14 @@ private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue d /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); } finally { @@ -615,53 +581,122 @@ private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue d /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, TContext userContext = default) - => Upsert(ref key, ref desiredValue, userContext); + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref desiredValue, ref upsertOptions, userContext); + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, ref upsertOptions, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) - => RMW(ref key, ref input, ref output, out _, userContext); + private Status Upsert(TKey key, long keyHash, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + { + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); + clientSession.UnsafeResumeThread(sessionFunctions); + try + { + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out _, userContext, sessionFunctions); + } + finally + { + clientSession.UnsafeSuspendThread(); + } + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out _, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + clientSession.UnsafeResumeThread(sessionFunctions); + try + { + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + } + finally + { + clientSession.UnsafeSuspendThread(); + } + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); + public Status Upsert(in TSourceLogRecord diskLogRecord) + where TSourceLogRecord : ISourceLogRecord + => Upsert(diskLogRecord, in diskLogRecord); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TInput input = default; + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TSourceLogRecord : ISourceLogRecord + => Upsert(inputLogRecord, ref input, in inputLogRecord, ref output, ref upsertOptions, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextRMW(ref key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextUpsert(key, keyHash, ref input, inputLogRecord: in inputLogRecord, ref output, out _, userContext, sessionFunctions); } finally { @@ -671,72 +706,76 @@ private Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput out /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => RMW(key, ref input, ref output, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + private Status RMW(TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) { - TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); + clientSession.UnsafeResumeThread(sessionFunctions); + try + { + return clientSession.store.ContextRMW(key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); + } + finally + { + clientSession.UnsafeSuspendThread(); + } } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, TContext userContext = default) + public Status RMW(TKey key, ref TInput input, TContext userContext = default) { TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); + return RMW(key, ref input, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) { TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + return RMW(key, ref input, ref output, ref rmwOptions, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, TContext userContext = default) - => Delete(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status Delete(TKey key, TContext userContext = default) + => Delete(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) + => Delete(key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Delete(ref TKey key, long keyHash, TContext userContext = default) + private Status Delete(TKey key, long keyHash, TContext userContext = default) { Debug.Assert(!clientSession.store.epoch.ThisInstanceProtected()); clientSession.UnsafeResumeThread(sessionFunctions); try { - return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( - ref key, keyHash, userContext, sessionFunctions); + return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( + key, keyHash, userContext, sessionFunctions); } finally { @@ -746,23 +785,13 @@ private Status Delete(ref TKey key, long keyHash, TContext userContext = default /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, TContext userContext = default) - => Delete(ref key, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, ref deleteOptions, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetModified(ref TKey key) - => clientSession.ResetModified(sessionFunctions, ref key); + public void ResetModified(TKey key) + => clientSession.ResetModified(sessionFunctions, key); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool IsModified(TKey key) - => clientSession.IsModified(sessionFunctions, ref key); + => clientSession.IsModified(sessionFunctions, key); /// public void Refresh() @@ -771,7 +800,7 @@ public void Refresh() clientSession.UnsafeResumeThread(sessionFunctions); try { - clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); + clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); } finally { diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalUnsafeContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalUnsafeContext.cs new file mode 100644 index 00000000000..5dbed0aa137 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/TransactionalUnsafeContext.cs @@ -0,0 +1,504 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + /// + /// Tsavorite Context implementation that allows Transactional control of locking and manual epoch management. For advanced use only. + /// + public readonly struct TransactionalUnsafeContext + : ITsavoriteContext, ITransactionalContext, IUnsafeContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + readonly ClientSession clientSession; + readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; + + /// + public bool IsNull => clientSession is null; + + internal TransactionalUnsafeContext(ClientSession clientSession) + { + this.clientSession = clientSession; + sessionFunctions = new(clientSession); + } + + #region Begin/EndUnsafe + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void BeginUnsafe() => clientSession.UnsafeResumeThread(sessionFunctions); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void EndUnsafe() => clientSession.UnsafeSuspendThread(); + + #endregion Begin/EndUnsafe + + #region Begin/EndTransaction + + /// + public void BeginTransaction() => clientSession.AcquireTransactional(sessionFunctions); + + /// + public void LocksAcquired(long txnVersion) => clientSession.LocksAcquired(sessionFunctions, txnVersion); + + /// + public void EndTransaction() => clientSession.ReleaseTransactional(sessionFunctions); + #endregion Begin/EndTransaction + + #region Key Locking + + /// + public int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => clientSession.CompareKeyHashes(key1, key2); + + /// + public int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) where TTransactionalKey : ITransactionalKey => clientSession.CompareKeyHashes(ref key1, ref key2); + + /// + public void SortKeyHashes(Span keys) where TTransactionalKey : ITransactionalKey => clientSession.SortKeyHashes(keys); + + /// + public void Lock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey + { + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for TransactionalUnsafeContext.Lock()"); + while (true) + { + if (TransactionalContext.DoTransactionalLock(sessionFunctions, clientSession, keys)) + { + break; + } + // Suspend and resume epoch protection to give others a fair chance to progress + clientSession.store.epoch.Suspend(); + clientSession.store.epoch.Resume(); + } + } + + /// + public bool TryLock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey + => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken: default); + + /// + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout) + where TTransactionalKey : ITransactionalKey + => TryLock(keys, timeout, cancellationToken: default); + + /// + public bool TryLock(ReadOnlySpan keys, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey + => TryLock(keys, Timeout.InfiniteTimeSpan, cancellationToken); + + /// + public bool TryLock(ReadOnlySpan keys, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey + { + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for TransactionalUnsafeContext.TryLock()"); + + return TransactionalContext.DoTransactionalTryLock(sessionFunctions, clientSession, keys, timeout, cancellationToken); + } + + /// + public bool TryPromoteLock(TTransactionalKey key) + where TTransactionalKey : ITransactionalKey + => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken: default); + + /// + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout) + where TTransactionalKey : ITransactionalKey + => TryPromoteLock(key, timeout, cancellationToken: default); + + /// + public bool TryPromoteLock(TTransactionalKey key, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey + => TryPromoteLock(key, Timeout.InfiniteTimeSpan, cancellationToken); + + /// + public bool TryPromoteLock(TTransactionalKey key, TimeSpan timeout, CancellationToken cancellationToken) + where TTransactionalKey : ITransactionalKey + { + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for TransactionalUnsafeContext.TryPromoteLock()"); + + return TransactionalContext.DoManualTryPromoteLock(sessionFunctions, clientSession, key, timeout, cancellationToken); + } + + /// + public void Unlock(ReadOnlySpan keys) + where TTransactionalKey : ITransactionalKey + { + clientSession.CheckIsAcquiredTransactional(sessionFunctions); + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected(), "Epoch protection required for TransactionalUnsafeContext.Unlock()"); + + TransactionalContext.DoTransactionalUnlock(clientSession, keys); + } + + /// + /// The id of the current Tsavorite Session + /// + public int SessionID { get { return clientSession.ctx.sessionID; } } + + #endregion Key Locking + + #region ITsavoriteContext + + /// + public ClientSession Session => clientSession; + + /// + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => clientSession.store.GetKeyHash(key); + + /// + public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.UnsafeCompletePending(sessionFunctions, false, wait, spinWaitForCommit); + } + + /// + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.UnsafeCompletePendingWithOutputs(sessionFunctions, out completedOutputs, wait, spinWaitForCommit); + } + + /// + public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationToken token = default) + => clientSession.CompletePendingAsync(sessionFunctions, waitForCommit, token); + + /// + public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + => clientSession.CompletePendingWithOutputsAsync(sessionFunctions, waitForCommit, token); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextRead(key, ref input, ref output, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextRead(key, ref input, ref output, ref readOptions, out _, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + return Read(key, ref input, ref output, ref readOptions, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public (Status status, TOutput output) Read(TKey key, ref ReadOptions readOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return (Read(key, ref input, ref output, ref readOptions, userContext), output); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextRead(key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextReadAtAddress, TStoreFunctions, TAllocator>>(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ReadWithPrefetch(ref TBatch batch, TContext userContext) + where TBatch : IReadArgBatch +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextReadAtAddress(address, key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Status Upsert(TKey key, long keyHash, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out _, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private Status Upsert(TKey key, long keyHash, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out _, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(in TSourceLogRecord diskLogRecord) + where TSourceLogRecord : ISourceLogRecord + => Upsert(diskLogRecord, in diskLogRecord); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TInput input = default; + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TSourceLogRecord : ISourceLogRecord + => Upsert(inputLogRecord, ref input, in inputLogRecord, ref output, ref upsertOptions, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, inputLogRecord: in inputLogRecord, ref output, out _, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => RMW(key, ref input, ref output, out _, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out _, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextRMW(key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, TContext userContext = default) + { + TOutput output = default; + return RMW(key, ref input, ref output, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + { + TOutput output = default; + return RMW(key, ref input, ref output, ref rmwOptions, userContext); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Delete(TKey key, TContext userContext = default) + => Delete(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); + + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) + => Delete(key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status Delete(TKey key, long keyHash, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( + key, keyHash, userContext, sessionFunctions); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void ResetModified(TKey key) + => clientSession.UnsafeResetModified(sessionFunctions, key); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsModified(TKey key) + => clientSession.UnsafeIsModified(sessionFunctions, key); + + /// + public void Refresh() + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); + } + + #endregion ITsavoriteContext + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/ClientSession/UnsafeContext.cs b/libs/storage/Tsavorite/cs/src/core/ClientSession/UnsafeContext.cs index a2391b05933..0ab48793c29 100644 --- a/libs/storage/Tsavorite/cs/src/core/ClientSession/UnsafeContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/ClientSession/UnsafeContext.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Diagnostics; using System.Runtime.CompilerServices; using System.Threading; @@ -9,21 +10,25 @@ namespace Tsavorite.core { /// - /// Tsavorite Operations implementation that allows manual control of record epoch management. For advanced use only. + /// Tsavorite Operations implementation that allows manual control of epoch management. For advanced use only. /// - public readonly struct UnsafeContext - : ITsavoriteContext, IUnsafeContext - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public readonly struct UnsafeContext + : ITsavoriteContext, IUnsafeContext + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - readonly ClientSession clientSession; - internal readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; + readonly ClientSession clientSession; + internal readonly SessionFunctionsWrapper, TStoreFunctions, TAllocator> sessionFunctions; /// public bool IsNull => clientSession is null; - internal UnsafeContext(ClientSession clientSession) + internal UnsafeContext(ClientSession clientSession) { this.clientSession = clientSession; sessionFunctions = new(clientSession); @@ -44,13 +49,15 @@ internal UnsafeContext(ClientSession - public ClientSession Session => clientSession; + public ClientSession Session => clientSession; /// - public long GetKeyHash(TKey key) => clientSession.store.GetKeyHash(ref key); - - /// - public long GetKeyHash(ref TKey key) => clientSession.store.GetKeyHash(ref key); + public long GetKeyHash(TOpKey key) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => clientSession.store.GetKeyHash(key); /// public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) @@ -60,7 +67,7 @@ public bool CompletePending(bool wait = false, bool spinWaitForCommit = false) } /// - public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) + public bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); return clientSession.UnsafeCompletePendingWithOutputs(sessionFunctions, out completedOutputs, wait, spinWaitForCommit); @@ -71,73 +78,39 @@ public ValueTask CompletePendingAsync(bool waitForCommit = false, CancellationTo => clientSession.CompletePendingAsync(sessionFunctions, waitForCommit, token); /// - public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) + public ValueTask> CompletePendingWithOutputsAsync(bool waitForCommit = false, CancellationToken token = default) => clientSession.CompletePendingWithOutputsAsync(sessionFunctions, waitForCommit, token); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, ref readOptions, out _, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, ref readOptions, out _, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, TInput input, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, TContext userContext = default) { TInput input = default; - return Read(ref key, ref input, ref output, userContext); + return Read(key, ref input, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) + public Status Read(TKey key, ref TOutput output, ref ReadOptions readOptions, TContext userContext = default) { TInput input = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, TContext userContext = default) - { - TInput input = default; - output = default; - return Read(ref key, ref input, ref output, userContext); - } - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TContext userContext = default) - { - TInput input = default; - output = default; - return Read(ref key, ref input, ref output, ref readOptions, userContext); + return Read(key, ref input, ref output, ref readOptions, userContext); } /// @@ -146,7 +119,7 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, userContext), output); + return (Read(key, ref input, ref output, userContext), output); } /// @@ -155,15 +128,15 @@ public Status Read(TKey key, out TOutput output, ref ReadOptions readOptions, TC { TInput input = default; TOutput output = default; - return (Read(ref key, ref input, ref output, ref readOptions, userContext), output); + return (Read(key, ref input, ref output, ref readOptions, userContext), output); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Read(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Read(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRead(ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextRead(key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } /// @@ -175,7 +148,7 @@ public void ReadWithPrefetch(ref TBatch batch, TContext userContext) #endif { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); + clientSession.store.ContextReadWithPrefetch, TStoreFunctions, TAllocator>>(ref batch, userContext, sessionFunctions); } /// @@ -183,209 +156,239 @@ public void ReadWithPrefetch(ref TBatch batch, TContext userContext) public Status ReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextReadAtAddress(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextReadAtAddress, TStoreFunctions, TAllocator>>(address, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status ReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) + public Status ReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); + return clientSession.store.ContextReadAtAddress(address, key, ref input, ref output, ref readOptions, out recordMetadata, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out _, userContext); + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) + public Status Upsert(TKey key, ReadOnlySpan desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) { TInput input = default; TOutput output = default; - return Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, TContext userContext = default) + private Status Upsert(TKey key, long keyHash, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, userContext, sessionFunctions); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out _, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); + public Status Upsert(TKey key, ref TInput input, ReadOnlySpan desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcStringValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(ref TKey key, ref TInput input, ref TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => Upsert(ref key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref desiredValue, ref output, out recordMetadata, userContext); + public Status Upsert(TKey key, IHeapObject desiredValue, TContext userContext = default) + { + TInput input = default; + TOutput output = default; + return Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status Upsert(ref TKey key, long keyHash, ref TInput input, ref TValue desiredValue, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TKey key, IHeapObject desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) { - Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextUpsert(ref key, keyHash, ref input, ref desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + TInput input = default; + TOutput output = default; + return Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, TContext userContext = default) - => Upsert(ref key, ref desiredValue, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + => Upsert(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TValue desiredValue, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref desiredValue, ref upsertOptions, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + => Upsert(key, upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, desiredValue, ref output, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, ref upsertOptions, userContext); + private Status Upsert(TKey key, long keyHash, ref TInput input, IHeapObject desiredValue, ref TOutput output, TContext userContext = default) + { + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out _, userContext, sessionFunctions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Upsert(TKey key, TInput input, TValue desiredValue, ref TOutput output, TContext userContext = default) - => Upsert(ref key, ref input, ref desiredValue, ref output, userContext); + public Status Upsert(TKey key, ref TInput input, IHeapObject desiredValue, ref TOutput output, ref UpsertOptions upsertOptions, out RecordMetadata recordMetadata, TContext userContext = default) + { + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, srcObjectValue: desiredValue, ref output, out recordMetadata, userContext, sessionFunctions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, TContext userContext = default) - => RMW(ref key, ref input, ref output, out _, userContext); + public Status Upsert(in TSourceLogRecord diskLogRecord) + where TSourceLogRecord : ISourceLogRecord + => Upsert(diskLogRecord, in diskLogRecord); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out _, userContext); + public Status Upsert(TOpKey key, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TInput input = default; + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord diskLogRecord) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + { + TOutput output = default; + UpsertOptions upsertOptions = default; + return Upsert(key, ref input, in diskLogRecord, ref output, ref upsertOptions); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) - => RMW(ref key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), ref input, ref output, out recordMetadata, userContext); + public Status Upsert(ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TSourceLogRecord : ISourceLogRecord + => Upsert(inputLogRecord, ref input, in inputLogRecord, ref output, ref upsertOptions, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + public Status Upsert(TOpKey key, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertOptions upsertOptions, TContext userContext = default) + where TOpKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextRMW(ref key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); + var keyHash = upsertOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key); + return clientSession.store.ContextUpsert(key, keyHash, ref input, inputLogRecord: in inputLogRecord, ref output, out _, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, TContext userContext = default) + => RMW(key, ref input, ref output, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, out TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) - { - output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out _, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, TContext userContext = default) - { - TOutput output = default; - return RMW(ref key, ref input, ref output, userContext); - } + public Status RMW(TKey key, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Status RMW(TKey key, ref TInput input, ref TOutput output, ref RMWOptions rmwOptions, out RecordMetadata recordMetadata, TContext userContext = default) + => RMW(key, rmwOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), ref input, ref output, out recordMetadata, userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + public Status RMW(TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext userContext = default) { - TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); + return clientSession.store.ContextRMW(key, keyHash, ref input, ref output, out recordMetadata, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, TContext userContext = default) - => RMW(ref key, ref input, userContext); + public Status RMW(TKey key, ref TInput input, TContext userContext = default) + { + TOutput output = default; + return RMW(key, ref input, ref output, userContext); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(TKey key, TInput input, ref RMWOptions rmwOptions, TContext userContext = default) + public Status RMW(TKey key, ref TInput input, ref RMWOptions rmwOptions, TContext userContext = default) { TOutput output = default; - return RMW(ref key, ref input, ref output, ref rmwOptions, userContext); + return RMW(key, ref input, ref output, ref rmwOptions, userContext); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, TContext userContext = default) - => Delete(ref key, clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status Delete(TKey key, TContext userContext = default) + => Delete(key, clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(ref key), userContext); + public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) + => Delete(key, deleteOptions.KeyHash ?? clientSession.store.storeFunctions.GetKeyHashCode64(key), userContext); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref TKey key, long keyHash, TContext userContext = default) + public Status Delete(TKey key, long keyHash, TContext userContext = default) { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( - ref key, keyHash, userContext, sessionFunctions); + return clientSession.store.ContextDelete, TStoreFunctions, TAllocator>>( + key, keyHash, userContext, sessionFunctions); } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, TContext userContext = default) - => Delete(ref key, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(TKey key, ref DeleteOptions deleteOptions, TContext userContext = default) - => Delete(ref key, ref deleteOptions, userContext); - - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ResetModified(ref TKey key) - => clientSession.UnsafeResetModified(sessionFunctions, ref key); + public void ResetModified(TKey key) + => clientSession.UnsafeResetModified(sessionFunctions, key); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool IsModified(TKey key) - => clientSession.UnsafeIsModified(sessionFunctions, ref key); + => clientSession.UnsafeIsModified(sessionFunctions, key); /// public void Refresh() { Debug.Assert(clientSession.store.epoch.ThisInstanceProtected()); - clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); + clientSession.store.InternalRefresh, TStoreFunctions, TAllocator>>(sessionFunctions); } #endregion ITsavoriteContext } diff --git a/libs/storage/Tsavorite/cs/src/core/Compaction/ICompactionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Compaction/ICompactionFunctions.cs index 6e6f82d9677..29dad3630a8 100644 --- a/libs/storage/Tsavorite/cs/src/core/Compaction/ICompactionFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Compaction/ICompactionFunctions.cs @@ -6,30 +6,24 @@ namespace Tsavorite.core /// /// Optional functions to be called during compaction. /// - /// - /// - public interface ICompactionFunctions + public interface ICompactionFunctions { /// /// Checks if record in the Tsavorite log is logically deleted. - /// If the record was deleted via - /// then this function is not called for such a record. + /// If the record was deleted the usual Delete() (i.e. its tombstone is set), then this function is not called for it. /// /// - /// - /// One possible scenario is if Tsavorite is used to store reference counted records. - /// Once the record count reaches zero it can be considered to be no longer relevant and - /// compaction can skip the record. - /// + /// One possible scenario is if Tsavorite is used to store reference counted records. If the refcount reaches zero + /// it can be considered to be no longer relevant and compaction can skip the record. /// - /// - /// - /// - bool IsDeleted(ref TKey key, ref TValue value); + bool IsDeleted(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord; } - internal struct DefaultCompactionFunctions : ICompactionFunctions + internal struct DefaultCompactionFunctions : ICompactionFunctions { - public bool IsDeleted(ref TKey key, ref TValue value) => false; + public bool IsDeleted(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => false; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs deleted file mode 100644 index 635ea16332d..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Compaction/LogCompactionFunctions.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -namespace Tsavorite.core -{ - internal sealed class LogCompactionFunctions : ISessionFunctions - where TFunctions : ISessionFunctions - { - readonly TFunctions _functions; - - public LogCompactionFunctions(TFunctions functions) - { - _functions = functions; - } - - /// - /// No reads during compaction - /// - public bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) => true; - - public bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - - public void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo) { } - - /// - /// No ConcurrentDeleter needed for compaction - /// - public bool ConcurrentDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - - public void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - /// - /// For compaction, we never perform concurrent writes as rolled over data defers to - /// newly inserted data for the same key. - /// - public bool ConcurrentWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) => true; - - public void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - public bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - - public bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo) => true; - - public bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - public void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo) { } - - public bool InPlaceUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - - public bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; - - public bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo) => true; - - public void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } - - public void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) - where TEpochAccessor : IEpochAccessor - { } - - public void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } - - public int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => 0; - public int GetRMWInitialValueLength(ref TInput input) => 0; - public int GetUpsertValueLength(ref TValue value, ref TInput input) => _functions.GetUpsertValueLength(ref value, ref input); - - /// - /// No reads during compaction - /// - public bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo) => true; - - /// - /// Write compacted live value to store - /// - public bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => _functions.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); - - public void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason) { } - - public void ConvertOutputToHeap(ref TInput input, ref TOutput output) { } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Compaction/TsavoriteCompaction.cs b/libs/storage/Tsavorite/cs/src/core/Compaction/TsavoriteCompaction.cs index 4bef9caa879..f2920dfc0cd 100644 --- a/libs/storage/Tsavorite/cs/src/core/Compaction/TsavoriteCompaction.cs +++ b/libs/storage/Tsavorite/cs/src/core/Compaction/TsavoriteCompaction.cs @@ -6,58 +6,52 @@ namespace Tsavorite.core /// /// Compaction methods /// - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate /// - /// Functions used to manage key-values during compaction - /// User provided compaction functions (see ). - /// Input for SingleWriter - /// Output from SingleWriter; it will be called all records that are moved, before Compact() returns, so the user must supply buffering or process each output completely + /// User provided compaction functions (see ). /// Compact log until this address /// Compaction type (whether we lookup records or scan log for liveness checking) /// Address until which compaction was done - internal long Compact(TFunctions functions, TCompactionFunctions cf, ref TInput input, ref TOutput output, long untilAddress, CompactionType compactionType) - where TFunctions : ISessionFunctions - where TCompactionFunctions : ICompactionFunctions + internal long Compact(TCompactionFunctions cf, long untilAddress, CompactionType compactionType) + where TCompactionFunctions : ICompactionFunctions { return compactionType switch { - CompactionType.Scan => CompactScan(functions, cf, ref input, ref output, untilAddress), - CompactionType.Lookup => CompactLookup(functions, cf, ref input, ref output, untilAddress), + CompactionType.Scan => CompactScan(cf, untilAddress), + CompactionType.Lookup => CompactLookup(cf, untilAddress), _ => throw new TsavoriteException("Invalid compaction type"), }; } - private long CompactLookup(TFunctions functions, TCompactionFunctions cf, ref TInput input, ref TOutput output, long untilAddress) - where TFunctions : ISessionFunctions - where TCompactionFunctions : ICompactionFunctions + private long CompactLookup(TCompactionFunctions cf, long untilAddress) + where TCompactionFunctions : ICompactionFunctions { if (untilAddress > hlogBase.SafeReadOnlyAddress) throw new TsavoriteException("Can compact only until Log.SafeReadOnlyAddress"); - var lf = new LogCompactionFunctions(functions); - using var storeSession = NewSession>(lf); + using var storeSession = NewSession>(new()); var storebContext = storeSession.BasicContext; using (var iter1 = Log.Scan(Log.BeginAddress, untilAddress)) { long numPending = 0; - while (iter1.GetNext(out var recordInfo)) + while (iter1.GetNext()) { - ref var key = ref iter1.GetKey(); - ref var value = ref iter1.GetValue(); + var key = iter1.Key; - if (!recordInfo.Tombstone && !cf.IsDeleted(ref key, ref value)) + if (!iter1.Info.Tombstone && !cf.IsDeleted(in iter1)) { - var status = storebContext.CompactionCopyToTail(ref key, ref input, ref value, ref output, iter1.CurrentAddress, iter1.NextAddress); + var iter1AsLogSource = iter1 as ISourceLogRecord; // Can't use 'ref' on a 'using' variable + var status = storebContext.CompactionCopyToTail(in iter1AsLogSource, iter1.CurrentAddress, iter1.NextAddress); if (status.IsPending && ++numPending > 256) { - storebContext.CompletePending(wait: true); + _ = storebContext.CompletePending(wait: true); numPending = 0; } } @@ -66,47 +60,45 @@ private long CompactLookup 0) - storebContext.CompletePending(wait: true); + _ = storebContext.CompletePending(wait: true); } Log.ShiftBeginAddress(untilAddress, false); return untilAddress; } - private long CompactScan(TFunctions functions, TCompactionFunctions cf, ref TInput input, ref TOutput output, long untilAddress) - where TFunctions : ISessionFunctions - where TCompactionFunctions : ICompactionFunctions + private long CompactScan(TCompactionFunctions cf, long untilAddress) + where TCompactionFunctions : ICompactionFunctions { if (untilAddress > hlogBase.SafeReadOnlyAddress) throw new TsavoriteException("Can compact only until Log.SafeReadOnlyAddress"); var originalUntilAddress = untilAddress; - var lf = new LogCompactionFunctions(functions); - using var storeSession = NewSession>(lf); + using var storeSession = NewSession>(new()); var storebContext = storeSession.BasicContext; - var tempKVSettings = new KVSettings(baseDir: null, loggerFactory: loggerFactory) + var tempKVSettings = new KVSettings(baseDir: null, loggerFactory: loggerFactory) { - IndexSize = KVSettings.SetIndexSizeFromCacheLines(IndexSize), + IndexSize = KVSettings.SetIndexSizeFromCacheLines(IndexSize), LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice() }; - using (var tempKv = new TsavoriteKV(tempKVSettings, storeFunctions, allocatorFactory)) - using (var tempKvSession = tempKv.NewSession(functions)) + using (var tempKv = new TsavoriteKV(tempKVSettings, storeFunctions, allocatorFactory)) + using (var tempKvSession = tempKv.NewSession>(new())) { var tempbContext = tempKvSession.BasicContext; using (var iter1 = Log.Scan(hlogBase.BeginAddress, untilAddress)) { - while (iter1.GetNext(out var recordInfo)) + while (iter1.GetNext()) { - ref var key = ref iter1.GetKey(); - ref var value = ref iter1.GetValue(); - - if (recordInfo.Tombstone || cf.IsDeleted(ref key, ref value)) - tempbContext.Delete(ref key); + if (iter1.Info.Tombstone || cf.IsDeleted(in iter1)) + _ = tempbContext.Delete(iter1); else - tempbContext.Upsert(ref key, ref value); + { + var iterLogRecord = iter1 as ISourceLogRecord; // Can't use 'ref' on a 'using' variable + _ = tempbContext.Upsert(in iterLogRecord); + } } // Ensure address is at record boundary untilAddress = originalUntilAddress = iter1.NextAddress; @@ -119,9 +111,9 @@ private long CompactScan= untilAddress (scan boundary), we are safe to copy the old record // to the tail. We don't know the actualAddress of the key in the main kv, but we it will not be below untilAddress. - var status = storebContext.CompactionCopyToTail(ref iter3.GetKey(), ref input, ref iter3.GetValue(), ref output, iter3.CurrentAddress, untilAddress - 1); + var iter3AsLogSource = iter3 as ISourceLogRecord; // Can't use 'ref' on a 'using' variable + var status = storebContext.CompactionCopyToTail(in iter3AsLogSource, iter3.CurrentAddress, untilAddress - 1); if (status.IsPending && ++numPending > 256) { - storebContext.CompletePending(wait: true); + _ = storebContext.CompletePending(wait: true); numPending = 0; } } if (numPending > 0) - storebContext.CompletePending(wait: true); + _ = storebContext.CompletePending(wait: true); } Log.ShiftBeginAddress(originalUntilAddress, false); return originalUntilAddress; } private void ScanImmutableTailToRemoveFromTempKv(ref long untilAddress, long scanUntil, - BasicContext tempbContext) - where TFunctions : ISessionFunctions + BasicContext tempbContext) + where TFunctions : ISessionFunctions { using var iter = Log.Scan(untilAddress, scanUntil); - while (iter.GetNext(out var _)) + while (iter.GetNext()) { - tempbContext.Delete(ref iter.GetKey(), default); + _ = tempbContext.Delete(iter, default); untilAddress = iter.NextAddress; } } diff --git a/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs b/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs index d424129f2e4..a5e23882a2f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/Devices.cs @@ -45,9 +45,9 @@ public static IDevice CreateLogDevice(string logPath = null, DeviceType deviceTy return deviceType switch { DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Linux) => new NativeStorageDevice(logPath, deleteOnClose, disableFileBuffering, capacity, logger: logger), - DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => new LocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort, readOnly: readOnly), - DeviceType.RandomAccess => new RandomAccessLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly), - DeviceType.FileStream => new ManagedLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly), + DeviceType.Native when RuntimeInformation.IsOSPlatform(OSPlatform.Windows) => new LocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, useIoCompletionPort, readOnly: readOnly, logger: logger), + DeviceType.RandomAccess => new RandomAccessLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly, logger: logger), + DeviceType.FileStream => new ManagedLocalStorageDevice(logPath, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, readOnly: readOnly, logger: logger), DeviceType.Null => new NullDevice(), _ => throw new TsavoriteException($"Unsupported local device {deviceType}"), }; diff --git a/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs index 901438e3827..5c83b173c1b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/IDevice.cs @@ -61,7 +61,7 @@ public interface IDevice : IDisposable /// /// Initialize device. This function is used to pass optional information that may only be known after /// Tsavorite initialization (whose constructor takes in IDevice upfront). Implementation are free to ignore - /// information if it does not need the supplied information. Segment size of -1 is used for object log. + /// information if it does not need the supplied information. /// /// This is a bit of a hack. /// @@ -86,7 +86,7 @@ public interface IDevice : IDisposable /* Segmented addressing API */ /// - /// Write + /// Write to the file. The alignedSourceAddress must be pinned. /// /// /// @@ -94,10 +94,13 @@ public interface IDevice : IDisposable /// /// /// + /// While this supports concurrent writes, the caller should try as much as possible to sequentialize the writes, as the IDevice implementation + /// may require append-only behavior and thus will have to buffer. For similar reasons, do not back up and re-write; depending on the IDevice implementation, + /// this may fail or be inefficient. void WriteAsync(IntPtr sourceAddress, int segmentId, ulong destinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, object context); /// - /// Read + /// Read from the file. The alignedSourceAddress must be pinned. /// /// /// @@ -110,17 +113,22 @@ public interface IDevice : IDisposable /* Direct addressing API */ /// - /// Write + /// Write to the file. The alignedSourceAddress must be pinned. If inheriting from , that provides an implementation of this that calculates the segmentId + /// and then invokes the overload with that segmentId. /// /// /// /// /// /// + /// While this supports concurrent writes, the caller should try as much as possible to sequentialize the writes, as the IDevice implementation + /// may require append-only behavior and thus will have to buffer. For similar reasons, do not back up and re-write; depending on the IDevice implementation, + /// this may fail or be inefficient. void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDestinationAddress, uint numBytesToWrite, DeviceIOCompletionCallback callback, object context); /// - /// Read + /// Read from the file. The alignedDestinationAddress must be pinned. If inheriting from , that provides an implementation of this that calculates the segmentId + /// and then invokes the overload with that segmentId. /// /// /// diff --git a/libs/storage/Tsavorite/cs/src/core/Device/LocalMemoryDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/LocalMemoryDevice.cs index 1f6186c86ce..83121c53750 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/LocalMemoryDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/LocalMemoryDevice.cs @@ -148,7 +148,7 @@ public override void WriteAsync(IntPtr sourceAddress, var req = new IORequestLocalMemory { srcAddress = (void*)sourceAddress, - dstAddress = ram_segment_ptrs[segmentId % parallelism] + destinationAddress, + dstAddress = ram_segment_ptrs[segmentId] + destinationAddress, bytes = numBytesToWrite, callback = callback, context = context diff --git a/libs/storage/Tsavorite/cs/src/core/Device/LocalStorageDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/LocalStorageDevice.cs index 112b82eaa50..d0791b180c6 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/LocalStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/LocalStorageDevice.cs @@ -8,6 +8,7 @@ using System.IO; using System.Runtime.InteropServices; using System.Threading; +using Microsoft.Extensions.Logging; using Microsoft.Win32.SafeHandles; namespace Tsavorite.core @@ -40,6 +41,7 @@ public unsafe class LocalStorageDevice : StorageDeviceBase private static uint sectorSize = 0; private bool _disposed; readonly bool readOnly; + readonly ILogger logger; /// /// Number of pending reads on device @@ -48,6 +50,13 @@ public unsafe class LocalStorageDevice : StorageDeviceBase private IntPtr ioCompletionPort; + /// + public override string ToString() + { + static string bstr(bool value) => value ? "T" : "F"; + return $"secSize {sectorSize}, numPend {numPending}, RO {bstr(readOnly)}, preAll {bstr(preallocateFile)}, delClose {bstr(deleteOnClose)}, noFileBuf {bstr(disableFileBuffering)}"; + } + /// /// Constructor /// @@ -58,6 +67,8 @@ public unsafe class LocalStorageDevice : StorageDeviceBase /// The maximum number of bytes this storage device can accommodate, or CAPACITY_UNSPECIFIED if there is no such limit /// Whether to recover device metadata from existing files /// Whether we use IO completion port with polling + /// Indicate if this is a readonly device + /// ILogger instance public LocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, @@ -65,7 +76,8 @@ public LocalStorageDevice(string filename, long capacity = Devices.CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool useIoCompletionPort = false, - bool readOnly = false) + bool readOnly = false, + ILogger logger = null) : this(filename, preallocateFile, deleteOnClose, disableFileBuffering, capacity, recoverDevice, null, useIoCompletionPort, readOnly: readOnly) { } @@ -92,6 +104,8 @@ void _callback(uint errorCode, uint numBytes, NativeOverlapped* pOVERLAP) /// Whether to recover device metadata from existing files /// Optional set of preloaded safe file handles, which can speed up hydration of preexisting log file handles /// Whether we use IO completion port with polling + /// Indicate if this is a readonly device + /// ILogger instance protected internal LocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, @@ -100,7 +114,8 @@ protected internal LocalStorageDevice(string filename, bool recoverDevice = false, IEnumerable> initialLogFileHandles = null, bool useIoCompletionPort = true, - bool readOnly = false) + bool readOnly = false, + ILogger logger = null) : base(filename, GetSectorSize(filename), capacity) { if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) @@ -139,6 +154,7 @@ protected internal LocalStorageDevice(string filename, this.deleteOnClose = deleteOnClose; this.disableFileBuffering = disableFileBuffering; this.readOnly = readOnly; + this.logger = logger; results = new ConcurrentQueue(); logHandles = initialLogFileHandles != null @@ -248,12 +264,14 @@ public override void ReadAsync(int segmentId, ulong sourceAddress, } catch (IOException e) { + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); Interlocked.Decrement(ref numPending); callback((uint)(e.HResult & 0x0000FFFF), 0, context); results.Enqueue(result); } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); Interlocked.Decrement(ref numPending); callback(uint.MaxValue, 0, context); results.Enqueue(result); @@ -315,12 +333,14 @@ public override unsafe void WriteAsync(IntPtr sourceAddress, } catch (IOException e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); callback((uint)(e.HResult & 0x0000FFFF), 0, context); results.Enqueue(result); } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); callback(uint.MaxValue, 0, context); results.Enqueue(result); @@ -355,6 +375,8 @@ public override void RemoveSegmentAsync(int segment, AsyncCallback callback, IAs /// public override void Dispose() { + if (_disposed) + return; _disposed = true; foreach (var logHandle in logHandles.Values) logHandle.Dispose(); @@ -363,9 +385,7 @@ public override void Dispose() new SafeFileHandle(ioCompletionPort, true).Dispose(); while (results.TryDequeue(out var entry)) - { Overlapped.Free(entry.nativeOverlapped); - } } /// diff --git a/libs/storage/Tsavorite/cs/src/core/Device/ManagedLocalStorageDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/ManagedLocalStorageDevice.cs index 2c64ef2a85c..d3caf328787 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/ManagedLocalStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/ManagedLocalStorageDevice.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Tsavorite.core { @@ -21,6 +22,7 @@ public sealed class ManagedLocalStorageDevice : StorageDeviceBase private readonly bool disableFileBuffering; private readonly bool osReadBuffering; private readonly bool readOnly; + private readonly ILogger logger; private readonly SafeConcurrentDictionary, AsyncPool)> logHandles; private readonly SectorAlignedBufferPool pool; @@ -42,7 +44,8 @@ public sealed class ManagedLocalStorageDevice : StorageDeviceBase /// Whether to recover device metadata from existing files /// Enable OS read buffering /// Open file in readOnly mode - public ManagedLocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, long capacity = Devices.CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool osReadBuffering = false, bool readOnly = false) + /// + public ManagedLocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, long capacity = Devices.CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool osReadBuffering = false, bool readOnly = false, ILogger logger = null) : base(filename, GetSectorSize(filename), capacity) { pool = new(1, 1); @@ -58,6 +61,7 @@ public ManagedLocalStorageDevice(string filename, bool preallocateFile = false, this.disableFileBuffering = disableFileBuffering; this.osReadBuffering = osReadBuffering; this.readOnly = readOnly; + this.logger = logger; logHandles = new(); if (recoverDevice) RecoverFiles(); @@ -158,8 +162,9 @@ public override void ReadAsync(int segmentId, ulong sourceAddress, readTask = logReadHandle.ReadAsync(umm.Memory).AsTask(); } } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); Interlocked.Decrement(ref numPending); // Perform pool returns and disposals @@ -203,6 +208,7 @@ public override void ReadAsync(int segmentId, ulong sourceAddress, } catch (Exception ex) { + logger?.LogCritical(ex, $"{nameof(ReadAsync)}"); if (ex.InnerException != null && ex.InnerException is IOException ioex) errorCode = (uint)(ioex.HResult & 0x0000FFFF); else @@ -263,8 +269,9 @@ public override void WriteAsync(IntPtr sourceAddress, writeTask = logWriteHandle.WriteAsync(umm.Memory).AsTask(); } } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); // Perform pool returns and disposals @@ -290,8 +297,9 @@ public override void WriteAsync(IntPtr sourceAddress, writeTask = logWriteHandle.WriteAsync(umm.Memory).AsTask(); } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); // Perform pool returns and disposals @@ -309,6 +317,7 @@ public override void WriteAsync(IntPtr sourceAddress, } catch (Exception ex) { + logger?.LogCritical(ex, $"{nameof(WriteAsync)}"); if (ex.InnerException != null && ex.InnerException is IOException ioex) errorCode = (uint)(ioex.HResult & 0x0000FFFF); else diff --git a/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs index 496f2b27c71..f470f04e21c 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/NativeStorageDevice.cs @@ -313,6 +313,24 @@ void _callback(IntPtr context, int errorCode, ulong numBytes) /// public override bool Throttle() => numPending > ThrottleLimit; + /// + /// Selects the Linux asynchronous I/O backend used by the underlying native device. + /// On non-Linux platforms this value is ignored. + /// + public enum IoBackend : int + { + /// Platform default (libaio on Linux, ThreadPool-style on Windows). + Default = 0, + /// Linux libaio (io_submit / io_getevents). Same as Default on Linux. + Libaio = 1, + } + + /// + /// Configured I/O backend. Stored for reporting and forward-compatibility; the shipped + /// native library currently uses libaio on Linux for both Default and Libaio. + /// + public IoBackend ConfiguredIoBackend { get; } + /// /// Constructor with more options for derived classes /// @@ -322,14 +340,20 @@ void _callback(IntPtr context, int errorCode, ulong numBytes) /// The maximum number of bytes this storage device can accommodate, or CAPACITY_UNSPECIFIED if there is no such limit /// Number of IO completion threads /// + /// Linux async I/O backend selector (Default, Libaio). Ignored on Windows. public NativeStorageDevice(string filename, bool deleteOnClose = false, bool disableFileBuffering = true, - long capacity = Devices.CAPACITY_UNSPECIFIED, int numCompletionThreads = 1, ILogger logger = null) + long capacity = Devices.CAPACITY_UNSPECIFIED, + int numCompletionThreads = 1, + ILogger logger = null, + IoBackend ioBackend = IoBackend.Default) : base(filename, GetSectorSize(filename), capacity) { Debug.Assert(numCompletionThreads >= 1); + ConfiguredIoBackend = ioBackend; + // Native device uses a fixed segment size nativeSegmentSizeBits = 30; @@ -412,12 +436,14 @@ public override void ReadAsync(int segmentId, ulong sourceAddress, } catch (IOException e) { + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); Interlocked.Decrement(ref numPending); callback((uint)(e.HResult & 0x0000FFFF), 0, context); freeResults.Enqueue(offset); } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(ReadAsync)}"); Interlocked.Decrement(ref numPending); callback(uint.MaxValue, 0, context); freeResults.Enqueue(offset); @@ -459,11 +485,13 @@ public override unsafe void WriteAsync(IntPtr sourceAddress, } catch (IOException e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); callback((uint)(e.HResult & 0x0000FFFF), 0, context); } - catch + catch (Exception e) { + logger?.LogCritical(e, $"{nameof(WriteAsync)}"); Interlocked.Decrement(ref numPending); callback(uint.MaxValue, 0, context); } diff --git a/libs/storage/Tsavorite/cs/src/core/Device/RandomAccessLocalStorageDevice.cs b/libs/storage/Tsavorite/cs/src/core/Device/RandomAccessLocalStorageDevice.cs index d1543394538..67c00bb4e75 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/RandomAccessLocalStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/RandomAccessLocalStorageDevice.cs @@ -9,6 +9,7 @@ using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Tsavorite.core { @@ -33,6 +34,7 @@ public sealed class RandomAccessLocalStorageDevice : StorageDeviceBase private readonly bool disableFileBuffering; private readonly bool osReadBuffering; private readonly bool readOnly; + private readonly ILogger logger; private readonly SafeConcurrentDictionary, AsyncPool)> logHandles; private readonly SectorAlignedBufferPool pool; @@ -54,7 +56,8 @@ public sealed class RandomAccessLocalStorageDevice : StorageDeviceBase /// Whether to recover device metadata from existing files /// Enable OS read buffering /// Open file in readOnly mode - public RandomAccessLocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, long capacity = Devices.CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool osReadBuffering = false, bool readOnly = false) + /// + public RandomAccessLocalStorageDevice(string filename, bool preallocateFile = false, bool deleteOnClose = false, bool disableFileBuffering = true, long capacity = Devices.CAPACITY_UNSPECIFIED, bool recoverDevice = false, bool osReadBuffering = false, bool readOnly = false, ILogger logger = null) : base(filename, GetSectorSize(filename), capacity) { pool = new(1, 1); @@ -70,6 +73,7 @@ public RandomAccessLocalStorageDevice(string filename, bool preallocateFile = fa this.disableFileBuffering = disableFileBuffering; this.osReadBuffering = osReadBuffering; this.readOnly = readOnly; + this.logger = logger; logHandles = new(); if (recoverDevice) RecoverFiles(); @@ -172,6 +176,7 @@ async ValueTask ReadWorkerAsync(int segmentId, ulong sourceAddress, IntPtr desti } catch (Exception ex) { + logger?.LogCritical(ex, $"{nameof(ReadAsync)}"); var ioex = ex as IOException ?? ex.InnerException as IOException; if (ioex is not null) errorCode = (uint)(ioex.HResult & 0x0000FFFF); @@ -226,10 +231,11 @@ async ValueTask WriteWorkerAsync(IntPtr sourceAddress, int segmentId, ulong dest { storageAccessContext.memoryManager.SetDestination((byte*)sourceAddress, (int)numBytesToWrite); } - await RandomAccess.WriteAsync(storageAccessContext.handle.SafeFileHandle, storageAccessContext.memoryManager.Memory, (long)destinationAddress); + await RandomAccess.WriteAsync(storageAccessContext.handle.SafeFileHandle, storageAccessContext.memoryManager.Memory, (long)destinationAddress).ConfigureAwait(false); } catch (Exception ex) { + logger?.LogCritical(ex, $"{nameof(ReadAsync)}"); var ioex = ex as IOException ?? ex.InnerException as IOException; if (ioex is not null) errorCode = (uint)(ioex.HResult & 0x0000FFFF); diff --git a/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs b/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs index 4903dcf103f..f417614d52a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Device/StorageDeviceBase.cs @@ -146,7 +146,7 @@ protected internal static string GetSegmentFilename(string filename, int segment public virtual bool Throttle() => false; /// - /// Write operation + /// Write operation; compute the segment id and call the IDevice implementation. /// /// /// @@ -163,7 +163,7 @@ public void WriteAsync(IntPtr alignedSourceAddress, ulong alignedDestinationAddr } /// - /// Read operation + /// Read operation; compute the segment id and call the IDevice implementation. /// /// /// @@ -221,21 +221,21 @@ public virtual void RemoveSegment(int segment) /// public void TruncateUntilSegmentAsync(int toSegment, AsyncCallback callback, IAsyncResult result) { - // Reset begin range to at least toAddress + // Reset begin range to at least toSegment if (!Utility.MonotonicUpdate(ref startSegment, toSegment, out int oldStart)) { // If no-op, invoke callback and return immediately callback(result); return; } + + // We will delete segments in parallel; create a countdown event that will signal when all are completed. CountdownEvent countdown = new(toSegment - oldStart); + // This action needs to be epoch-protected because readers may be issuing reads to the deleted segment, unaware of the delete. // Because of earlier compare-and-swap, the caller has exclusive access to the range [oldStartSegment, newStartSegment), and there will // be no double deletes. - - bool isProtected = epoch.ThisInstanceProtected(); - if (!isProtected) - epoch.Resume(); + var suspendEpochWhenDone = epoch.ResumeIfNotProtected(); try { epoch.BumpCurrentEpoch(() => @@ -255,7 +255,8 @@ public void TruncateUntilSegmentAsync(int toSegment, AsyncCallback callback, IAs } finally { - if (!isProtected) epoch.Suspend(); + if (suspendEpochWhenDone) + epoch.Suspend(); } } diff --git a/libs/storage/Tsavorite/cs/src/core/Epochs/LightEpoch.cs b/libs/storage/Tsavorite/cs/src/core/Epochs/LightEpoch.cs index 64deff5bef4..06fa177991a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Epochs/LightEpoch.cs +++ b/libs/storage/Tsavorite/cs/src/core/Epochs/LightEpoch.cs @@ -3,6 +3,8 @@ using System; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; @@ -19,28 +21,28 @@ public sealed unsafe class LightEpoch : IEpochAccessor /// (1) in AssignInstance, to assign a unique instanceId to each LightEpoch instance, and /// (2) in Metadata, to track per-thread epoch table entries for each LightEpoch instance. /// - [StructLayout(LayoutKind.Explicit, Size = MaxInstances * sizeof(int))] + [InlineArray(MaxInstances)] private struct InstanceIndexBuffer { /// /// Maximum number of concurrent instances of LightEpoch supported. /// - internal const int MaxInstances = 16; + internal const int MaxInstances = 1024; /// /// Anchor field for the buffer. /// - [FieldOffset(0)] int field0; /// /// Reference to the entry for the given instance ID. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] + [UnscopedRef] internal ref int GetRef(int instanceId) { Debug.Assert(instanceId >= 0 && instanceId < MaxInstances); - return ref Unsafe.AsRef((int*)Unsafe.AsPointer(ref field0) + instanceId); + return ref Unsafe.Add(ref field0, instanceId); } } @@ -151,6 +153,20 @@ private class Metadata /// readonly int instanceId; + /// + /// Maximum number of general-purpose per-thread user-word slots that subsystems + /// can claim via . Bounded by the free space in 's + /// cache line (48 bytes = 6 longs). + /// + public const int MaxUserWords = 6; + + /// + /// Bitmask of claimed user-word slots. Each set bit means that word index is in use by some + /// subsystem. Managed exclusively via CAS in and + /// . Not read on the epoch Acquire/Release hot path. + /// + int userWordMask; + /// /// This is the LightEpoch-level static buffer (array) of available instance slots. /// On LightEpoch instance creation, it is used by SelectInstance() to find an @@ -193,7 +209,7 @@ int SelectInstance() if (kInvalidIndex == Interlocked.CompareExchange(ref entry, 1, kInvalidIndex)) return i; } - throw new InvalidOperationException("Exceeded maximum number of active LightEpoch instances"); + throw new InvalidOperationException($"Exceeded maximum number of active LightEpoch instances {ActiveInstanceCount()} {InstanceIndexBuffer.MaxInstances}"); } /// @@ -252,15 +268,11 @@ public void Dispose() /// Check whether current epoch instance is protected on this thread /// /// Result of the check + [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool ThisInstanceProtected() { ref var entry = ref Metadata.Entries.GetRef(instanceId); - if (kInvalidIndex != entry) - { - if ((*(tableAligned + entry)).threadId == Metadata.threadId) - return true; - } - return false; + return kInvalidIndex != entry && (*(tableAligned + entry)).threadId == Metadata.threadId; } /// @@ -296,9 +308,7 @@ public void ProtectAndDrain() // Max epoch across all threads may have advanced, so check for pending drain actions to process if (drainCount > 0) - { Drain((*(tableAligned + entry)).localCurrentEpoch); - } if (waiterCount > 0) { @@ -323,7 +333,8 @@ public void SuspendResume() public void Suspend() { Release(); - if (drainCount > 0) SuspendDrain(); + if (drainCount > 0) + SuspendDrain(); } /// @@ -335,6 +346,18 @@ public void Resume() Acquire(); } + /// + /// Thread resumes its epoch entry if it has not already been acquired + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool ResumeIfNotProtected() + { + if (ThisInstanceProtected()) + return false; + Resume(); + return true; + } + /// /// Increment global current epoch /// @@ -342,12 +365,12 @@ public void Resume() internal long BumpCurrentEpoch() { Debug.Assert(ThisInstanceProtected(), "BumpCurrentEpoch must be called on a protected thread"); - long nextEpoch = Interlocked.Increment(ref CurrentEpoch); + var nextEpoch = Interlocked.Increment(ref CurrentEpoch); if (drainCount > 0) Drain(nextEpoch); else - ComputeNewSafeToReclaimEpoch(nextEpoch); + _ = ComputeNewSafeToReclaimEpoch(nextEpoch); return nextEpoch; } @@ -360,9 +383,9 @@ internal long BumpCurrentEpoch() /// public void BumpCurrentEpoch(Action onDrain) { - long PriorEpoch = BumpCurrentEpoch() - 1; + var PriorEpoch = BumpCurrentEpoch() - 1; - int i = 0; + var i = 0; while (true) { if (drainList[i].epoch == long.MaxValue) @@ -372,7 +395,7 @@ public void BumpCurrentEpoch(Action onDrain) { drainList[i].action = onDrain; drainList[i].epoch = PriorEpoch; - Interlocked.Increment(ref drainCount); + _ = Interlocked.Increment(ref drainCount); break; } } @@ -399,7 +422,7 @@ public void BumpCurrentEpoch(Action onDrain) // We are at the end of the drain list and found no empty or reclaimable slot. ProtectAndDrain, which should clear one or more slots. ProtectAndDrain(); i = 0; - Thread.Yield(); + _ = Thread.Yield(); } } @@ -407,12 +430,6 @@ public void BumpCurrentEpoch(Action onDrain) ProtectAndDrain(); } - /// - /// Looks at all threads and return the latest safe epoch - /// - /// Safe epoch - internal long ComputeNewSafeToReclaimEpoch() => ComputeNewSafeToReclaimEpoch(CurrentEpoch); - /// /// Looks at all threads and return the latest safe epoch /// @@ -420,17 +437,15 @@ public void BumpCurrentEpoch(Action onDrain) /// Safe epoch long ComputeNewSafeToReclaimEpoch(long currentEpoch) { - long oldestOngoingCall = currentEpoch; + var oldestOngoingCall = currentEpoch; - for (int index = 1; index <= kTableSize; ++index) + for (var index = 1; index <= kTableSize; index++) { - long entry_epoch = (*(tableAligned + index)).localCurrentEpoch; + var entry_epoch = (*(tableAligned + index)).localCurrentEpoch; if (0 != entry_epoch) { if (entry_epoch < oldestOngoingCall) - { oldestOngoingCall = entry_epoch; - } } } @@ -450,13 +465,11 @@ void SuspendDrain() // Barrier ensures we see the latest epoch table entries. Ensures // that the last suspended thread drains all pending actions. Thread.MemoryBarrier(); - for (int index = 1; index <= kTableSize; ++index) + for (var index = 1; index <= kTableSize; index++) { - long entry_epoch = (*(tableAligned + index)).localCurrentEpoch; + var entry_epoch = (*(tableAligned + index)).localCurrentEpoch; if (0 != entry_epoch) - { return; - } } Resume(); Release(); @@ -470,9 +483,9 @@ void SuspendDrain() [MethodImpl(MethodImplOptions.NoInlining)] void Drain(long nextEpoch) { - ComputeNewSafeToReclaimEpoch(nextEpoch); + _ = ComputeNewSafeToReclaimEpoch(nextEpoch); - for (int i = 0; i < kDrainListSize; i++) + for (var i = 0; i < kDrainListSize; i++) { var trigger_epoch = drainList[i].epoch; @@ -484,7 +497,7 @@ void Drain(long nextEpoch) var trigger_action = drainList[i].action; drainList[i].action = null; drainList[i].epoch = long.MaxValue; - Interlocked.Decrement(ref drainCount); + _ = Interlocked.Decrement(ref drainCount); // Execute the action trigger_action(); @@ -663,7 +676,7 @@ void ReserveEntryForThread(ref int entry) if (Metadata.threadId == 0) // run once per thread for performance { Metadata.threadId = Environment.CurrentManagedThreadId; - uint code = (uint)Utility.Murmur3(Metadata.threadId); + var code = (uint)Utility.Murmur3(Metadata.threadId); Metadata.startOffset1 = (ushort)(1 + (code % kTableSize)); Metadata.startOffset2 = (ushort)(1 + ((code >> 16) % kTableSize)); } @@ -713,8 +726,120 @@ public override string ToString() return sb.ToString(); } + #region User-word API + + /// + /// Number of entries in the epoch table. + /// + public int EntryCount => kTableSize; + + /// + /// Claim a per-thread user-word slot. Returns the word index to pass to + /// and . + /// The column across all entries is initialized to . + /// After allocation, the application owns the slot contents — LightEpoch does not + /// automatically reset slots on epoch Acquire/Release. Throws if all + /// slots are already claimed. + /// + /// Value written to every entry's slot at allocation time. + /// Word index in the range [0, ). + public int AllocateUserWord(long initialValue) + { + while (true) + { + var mask = Volatile.Read(ref userWordMask); + int idx = BitOperations.TrailingZeroCount(~mask); + if (idx >= MaxUserWords) + throw new InvalidOperationException($"All {MaxUserWords} LightEpoch user-word slots are claimed."); + + // CAS to claim the slot. Only the winner proceeds to initialize. + var newMask = mask | (1 << idx); + if (Interlocked.CompareExchange(ref userWordMask, newMask, mask) != mask) + continue; // another thread modified the mask; retry + + // We exclusively own this slot — initialize the column across all entries. + for (int i = 1; i <= kTableSize; i++) + Volatile.Write(ref UserWordRef(i, idx), initialValue); + + return idx; + } + } + + /// + /// Release a previously claimed user-word slot. Caller is responsible for ensuring that no + /// producer thread still holds or can still issue writes to the slot (e.g., by calling this + /// only after subsystem quiescence / Dispose). + /// + public void ReleaseUserWord(int wordIndex) + { + if ((uint)wordIndex >= MaxUserWords) + throw new ArgumentOutOfRangeException(nameof(wordIndex)); + while (true) + { + var mask = Volatile.Read(ref userWordMask); + var newMask = mask & ~(1 << wordIndex); + if (Interlocked.CompareExchange(ref userWordMask, newMask, mask) == mask) + return; + } + } + + /// + /// Get a ref to the current thread's user-word slot. Caller MUST be inside epoch protection + /// ( / before ). Returns the same cache line that is + /// already hot due to epoch Resume, so writes are essentially free. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref long ThisThreadUserWord(int wordIndex) + { + Debug.Assert((uint)wordIndex < MaxUserWords, "Invalid user-word index"); + Debug.Assert(ThisInstanceProtected(), "ThisThreadUserWord must be called while epoch is protected"); + int entryIndex = Metadata.Entries.GetRef(instanceId); + return ref UserWordRef(entryIndex, wordIndex); + } + + /// + /// Compute the minimum value of the user-word at across all epoch + /// table entries, using a direct unsafe pointer walk. + /// + /// User-word slot index (0-based). + /// The minimum value observed across all entries. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetMinUserWord(int wordIndex) + { + Debug.Assert((uint)wordIndex < MaxUserWords, "Invalid user-word index"); + + // Derive the base address from the actual Entry field layout via UserWordRef, + // rather than hardcoding byte offsets. Entries occupy indices 1..kTableSize + // (index 0 is kInvalidIndex and unused). Stride between entries is kCacheLineBytes. + long min = long.MaxValue; + byte* basePtr = (byte*)Unsafe.AsPointer(ref UserWordRef(1, wordIndex)); + int stride = kCacheLineBytes; + int count = kTableSize; + + for (int i = 0; i < count; i++) + { + long v = Volatile.Read(ref Unsafe.AsRef(basePtr + (long)i * stride)); + if (v < min) min = v; + } + return min; + } + + /// + /// Get a ref to the user word at for entry . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + ref long UserWordRef(int entryIndex, int wordIndex) + => ref Unsafe.Add(ref (*(tableAligned + entryIndex)).userWord0, wordIndex); + + #endregion + /// /// Epoch table entry (cache line size). + /// Existing epoch fields occupy the first 16 bytes (localCurrentEpoch + threadId + 4 bytes padding). + /// The remaining 48 bytes host general-purpose per-thread slots that + /// subsystems can claim via . This reuses the cache line that is already + /// hot from epoch Resume/Suspend, so user-word access is essentially free compared to touching a separate + /// data structure. /// [StructLayout(LayoutKind.Explicit, Size = kCacheLineBytes)] struct Entry @@ -731,6 +856,28 @@ struct Entry [FieldOffset(8)] public int threadId; + /// + /// First user-word slot. Remaining - 1 slots are contiguous after this + /// field at 8-byte stride. Access via Unsafe.Add(ref userWord0, wordIndex). + /// + [FieldOffset(16)] + public long userWord0; + + [FieldOffset(24)] + public long userWord1; + + [FieldOffset(32)] + public long userWord2; + + [FieldOffset(40)] + public long userWord3; + + [FieldOffset(48)] + public long userWord4; + + [FieldOffset(56)] + public long userWord5; + public override string ToString() => $"lce = {localCurrentEpoch}, tid = {threadId}"; } @@ -742,7 +889,7 @@ struct EpochActionPair public long epoch; public Action action; - public override string ToString() => $"epoch = {epoch}, action = {(action is null ? "n/a" : action.Method.ToString())}"; + public override readonly string ToString() => $"epoch = {epoch}, action = {(action is null ? "n/a" : action.Method.ToString())}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DefaultCheckpointNamingScheme.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DefaultCheckpointNamingScheme.cs index b0611c6b115..de7c4120871 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DefaultCheckpointNamingScheme.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DefaultCheckpointNamingScheme.cs @@ -14,13 +14,16 @@ public class DefaultCheckpointNamingScheme : ICheckpointNamingScheme /// public string BaseName { get; } + int SublogIdx { get; } + /// /// Create instance of default naming scheme /// /// Overall location specifier (e.g., local path or cloud container name) - public DefaultCheckpointNamingScheme(string baseName = "") + public DefaultCheckpointNamingScheme(string baseName = "", int sublogIdx = -1) { BaseName = baseName; + SublogIdx = sublogIdx; } /// @@ -31,9 +34,6 @@ public DefaultCheckpointNamingScheme(string baseName = "") public FileDescriptor LogSnapshot(Guid token) => new(Path.Join(LogCheckpointBasePath, token.ToString()), "snapshot.dat"); /// public FileDescriptor ObjectLogSnapshot(Guid token) => new(Path.Join(LogCheckpointBasePath, token.ToString()), "snapshot.obj.dat"); - /// - public FileDescriptor DeltaLog(Guid token) => new(Path.Join(LogCheckpointBasePath, token.ToString()), "delta.dat"); - /// public FileDescriptor IndexCheckpointBase(Guid token) => new(Path.Join(IndexCheckpointBasePath, token.ToString()), null); /// @@ -41,7 +41,7 @@ public DefaultCheckpointNamingScheme(string baseName = "") /// public FileDescriptor HashTable(Guid token) => new(Path.Join(IndexCheckpointBasePath, token.ToString()), "ht.dat"); /// - public FileDescriptor TsavoriteLogCommitMetadata(long commitNumber) => new(TsavoriteLogCommitBasePath, $"commit.{commitNumber}"); + public FileDescriptor TsavoriteLogCommitMetadata(long commitNumber) => new(TsavoriteLogCommitBasePath, SublogIdx == -1 ? $"commit.{commitNumber}" : $"commit.{commitNumber}.{SublogIdx}"); /// public Guid Token(FileDescriptor fileDescriptor) => Guid.Parse(new DirectoryInfo(fileDescriptor.directoryName).Name); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DeviceLogCommitCheckpointManager.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DeviceLogCommitCheckpointManager.cs index e15d6a26009..e48d67136c8 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DeviceLogCommitCheckpointManager.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/DeviceLogCommitCheckpointManager.cs @@ -95,7 +95,7 @@ public void Purge(Guid token) #region ILogCommitManager /// - public unsafe void Commit(long beginAddress, long untilAddress, byte[] commitMetadata, long commitNum, bool forceWriteMetadata) + public void Commit(long beginAddress, long untilAddress, byte[] commitMetadata, long commitNum, bool forceWriteMetadata) { if (!forceWriteMetadata && fastCommitThrottleFreq > 0 && (commitCount++ % fastCommitThrottleFreq != 0)) return; @@ -171,7 +171,7 @@ public byte[] GetCommitMetadata(long commitNum) public virtual byte[] GetCookie() => null; /// - public unsafe void CommitIndexCheckpoint(Guid indexToken, byte[] commitMetadata) + public void CommitIndexCheckpoint(Guid indexToken, byte[] commitMetadata) { var device = NextIndexCheckpointDevice(indexToken); @@ -186,7 +186,10 @@ public unsafe void CommitIndexCheckpoint(Guid indexToken, byte[] commitMetadata) } /// - public unsafe void CleanupIndexCheckpoint(Guid indexToken) + public virtual bool PerformAutomaticCleanup => true; + + /// + public void CleanupIndexCheckpoint(Guid indexToken) { if (removeOutdated) { @@ -222,7 +225,7 @@ public byte[] GetIndexCheckpointMetadata(Guid indexToken) } /// - public unsafe void CommitLogCheckpointMetadata(Guid logToken, byte[] commitMetadata) + public void CommitLogCheckpointMetadata(Guid logToken, byte[] commitMetadata) { var device = NextLogCheckpointDevice(logToken); @@ -237,7 +240,7 @@ public unsafe void CommitLogCheckpointMetadata(Guid logToken, byte[] commitMetad } /// - public unsafe void CleanupLogCheckpoint(Guid logToken) + public void CleanupLogCheckpoint(Guid logToken) { if (removeOutdated) { @@ -249,33 +252,6 @@ public unsafe void CleanupLogCheckpoint(Guid logToken) } } - /// - public virtual unsafe void CommitLogIncrementalCheckpoint(Guid logToken, byte[] commitMetadata, DeltaLog deltaLog) - { - deltaLog.Allocate(out var length, out var physicalAddress); - if (length < commitMetadata.Length) - { - deltaLog.Seal(0, DeltaLogEntryType.CHECKPOINT_METADATA); - deltaLog.Allocate(out length, out physicalAddress); - if (length < commitMetadata.Length) - { - deltaLog.Seal(0); - throw new Exception($"Metadata of size {commitMetadata.Length} does not fit in delta log space of size {length}"); - } - } - fixed (byte* ptr = commitMetadata) - { - Buffer.MemoryCopy(ptr, (void*)physicalAddress, commitMetadata.Length, commitMetadata.Length); - } - deltaLog.Seal(commitMetadata.Length, DeltaLogEntryType.CHECKPOINT_METADATA); - deltaLog.FlushAsync().Wait(); - } - - /// - public virtual unsafe void CleanupLogIncrementalCheckpoint(Guid logToken) - { - } - /// public IEnumerable GetLogCheckpointTokens() { @@ -283,45 +259,8 @@ public IEnumerable GetLogCheckpointTokens() } /// - public virtual byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta = false, long recoverTo = -1) + public virtual byte[] GetLogCheckpointMetadata(Guid logToken) { - byte[] metadata = null; - if (deltaLog != null && scanDelta) - { - // Try to get latest valid metadata from delta-log - deltaLog.Reset(); - while (deltaLog.GetNext(out long physicalAddress, out int entryLength, out var type)) - { - switch (type) - { - case DeltaLogEntryType.DELTA: - // consider only metadata records - continue; - case DeltaLogEntryType.CHECKPOINT_METADATA: - metadata = new byte[entryLength]; - unsafe - { - fixed (byte* m = metadata) - Buffer.MemoryCopy((void*)physicalAddress, m, entryLength, entryLength); - } - - var hlri = new HybridLogRecoveryInfo(); - using (StreamReader s = new(new MemoryStream(metadata))) - { - hlri.Initialize(s); - // Finish recovery if only specific versions are requested - if (hlri.version == recoverTo || hlri.version < recoverTo && hlri.nextVersion > recoverTo) goto LoopEnd; - } - continue; - default: - throw new TsavoriteException("Unexpected entry type"); - } - LoopEnd: - break; - } - if (metadata != null) return metadata; - } - var device = deviceFactory.Get(checkpointNamingScheme.LogCheckpointMetadata(logToken)); ReadInto(device, 0, out byte[] writePad, sizeof(int)); @@ -355,12 +294,6 @@ public IDevice GetSnapshotObjectLogDevice(Guid token) return deviceFactory.Get(checkpointNamingScheme.ObjectLogSnapshot(token)); } - /// - public IDevice GetDeltaLogDevice(Guid token) - { - return deviceFactory.Get(checkpointNamingScheme.DeltaLog(token)); - } - /// public void InitializeIndexCheckpoint(Guid indexToken) { diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/ICheckpointNamingScheme.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/ICheckpointNamingScheme.cs index a0fa9d36f2f..2fab79d808b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/ICheckpointNamingScheme.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/ICheckpointNamingScheme.cs @@ -50,11 +50,6 @@ public interface ICheckpointNamingScheme /// FileDescriptor ObjectLogSnapshot(Guid token); - /// - /// Delta log - /// - FileDescriptor DeltaLog(Guid token); - /// /// TsavoriteLog commit metadata /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/RecoveryInfo.cs b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/RecoveryInfo.cs index e1ad380f735..7360ae0c71c 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/RecoveryInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/CheckpointManagement/RecoveryInfo.cs @@ -3,7 +3,7 @@ using System; using System.IO; -using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; namespace Tsavorite.core @@ -13,7 +13,7 @@ namespace Tsavorite.core /// public struct HybridLogRecoveryInfo { - public const int CheckpointVersion = 6; + public const int CheckpointVersion = 7; /// /// HybridLogRecoveryVersion @@ -32,55 +32,61 @@ public struct HybridLogRecoveryInfo /// public long version; /// - /// Next Version + /// The next version of the database when the checkpoint flush was started /// public long nextVersion; /// - /// Flushed logical address; indicates the latest immutable address on the main Tsavorite log at checkpoint commit time. + /// FlushedUntilAddress at the PERSISTENCE_CALLBACK phase; indicates the latest immutable (flushed) address on the main Tsavorite log at checkpoint commit time. /// public long flushedLogicalAddress; /// - /// Flushed logical address at snapshot start; indicates device offset for snapshot file + /// FlushedUntilAddress at the start of the WAIT_FLUSH phase; indicates device offset for snapshot file /// public long snapshotStartFlushedLogicalAddress; /// - /// Start logical address + /// Start logical address; the tail address at PREPARE phase, which is the start of the "fuzzy region" /// public long startLogicalAddress; /// - /// Final logical address + /// Final logical address; initially the tail address at WAIT_FLUSH phase, which is the the end of the "fuzzy region". It may be increase beyond this due to delta records. /// public long finalLogicalAddress; /// /// Snapshot end logical address: snapshot is [startLogicalAddress, snapshotFinalLogicalAddress) - /// Note that finalLogicalAddress may be higher due to delta records + /// Note that this is initially set to finalLogicalAddress at the start of WAIT_FLUSH, but finalLogicalAddress may be higher due to delta records /// public long snapshotFinalLogicalAddress; /// - /// Head address + /// hlog HeadAddress at the start of the WAIT_FLUSH phase. This is the initial address to start scanning from; the lowest address at which we will bring pages + /// into the circular buffer (may be in the middle of a page) /// public long headAddress; /// - /// Begin address + /// hlog BeginAddress at the start of the PREPARE phase /// public long beginAddress; /// - /// Placeholder to avoid checkpoint format change + /// The objectLog segment for hlog's BeginAddress () at PREPARE; + /// corresponds to . Will be zero unless the log has been truncated. /// - public bool placeholder; + internal int beginAddressObjectLogSegment; /// - /// Object log segment offsets + /// The taken at PERSISTENCE_CALLBACK (matching ). + /// This is incremented for any flushes due to ReadOnlyAddress growth during the snapshot. /// - public long[] objectLogSegmentOffsets; + internal ObjectLogFilePositionInfo hlogEndObjectLogTail; /// - /// Tail address of delta file: -1 indicates this is not a delta checkpoint metadata - /// At recovery, this value denotes the delta tail address excluding the metadata record for the checkpoint - /// because we create the metadata before writing to the delta file. + /// The at the start of the checkpoint (start of WAIT_FLUSH). /// - public long deltaTailAddress; + internal ObjectLogFilePositionInfo snapshotStartObjectLogTail; + + /// + /// The at the end of the checkpoint (at PERSISTENCE_CALLBACK). + /// + internal ObjectLogFilePositionInfo snapshotEndObjectLogTail; /// /// User cookie @@ -108,21 +114,20 @@ public void Initialize(Guid token, long _version) startLogicalAddress = 0; finalLogicalAddress = 0; snapshotFinalLogicalAddress = 0; - deltaTailAddress = -1; // indicates this is not a delta checkpoint metadata headAddress = 0; - objectLogSegmentOffsets = null; + hlogEndObjectLogTail = new(); // Marks as "unset" + snapshotStartObjectLogTail = new(); + snapshotEndObjectLogTail = new(); } - const int checkpointTokenCount = 0; // Temporary to keep compatibility with previous checkpoint versions - /// /// Initialize from stream /// /// public void Initialize(StreamReader reader) { - string value = reader.ReadLine(); + var value = reader.ReadLine(); var cversion = int.Parse(value); if (cversion != CheckpointVersion) @@ -167,56 +172,26 @@ public void Initialize(StreamReader reader) beginAddress = long.Parse(value); value = reader.ReadLine(); - deltaTailAddress = long.Parse(value); - - value = reader.ReadLine(); - placeholder = bool.Parse(value); + beginAddressObjectLogSegment = int.Parse(value); - value = reader.ReadLine(); - var numSessions = int.Parse(value); - - // Temporary for backward compatibility - for (int i = 0; i < numSessions; i++) - { - _ /*var sessionID*/ = int.Parse(reader.ReadLine()); - _ /*var sessionName*/ = reader.ReadLine(); - _ /*var serialno*/ = long.Parse(reader.ReadLine()); - - var exclusionCount = int.Parse(reader.ReadLine()); - for (int j = 0; j < exclusionCount; j++) - _ = reader.ReadLine(); - } + hlogEndObjectLogTail.Deserialize(reader); + snapshotStartObjectLogTail.Deserialize(reader); + snapshotEndObjectLogTail.Deserialize(reader); - // Read object log segment offsets + // Read user cookie value = reader.ReadLine(); - var numSegments = int.Parse(value); - if (numSegments > 0) + var cookieSize = int.Parse(value); + if (cookieSize > 0) { - objectLogSegmentOffsets = new long[numSegments]; - for (int i = 0; i < numSegments; i++) + cookie = new byte[cookieSize]; + for (var i = 0; i < cookieSize; i++) { value = reader.ReadLine(); - objectLogSegmentOffsets[i] = long.Parse(value); + cookie[i] = byte.Parse(value); } } - if (cversion >= 6) - { - // Read user cookie - value = reader.ReadLine(); - var cookieSize = int.Parse(value); - if (cookieSize > 0) - { - cookie = new byte[cookieSize]; - for (var i = 0; i < cookieSize; i++) - { - value = reader.ReadLine(); - cookie[i] = byte.Parse(value); - } - } - } - - if (checksum != Checksum(numSessions)) + if (checksum != Checksum()) throw new TsavoriteException("Invalid checksum for checkpoint"); Deserialized = true; @@ -227,17 +202,10 @@ public void Initialize(StreamReader reader) /// /// /// - /// - /// - /// whether to scan the delta log to obtain the latest info contained in an incremental snapshot checkpoint. - /// If false, this will recover the base snapshot info but avoid potentially expensive scans. - /// - /// specific version to recover to, if using delta log - internal void Recover(Guid token, ICheckpointManager checkpointManager, DeltaLog deltaLog = null, bool scanDelta = false, long recoverTo = -1) + internal void Recover(Guid token, ICheckpointManager checkpointManager) { - var metadata = checkpointManager.GetLogCheckpointMetadata(token, deltaLog, scanDelta, recoverTo); - if (metadata == null) - throw new TsavoriteException("Invalid log commit metadata for ID " + token.ToString()); + var metadata = checkpointManager.GetLogCheckpointMetadata(token) + ?? throw new TsavoriteException("Invalid log commit metadata for ID " + token.ToString()); using StreamReader s = new(new MemoryStream(metadata)); Initialize(s); } @@ -247,41 +215,27 @@ internal void Recover(Guid token, ICheckpointManager checkpointManager, DeltaLog /// /// /// - /// /// Any user-specified commit cookie written as part of the checkpoint - /// - /// whether to scan the delta log to obtain the latest info contained in an incremental snapshot checkpoint. - /// If false, this will recover the base snapshot info but avoid potentially expensive scans. - /// - /// specific version to recover to, if using delta log - - internal void Recover(Guid token, ICheckpointManager checkpointManager, out byte[] commitCookie, DeltaLog deltaLog = null, bool scanDelta = false, long recoverTo = -1) + internal void Recover(Guid token, ICheckpointManager checkpointManager, out byte[] commitCookie) { - var metadata = checkpointManager.GetLogCheckpointMetadata(token, deltaLog, scanDelta, recoverTo); - if (metadata == null) - throw new TsavoriteException("Invalid log commit metadata for ID " + token.ToString()); + var metadata = checkpointManager.GetLogCheckpointMetadata(token) + ?? throw new TsavoriteException("Invalid log commit metadata for ID " + token.ToString()); using StreamReader s = new(new MemoryStream(metadata)); Initialize(s); - if (scanDelta && deltaLog != null && deltaTailAddress >= 0) - { - // Adjust delta tail address to include the metadata record - deltaTailAddress = deltaLog.NextAddress; - } commitCookie = cookie; } /// /// Write info to byte array /// - public byte[] ToByteArray() + public readonly byte[] ToByteArray() { using (MemoryStream ms = new()) { using (StreamWriter writer = new(ms)) { writer.WriteLine(CheckpointVersion); // checkpoint version - - writer.WriteLine(Checksum(checkpointTokenCount)); // checksum + writer.WriteLine(Checksum()); writer.WriteLine(guid); writer.WriteLine(useSnapshotFile); @@ -294,43 +248,33 @@ public byte[] ToByteArray() writer.WriteLine(snapshotFinalLogicalAddress); writer.WriteLine(headAddress); writer.WriteLine(beginAddress); - writer.WriteLine(deltaTailAddress); - writer.WriteLine(placeholder); - writer.WriteLine(checkpointTokenCount); + writer.WriteLine(beginAddressObjectLogSegment); - // Write object log segment offsets - writer.WriteLine(objectLogSegmentOffsets == null ? 0 : objectLogSegmentOffsets.Length); - if (objectLogSegmentOffsets != null) - { - for (var i = 0; i < objectLogSegmentOffsets.Length; i++) - { - writer.WriteLine(objectLogSegmentOffsets[i]); - } - } + hlogEndObjectLogTail.Serialize(writer); + snapshotStartObjectLogTail.Serialize(writer); + snapshotEndObjectLogTail.Serialize(writer); - // User cookie write + // Write user cookie var cookieSize = cookie == null ? 0 : cookie.Length; writer.WriteLine(cookieSize); if (cookieSize > 0) { for (var i = 0; i < cookieSize; i++) - { writer.WriteLine(cookie[i]); - } } } return ms.ToArray(); } } - private readonly long Checksum(int checkpointTokensCount) + private readonly long Checksum() { var bytes = guid.ToByteArray(); var long1 = BitConverter.ToInt64(bytes, 0); var long2 = BitConverter.ToInt64(bytes, 8); - return long1 ^ long2 ^ version ^ flushedLogicalAddress ^ snapshotStartFlushedLogicalAddress ^ startLogicalAddress ^ finalLogicalAddress ^ snapshotFinalLogicalAddress ^ headAddress ^ beginAddress - ^ checkpointTokensCount ^ (objectLogSegmentOffsets == null ? 0 : objectLogSegmentOffsets.Length); + return long1 ^ long2 ^ version ^ flushedLogicalAddress ^ snapshotStartFlushedLogicalAddress ^ startLogicalAddress ^ finalLogicalAddress ^ snapshotFinalLogicalAddress + ^ headAddress ^ beginAddress ^ beginAddressObjectLogSegment ^ (long)hlogEndObjectLogTail.word ^ (long)snapshotStartObjectLogTail.word ^ (long)snapshotEndObjectLogTail.word; } /// @@ -349,7 +293,10 @@ public readonly void DebugPrint(ILogger logger) logger?.LogInformation("Snapshot Final Logical Address: {snapshotFinalLogicalAddress}", snapshotFinalLogicalAddress); logger?.LogInformation("Head Address: {headAddress}", headAddress); logger?.LogInformation("Begin Address: {beginAddress}", beginAddress); - logger?.LogInformation("Delta Tail Address: {deltaTailAddress}", deltaTailAddress); + logger?.LogInformation("Begin object log segment: {beginObjLogSegment}", beginAddressObjectLogSegment); + logger?.LogInformation("Hybrid Log End Object Tail Position: {hlogEndObjLogTail}", hlogEndObjectLogTail); + logger?.LogInformation("Snapshot Begin Object Log Tail Position: {snapshotStartObjLogTail}", snapshotStartObjectLogTail); + logger?.LogInformation("Snapshot End Object Log Tail Position: {snapshotEndObjLogTail}", snapshotEndObjectLogTail); } } @@ -358,10 +305,8 @@ internal struct HybridLogCheckpointInfo : IDisposable public HybridLogRecoveryInfo info; public IDevice snapshotFileDevice; public IDevice snapshotFileObjectLogDevice; - public IDevice deltaFileDevice; - public DeltaLog deltaLog; - public SemaphoreSlim flushedSemaphore; - public long prevVersion; + public Task flushedTask; + internal CircularDiskWriteBuffer objectLogFlushBuffers; public void Initialize(Guid token, long _version, ICheckpointManager checkpointManager) { @@ -373,62 +318,21 @@ public void Dispose() { snapshotFileDevice?.Dispose(); snapshotFileObjectLogDevice?.Dispose(); - deltaLog?.Dispose(); - deltaFileDevice?.Dispose(); + objectLogFlushBuffers?.Dispose(); this = default; } - public HybridLogCheckpointInfo Transfer() - { - // Ownership transfer of handles across struct copies - var dest = this; - dest.snapshotFileDevice = default; - dest.snapshotFileObjectLogDevice = default; - deltaLog = default; - deltaFileDevice = default; - return dest; - } - - public void Recover(Guid token, ICheckpointManager checkpointManager, int deltaLogPageSizeBits, - bool scanDelta = false, long recoverTo = -1) + public void Recover(Guid token, ICheckpointManager checkpointManager) { - deltaFileDevice = checkpointManager.GetDeltaLogDevice(token); - if (deltaFileDevice is not null) - { - deltaFileDevice.Initialize(-1); - if (deltaFileDevice.GetFileSize(0) > 0) - { - deltaLog = new DeltaLog(deltaFileDevice, deltaLogPageSizeBits, -1); - deltaLog.InitializeForReads(); - info.Recover(token, checkpointManager, deltaLog, scanDelta, recoverTo); - return; - } - } - info.Recover(token, checkpointManager, null); + info.Recover(token, checkpointManager); } - public void Recover(Guid token, ICheckpointManager checkpointManager, int deltaLogPageSizeBits, - out byte[] commitCookie, bool scanDelta = false, long recoverTo = -1) + public void Recover(Guid token, ICheckpointManager checkpointManager, out byte[] commitCookie) { - deltaFileDevice = checkpointManager.GetDeltaLogDevice(token); - if (deltaFileDevice is not null) - { - deltaFileDevice.Initialize(-1); - if (deltaFileDevice.GetFileSize(0) > 0) - { - deltaLog = new DeltaLog(deltaFileDevice, deltaLogPageSizeBits, -1); - deltaLog.InitializeForReads(); - info.Recover(token, checkpointManager, out commitCookie, deltaLog, scanDelta, recoverTo); - return; - } - } info.Recover(token, checkpointManager, out commitCookie); } - public bool IsDefault() - { - return info.guid == default; - } + public readonly bool IsDefault => info.guid == default; } internal struct IndexRecoveryInfo @@ -455,7 +359,7 @@ public void Initialize(Guid token, long _size) public void Initialize(StreamReader reader) { - string value = reader.ReadLine(); + var value = reader.ReadLine(); var cversion = int.Parse(value); value = reader.ReadLine(); @@ -531,8 +435,8 @@ public readonly void DebugPrint(ILogger logger) { logger?.LogInformation("******** Index Checkpoint Info for {token} ********", token); logger?.LogInformation("Table Size: {table_size}", table_size); - logger?.LogInformation("Main Table Size (in GB): {num_ht_bytes}", ((double)num_ht_bytes) / 1000.0 / 1000.0 / 1000.0); - logger?.LogInformation("Overflow Table Size (in GB): {num_ofb_bytes}", ((double)num_ofb_bytes) / 1000.0 / 1000.0 / 1000.0); + logger?.LogInformation("Main Table Size (in GB): {num_ht_bytes}", num_ht_bytes / 1000.0 / 1000.0 / 1000.0); + logger?.LogInformation("Overflow Table Size (in GB): {num_ofb_bytes}", num_ofb_bytes / 1000.0 / 1000.0 / 1000.0); logger?.LogInformation("Num Buckets: {num_buckets}", num_buckets); logger?.LogInformation("Start Logical Address: {startLogicalAddress}", startLogicalAddress); logger?.LogInformation("Final Logical Address: {finalLogicalAddress}", finalLogicalAddress); @@ -574,9 +478,6 @@ public void Reset() main_ht_device = null; } - public bool IsDefault() - { - return info.token == default; - } + public readonly bool IsDefault => info.token == default; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/Checkpoint.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/Checkpoint.cs index 1295be24618..3aa18f6b62f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/Checkpoint.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/Checkpoint.cs @@ -8,21 +8,21 @@ namespace Tsavorite.core public static class Checkpoint { #region Single-store APIs - public static IStateMachine Full(TsavoriteKV store, CheckpointType checkpointType, out Guid guid) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public static IStateMachine Full(TsavoriteKV store, CheckpointType checkpointType, out Guid guid) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { guid = Guid.NewGuid(); - var indexCheckpointTask = new IndexCheckpointSMTask(store, guid); + var indexCheckpointTask = new IndexCheckpointSMTask(store, guid); if (checkpointType == CheckpointType.FoldOver) { - var backend = new FoldOverSMTask(store, guid); + var backend = new FoldOverSMTask(store, guid); return new FullCheckpointSM(indexCheckpointTask, backend); } else if (checkpointType == CheckpointType.Snapshot) { - var backend = new SnapshotCheckpointSMTask(store, guid); + var backend = new SnapshotCheckpointSMTask(store, guid); return new FullCheckpointSM(indexCheckpointTask, backend); } else @@ -31,38 +31,38 @@ public static IStateMachine Full(Tsav } } - public static IStateMachine Streaming(TsavoriteKV store, out Guid guid) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public static IStateMachine Streaming(TsavoriteKV store, out Guid guid) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { guid = Guid.NewGuid(); - var backend = new StreamingSnapshotCheckpointSMTask(store, guid); + var backend = new StreamingSnapshotCheckpointSMTask(store, guid); return new StreamingSnapshotCheckpointSM(backend); } - public static IStateMachine IndexOnly(TsavoriteKV store, out Guid guid) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public static IStateMachine IndexOnly(TsavoriteKV store, out Guid guid) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { guid = Guid.NewGuid(); - var indexCheckpointTask = new IndexCheckpointSMTask(store, guid); + var indexCheckpointTask = new IndexCheckpointSMTask(store, guid); return new IndexCheckpointSM(indexCheckpointTask); } - public static IStateMachine HybridLogOnly(TsavoriteKV store, CheckpointType checkpointType, out Guid guid) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public static IStateMachine HybridLogOnly(TsavoriteKV store, CheckpointType checkpointType, out Guid guid) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { guid = Guid.NewGuid(); if (checkpointType == CheckpointType.FoldOver) { - var backend = new FoldOverSMTask(store, guid); + var backend = new FoldOverSMTask(store, guid); return new HybridLogCheckpointSM(backend); } else if (checkpointType == CheckpointType.Snapshot) { - var backend = new SnapshotCheckpointSMTask(store, guid); + var backend = new SnapshotCheckpointSMTask(store, guid); return new HybridLogCheckpointSM(backend); } else @@ -71,39 +71,32 @@ public static IStateMachine HybridLogOnly(TsavoriteKV store, Guid guid) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - { - var backend = new IncrementalSnapshotCheckpointSMTask(store, guid); - return new HybridLogCheckpointSM(backend); - } #endregion #region Two-store APIs - public static IStateMachine Full( - TsavoriteKV store1, - TsavoriteKV store2, + public static IStateMachine Full( + TsavoriteKV store1, + TsavoriteKV store2, CheckpointType checkpointType, out Guid guid) - where TStoreFunctions1 : IStoreFunctions - where TAllocator1 : IAllocator - where TStoreFunctions2 : IStoreFunctions - where TAllocator2 : IAllocator + where TStoreFunctions1 : IStoreFunctions + where TAllocator1 : IAllocator + where TStoreFunctions2 : IStoreFunctions + where TAllocator2 : IAllocator { guid = Guid.NewGuid(); - var indexCheckpointTask1 = new IndexCheckpointSMTask(store1, guid); - var indexCheckpointTask2 = new IndexCheckpointSMTask(store2, guid); + var indexCheckpointTask1 = new IndexCheckpointSMTask(store1, guid); + var indexCheckpointTask2 = new IndexCheckpointSMTask(store2, guid); if (checkpointType == CheckpointType.FoldOver) { - var backend1 = new FoldOverSMTask(store1, guid); - var backend2 = new FoldOverSMTask(store2, guid); + var backend1 = new FoldOverSMTask(store1, guid); + var backend2 = new FoldOverSMTask(store2, guid); return new FullCheckpointSM(indexCheckpointTask1, indexCheckpointTask2, backend1, backend2); } else if (checkpointType == CheckpointType.Snapshot) { - var backend1 = new SnapshotCheckpointSMTask(store1, guid); - var backend2 = new SnapshotCheckpointSMTask(store2, guid); + var backend1 = new SnapshotCheckpointSMTask(store1, guid); + var backend2 = new SnapshotCheckpointSMTask(store2, guid); return new FullCheckpointSM(indexCheckpointTask1, indexCheckpointTask2, backend1, backend2); } else @@ -112,57 +105,57 @@ public static IStateMachine Full( - TsavoriteKV store1, - TsavoriteKV store2, + public static IStateMachine Streaming( + TsavoriteKV store1, + TsavoriteKV store2, out Guid guid) - where TStoreFunctions1 : IStoreFunctions - where TAllocator1 : IAllocator - where TStoreFunctions2 : IStoreFunctions - where TAllocator2 : IAllocator + where TStoreFunctions1 : IStoreFunctions + where TAllocator1 : IAllocator + where TStoreFunctions2 : IStoreFunctions + where TAllocator2 : IAllocator { guid = Guid.NewGuid(); - var backend1 = new StreamingSnapshotCheckpointSMTask(store1, guid); - var backend2 = new StreamingSnapshotCheckpointSMTask(store2, guid); + var backend1 = new StreamingSnapshotCheckpointSMTask(store1, guid); + var backend2 = new StreamingSnapshotCheckpointSMTask(store2, guid); return new StreamingSnapshotCheckpointSM(backend1, backend2); } - public static IStateMachine IndexOnly( - TsavoriteKV store1, - TsavoriteKV store2, + public static IStateMachine IndexOnly( + TsavoriteKV store1, + TsavoriteKV store2, out Guid guid) - where TStoreFunctions1 : IStoreFunctions - where TAllocator1 : IAllocator - where TStoreFunctions2 : IStoreFunctions - where TAllocator2 : IAllocator + where TStoreFunctions1 : IStoreFunctions + where TAllocator1 : IAllocator + where TStoreFunctions2 : IStoreFunctions + where TAllocator2 : IAllocator { guid = Guid.NewGuid(); - var indexCheckpointTask1 = new IndexCheckpointSMTask(store1, guid); - var indexCheckpointTask2 = new IndexCheckpointSMTask(store2, guid); + var indexCheckpointTask1 = new IndexCheckpointSMTask(store1, guid); + var indexCheckpointTask2 = new IndexCheckpointSMTask(store2, guid); return new IndexCheckpointSM(indexCheckpointTask1, indexCheckpointTask2); } - public static IStateMachine HybridLogOnly( - TsavoriteKV store1, - TsavoriteKV store2, + public static IStateMachine HybridLogOnly( + TsavoriteKV store1, + TsavoriteKV store2, CheckpointType checkpointType, out Guid guid) - where TStoreFunctions1 : IStoreFunctions - where TAllocator1 : IAllocator - where TStoreFunctions2 : IStoreFunctions - where TAllocator2 : IAllocator + where TStoreFunctions1 : IStoreFunctions + where TAllocator1 : IAllocator + where TStoreFunctions2 : IStoreFunctions + where TAllocator2 : IAllocator { guid = Guid.NewGuid(); if (checkpointType == CheckpointType.FoldOver) { - var backend1 = new FoldOverSMTask(store1, guid); - var backend2 = new FoldOverSMTask(store2, guid); + var backend1 = new FoldOverSMTask(store1, guid); + var backend2 = new FoldOverSMTask(store2, guid); return new HybridLogCheckpointSM(backend1, backend2); } else if (checkpointType == CheckpointType.Snapshot) { - var backend1 = new SnapshotCheckpointSMTask(store1, guid); - var backend2 = new SnapshotCheckpointSMTask(store2, guid); + var backend1 = new SnapshotCheckpointSMTask(store1, guid); + var backend2 = new SnapshotCheckpointSMTask(store2, guid); return new HybridLogCheckpointSM(backend1, backend2); } else @@ -171,19 +164,6 @@ public static IStateMachine HybridLogOnly( - TsavoriteKV store1, - TsavoriteKV store2, - Guid guid) - where TStoreFunctions1 : IStoreFunctions - where TAllocator1 : IAllocator - where TStoreFunctions2 : IStoreFunctions - where TAllocator2 : IAllocator - { - var backend1 = new IncrementalSnapshotCheckpointSMTask(store1, guid); - var backend2 = new IncrementalSnapshotCheckpointSMTask(store2, guid); - return new HybridLogCheckpointSM(backend1, backend2); - } #endregion } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/FoldOverSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/FoldOverSMTask.cs index 348da3ece53..55b8d3766c1 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/FoldOverSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/FoldOverSMTask.cs @@ -10,11 +10,11 @@ namespace Tsavorite.core /// version on the log and waiting until it is flushed to disk. It is simple and fast, but can result /// in garbage entries on the log, and a slower recovery of performance. /// - internal sealed class FoldOverSMTask : HybridLogCheckpointSMTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class FoldOverSMTask : HybridLogCheckpointSMTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public FoldOverSMTask(TsavoriteKV store, Guid guid) + public FoldOverSMTask(TsavoriteKV store, Guid guid) : base(store, guid) { } @@ -25,10 +25,10 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri switch (next.Phase) { case Phase.PREPARE: - store._lastSnapshotCheckpoint.Dispose(); store._hybridLogCheckpointToken = guid; store.InitializeHybridLogCheckpoint(store._hybridLogCheckpointToken, next.Version); base.GlobalBeforeEnteringState(next, stateMachineDriver); + ObjectLog_OnPrepare(); break; case Phase.WAIT_FLUSH: @@ -36,12 +36,16 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri try { store.epoch.Resume(); - _ = store.hlogBase.ShiftReadOnlyToTail(out var tailAddress, out store._hybridLogCheckpoint.flushedSemaphore); - if (store._hybridLogCheckpoint.flushedSemaphore != null) - stateMachineDriver.AddToWaitingList(store._hybridLogCheckpoint.flushedSemaphore); + _ = store.hlogBase.ShiftReadOnlyToTail(out var tailAddress, out var flushedTask); + if (flushedTask != null) + { + store._hybridLogCheckpoint.flushedTask = flushedTask; + stateMachineDriver.AddToWaitingList(store._hybridLogCheckpoint.flushedTask, StateMachineTaskType.FoldOverSMTaskHybridLogFlushed); + } // Update final logical address to the flushed tail - this may not be necessary store._hybridLogCheckpoint.info.finalLogicalAddress = tailAddress; + _ = ObjectLog_OnWaitFlush(); } finally { @@ -49,6 +53,13 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri } break; + case Phase.PERSISTENCE_CALLBACK: + // Set actual FlushedUntil to the latest possible data in main log that is on disk + // If we are using a NullDevice then storage tier is not enabled and FlushedUntilAddress may be ReadOnlyAddress; get all records in memory. + ObjectLog_OnPersistenceCallback(); + base.GlobalBeforeEnteringState(next, stateMachineDriver); + break; + default: base.GlobalBeforeEnteringState(next, stateMachineDriver); break; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/HybridLogCheckpointSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/HybridLogCheckpointSMTask.cs index ad58f8e2118..4fe9c5f7f7d 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/HybridLogCheckpointSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/HybridLogCheckpointSMTask.cs @@ -11,16 +11,16 @@ namespace Tsavorite.core /// This task is the base class for a checkpoint "backend", which decides how a captured version is /// persisted on disk. /// - internal abstract class HybridLogCheckpointSMTask : IStateMachineTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal abstract class HybridLogCheckpointSMTask : IStateMachineTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - protected readonly TsavoriteKV store; + protected readonly TsavoriteKV store; protected long lastVersion; protected readonly Guid guid; protected bool isStreaming; - public HybridLogCheckpointSMTask(TsavoriteKV store, Guid guid) + public HybridLogCheckpointSMTask(TsavoriteKV store, Guid guid) { this.store = store; this.guid = guid; @@ -41,10 +41,12 @@ public virtual void GlobalBeforeEnteringState(SystemState next, StateMachineDriv case Phase.IN_PROGRESS: store.CheckpointVersionShiftStart(lastVersion, next.Version, isStreaming); + store.storeFunctions.OnCheckpoint(CheckpointTrigger.VersionShift, guid); break; case Phase.WAIT_FLUSH: store.CheckpointVersionShiftEnd(lastVersion, next.Version, isStreaming); + store.storeFunctions.OnCheckpoint(CheckpointTrigger.FlushBegin, guid); Debug.Assert(stateMachineDriver.GetNumActiveTransactions(lastVersion) == 0, $"Active transactions in last version: {stateMachineDriver.GetNumActiveTransactions(lastVersion)}"); stateMachineDriver.ResetLastVersion(); @@ -57,13 +59,13 @@ public virtual void GlobalBeforeEnteringState(SystemState next, StateMachineDriv break; case Phase.PERSISTENCE_CALLBACK: - CollectMetadata(next, store); store.WriteHybridLogMetaInfo(); store.lastVersion = lastVersion; break; case Phase.REST: store.CleanupLogCheckpoint(); + store.storeFunctions.OnCheckpoint(CheckpointTrigger.CheckpointCompleted, guid); store._hybridLogCheckpoint.Dispose(); var nextTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); store.checkpointTcs.SetResult(new LinkedCheckpointInfo { NextTask = nextTcs.Task }); @@ -72,16 +74,33 @@ public virtual void GlobalBeforeEnteringState(SystemState next, StateMachineDriv } } - protected static void CollectMetadata(SystemState next, TsavoriteKV store) + protected void ObjectLog_OnPrepare() { - // Collect object log offsets only after flushes - // are completed - var seg = store.hlog.GetSegmentOffsets(); - if (seg != null) + // This will be zero unless Truncate() has removed enough main-log segments to allow freeing one or more object-log segments. + store._hybridLogCheckpoint.info.beginAddressObjectLogSegment = store.hlogBase.LowestObjectLogSegmentInUse; + } + + protected CircularDiskWriteBuffer ObjectLog_OnWaitFlush() + { + if (store._hybridLogCheckpoint.info.useSnapshotFile != 0) { - store._hybridLogCheckpoint.info.objectLogSegmentOffsets = new long[seg.Length]; - Array.Copy(seg, store._hybridLogCheckpoint.info.objectLogSegmentOffsets, seg.Length); + // GetObjectTail().HasData may be false if we have not flushed the main log (ReadOnlyAddress has not advanced). + store._hybridLogCheckpoint.info.snapshotStartObjectLogTail = store.hlogBase.GetObjectLogTail(); + + // Flush buffers are only used for Snapshot checkpoints. + store._hybridLogCheckpoint.objectLogFlushBuffers = store.hlogBase.CreateCircularFlushBuffers(store._hybridLogCheckpoint.snapshotFileObjectLogDevice, store.hlogBase.logger); + store._hybridLogCheckpoint.objectLogFlushBuffers?.InitializeOwnObjectLogFilePosition(store._hybridLogCheckpoint.snapshotFileObjectLogDevice.SegmentSize); } + return store._hybridLogCheckpoint.objectLogFlushBuffers; + } + + protected void ObjectLog_OnPersistenceCallback() + { + // GetObjectTail().HasData may be false if we have not flushed the main log (ReadOnlyAddress has not advanced). + store._hybridLogCheckpoint.info.hlogEndObjectLogTail = store.hlogBase.GetObjectLogTail(); + + if (store._hybridLogCheckpoint.info.useSnapshotFile != 0) + store._hybridLogCheckpoint.info.snapshotEndObjectLogTail = store._hybridLogCheckpoint.objectLogFlushBuffers?.filePosition ?? new(); } /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IncrementalSnapshotCheckpointSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IncrementalSnapshotCheckpointSMTask.cs deleted file mode 100644 index 22d7dffe6c7..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IncrementalSnapshotCheckpointSMTask.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; - -namespace Tsavorite.core -{ - /// - /// A Incremental Snapshot makes a copy of only changes that have happened since the last full Snapshot. It is - /// slower and more complex than a foldover, but more space-efficient on the log, and retains in-place - /// update performance as it does not advance the readonly marker unnecessarily. - /// - internal sealed class IncrementalSnapshotCheckpointSMTask : HybridLogCheckpointSMTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - { - public IncrementalSnapshotCheckpointSMTask(TsavoriteKV store, Guid guid) - : base(store, guid) - { - } - - /// - public override void GlobalBeforeEnteringState(SystemState next, StateMachineDriver stateMachineDriver) - { - switch (next.Phase) - { - case Phase.PREPARE: - store._hybridLogCheckpoint = store._lastSnapshotCheckpoint; - base.GlobalBeforeEnteringState(next, stateMachineDriver); - store._hybridLogCheckpoint.prevVersion = next.Version; - break; - - case Phase.IN_PROGRESS: - base.GlobalBeforeEnteringState(next, stateMachineDriver); - break; - - case Phase.WAIT_FLUSH: - base.GlobalBeforeEnteringState(next, stateMachineDriver); - if (store._hybridLogCheckpoint.deltaLog == null) - { - store._hybridLogCheckpoint.deltaFileDevice = store.checkpointManager.GetDeltaLogDevice(store._hybridLogCheckpointToken); - store._hybridLogCheckpoint.deltaFileDevice.Initialize(-1); - store._hybridLogCheckpoint.deltaLog = new DeltaLog(store._hybridLogCheckpoint.deltaFileDevice, store.hlogBase.LogPageSizeBits, -1); - store._hybridLogCheckpoint.deltaLog.InitializeForWrites(store.hlogBase.bufferPool); - } - - // We are writing delta records outside epoch protection, so callee should be able to - // handle corrupted or unexpected concurrent page changes during the flush, e.g., by - // resuming epoch protection if necessary. Correctness is not affected as we will - // only read safe pages during recovery. - store.hlogBase.AsyncFlushDeltaToDevice( - store.hlogBase.FlushedUntilAddress, - store._hybridLogCheckpoint.info.finalLogicalAddress, - store._lastSnapshotCheckpoint.info.finalLogicalAddress, - store._hybridLogCheckpoint.prevVersion, - store._hybridLogCheckpoint.deltaLog, - out store._hybridLogCheckpoint.flushedSemaphore, - store.ThrottleCheckpointFlushDelayMs); - if (store._hybridLogCheckpoint.flushedSemaphore != null) - stateMachineDriver.AddToWaitingList(store._hybridLogCheckpoint.flushedSemaphore); - break; - - case Phase.PERSISTENCE_CALLBACK: - CollectMetadata(next, store); - store._hybridLogCheckpoint.info.deltaTailAddress = store._hybridLogCheckpoint.deltaLog.TailAddress; - store.WriteHybridLogIncrementalMetaInfo(store._hybridLogCheckpoint.deltaLog); - store._hybridLogCheckpoint.info.deltaTailAddress = store._hybridLogCheckpoint.deltaLog.TailAddress; - store._lastSnapshotCheckpoint = store._hybridLogCheckpoint.Transfer(); - break; - - case Phase.REST: - store.CleanupLogIncrementalCheckpoint(); - store._hybridLogCheckpoint.Dispose(); - break; - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexCheckpointSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexCheckpointSMTask.cs index 33c1217dbf5..ef5f1bc877f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexCheckpointSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexCheckpointSMTask.cs @@ -9,14 +9,14 @@ namespace Tsavorite.core /// /// This task performs an index checkpoint. /// - internal sealed class IndexCheckpointSMTask : IStateMachineTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class IndexCheckpointSMTask : IStateMachineTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - readonly TsavoriteKV store; + readonly TsavoriteKV store; readonly Guid guid; - public IndexCheckpointSMTask(TsavoriteKV store, Guid guid) + public IndexCheckpointSMTask(TsavoriteKV store, Guid guid) { this.store = store; this.guid = guid; @@ -28,7 +28,7 @@ public void GlobalBeforeEnteringState(SystemState next, StateMachineDriver state switch (next.Phase) { case Phase.PREPARE: - Debug.Assert(store._indexCheckpoint.IsDefault()); + Debug.Assert(store._indexCheckpoint.IsDefault); store._indexCheckpointToken = guid; store.InitializeIndexCheckpoint(store._indexCheckpointToken); store._indexCheckpoint.info.startLogicalAddress = store.hlogBase.GetTailAddress(); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexResizeSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexResizeSMTask.cs index 35acf257530..9164d1ac81e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexResizeSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/IndexResizeSMTask.cs @@ -8,14 +8,14 @@ namespace Tsavorite.core /// /// Resizes an index /// - internal sealed class IndexResizeSMTask : IStateMachineTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class IndexResizeSMTask : IStateMachineTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - readonly TsavoriteKV store; + readonly TsavoriteKV store; long lastVersion; - public IndexResizeSMTask(TsavoriteKV store) + public IndexResizeSMTask(TsavoriteKV store) { this.store = store; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SemaphoreWaiterMonitor.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SemaphoreWaiterMonitor.cs new file mode 100644 index 00000000000..bc2971739f9 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SemaphoreWaiterMonitor.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + /// + /// Identifies the type of semaphore added to the state machine driver waiting list, + /// including the originating state machine task. + /// + public enum StateMachineSemaphoreType + { + /// + /// Waiting for all transactions in the last version to complete. + /// + LastVersionTransactionsDone, + + /// + /// Waiting for the main index checkpoint to complete (IndexCheckpointSMTask). + /// + IndexCheckpointSMTaskMainIndexCheckpoint, + + /// + /// Waiting for the overflow buckets checkpoint to complete (IndexCheckpointSMTask). + /// + IndexCheckpointSMTaskOverflowBucketsCheckpoint, + + /// + /// Waiting for the hybrid log flush to complete (FoldOverSMTask). + /// + FoldOverSMTaskHybridLogFlushed, + + /// + /// Waiting for the hybrid log flush to complete (SnapshotCheckpointSMTask). + /// + SnapshotCheckpointSMTaskHybridLogFlushed, + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SnapshotCheckpointSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SnapshotCheckpointSMTask.cs index b1e313706a1..7c859c8d4fb 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SnapshotCheckpointSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/SnapshotCheckpointSMTask.cs @@ -10,11 +10,11 @@ namespace Tsavorite.core /// slower and more complex than a foldover, but more space-efficient on the log, and retains in-place /// update performance as it does not advance the readonly marker unnecessarily. /// - internal sealed class SnapshotCheckpointSMTask : HybridLogCheckpointSMTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class SnapshotCheckpointSMTask : HybridLogCheckpointSMTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public SnapshotCheckpointSMTask(TsavoriteKV store, Guid guid) + public SnapshotCheckpointSMTask(TsavoriteKV store, Guid guid) : base(store, guid) { } @@ -25,10 +25,10 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri switch (next.Phase) { case Phase.PREPARE: - store._lastSnapshotCheckpoint.Dispose(); store._hybridLogCheckpointToken = guid; store.InitializeHybridLogCheckpoint(store._hybridLogCheckpointToken, next.Version); store._hybridLogCheckpoint.info.useSnapshotFile = 1; + ObjectLog_OnPrepare(); base.GlobalBeforeEnteringState(next, stateMachineDriver); break; @@ -36,12 +36,10 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri base.GlobalBeforeEnteringState(next, stateMachineDriver); store._hybridLogCheckpoint.info.snapshotFinalLogicalAddress = store._hybridLogCheckpoint.info.finalLogicalAddress; - store._hybridLogCheckpoint.snapshotFileDevice = - store.checkpointManager.GetSnapshotLogDevice(store._hybridLogCheckpointToken); - store._hybridLogCheckpoint.snapshotFileObjectLogDevice = - store.checkpointManager.GetSnapshotObjectLogDevice(store._hybridLogCheckpointToken); - store._hybridLogCheckpoint.snapshotFileDevice.Initialize(store.hlogBase.GetSegmentSize()); - store._hybridLogCheckpoint.snapshotFileObjectLogDevice.Initialize(-1); + store._hybridLogCheckpoint.snapshotFileDevice = store.checkpointManager.GetSnapshotLogDevice(store._hybridLogCheckpointToken); + store._hybridLogCheckpoint.snapshotFileObjectLogDevice = store.checkpointManager.GetSnapshotObjectLogDevice(store._hybridLogCheckpointToken); + store._hybridLogCheckpoint.snapshotFileDevice.Initialize(store.hlogBase.GetMainLogSegmentSize()); + store._hybridLogCheckpoint.snapshotFileObjectLogDevice.Initialize(store.hlogBase.GetObjectLogSegmentSize()); // If we are using a NullDevice then storage tier is not enabled and FlushedUntilAddress may be ReadOnlyAddress; get all records in memory. store._hybridLogCheckpoint.info.snapshotStartFlushedLogicalAddress = store.hlogBase.IsNullDevice ? store.hlogBase.HeadAddress : store.hlogBase.FlushedUntilAddress; @@ -52,37 +50,38 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri break; } - long startPage = store.hlogBase.GetPage(store._hybridLogCheckpoint.info.snapshotStartFlushedLogicalAddress); - long endPage = store.hlogBase.GetPage(store._hybridLogCheckpoint.info.finalLogicalAddress); - if (store._hybridLogCheckpoint.info.finalLogicalAddress > - store.hlog.GetStartLogicalAddress(endPage)) - { + var startPage = store.hlogBase.GetPage(store._hybridLogCheckpoint.info.snapshotStartFlushedLogicalAddress); + var endPage = store.hlogBase.GetPage(store._hybridLogCheckpoint.info.finalLogicalAddress); + if (store._hybridLogCheckpoint.info.finalLogicalAddress > store.hlogBase.GetLogicalAddressOfStartOfPage(endPage)) endPage++; - } // We are writing pages outside epoch protection, so callee should be able to // handle corrupted or unexpected concurrent page changes during the flush, e.g., by // resuming epoch protection if necessary. Correctness is not affected as we will // only read safe pages during recovery. - store.hlogBase.AsyncFlushPagesToDevice( - startPage, - endPage, - store._hybridLogCheckpoint.info.finalLogicalAddress, - store._hybridLogCheckpoint.info.startLogicalAddress, - store._hybridLogCheckpoint.snapshotFileDevice, - store._hybridLogCheckpoint.snapshotFileObjectLogDevice, - out store._hybridLogCheckpoint.flushedSemaphore, + store.hlogBase.AsyncFlushPagesForSnapshot(ObjectLog_OnWaitFlush(), + startPage, endPage, + startLogicalAddress: store._hybridLogCheckpoint.info.snapshotStartFlushedLogicalAddress, + endLogicalAddress: store._hybridLogCheckpoint.info.finalLogicalAddress, + fuzzyStartLogicalAddress: store._hybridLogCheckpoint.info.startLogicalAddress, + logDevice: store._hybridLogCheckpoint.snapshotFileDevice, + objectLogDevice: store._hybridLogCheckpoint.snapshotFileObjectLogDevice, + out store._hybridLogCheckpoint.flushedTask, store.ThrottleCheckpointFlushDelayMs); - if (store._hybridLogCheckpoint.flushedSemaphore != null) - stateMachineDriver.AddToWaitingList(store._hybridLogCheckpoint.flushedSemaphore); + if (store._hybridLogCheckpoint.flushedTask != null) + stateMachineDriver.AddToWaitingList(store._hybridLogCheckpoint.flushedTask, StateMachineTaskType.SnapshotCheckpointSMTaskHybridLogFlushed); break; case Phase.PERSISTENCE_CALLBACK: // Set actual FlushedUntil to the latest possible data in main log that is on disk // If we are using a NullDevice then storage tier is not enabled and FlushedUntilAddress may be ReadOnlyAddress; get all records in memory. + ObjectLog_OnPersistenceCallback(); store._hybridLogCheckpoint.info.flushedLogicalAddress = store.hlogBase.IsNullDevice ? store.hlogBase.HeadAddress : store.hlogBase.FlushedUntilAddress; base.GlobalBeforeEnteringState(next, stateMachineDriver); - store._lastSnapshotCheckpoint = store._hybridLogCheckpoint.Transfer(); + store._hybridLogCheckpoint.snapshotFileDevice?.Dispose(); + store._hybridLogCheckpoint.snapshotFileDevice = null; + store._hybridLogCheckpoint.snapshotFileObjectLogDevice?.Dispose(); + store._hybridLogCheckpoint.snapshotFileObjectLogDevice = null; break; default: diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineDriver.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineDriver.cs index b6edc09960e..f06a926ae76 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineDriver.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineDriver.cs @@ -17,7 +17,7 @@ public class StateMachineDriver { SystemState systemState; IStateMachine stateMachine; - readonly List waitingList; + readonly List<(Task task, StateMachineTaskType type)> waitingList; TaskCompletionSource stateMachineCompleted; // All threads have entered the given state SemaphoreSlim waitForTransitionIn; @@ -26,7 +26,7 @@ public class StateMachineDriver SemaphoreSlim waitForTransitionOut; // Transactions drained in last version long lastVersion; - SemaphoreSlim lastVersionTransactionsDone; + TaskCompletionSource lastVersionTransactionsDone; List callbacks; readonly LightEpoch epoch; readonly ILogger logger; @@ -59,7 +59,7 @@ void DecrementActiveTransactions(long txnVersion) var _lastVersionTransactionsDone = lastVersionTransactionsDone; if (_lastVersionTransactionsDone != null && txnVersion == lastVersion) { - _lastVersionTransactionsDone.Release(); + _lastVersionTransactionsDone.TrySetResult(true); } } } @@ -68,19 +68,19 @@ internal void TrackLastVersion(long version) { if (GetNumActiveTransactions(version) > 0) { - // Set version number first, then create semaphore + // Set version number first, then create TCS lastVersion = version; - lastVersionTransactionsDone = new(0); + lastVersionTransactionsDone = new(TaskCreationOptions.RunContinuationsAsynchronously); } // We have to re-check the number of active transactions after assigning lastVersion and lastVersionTransactionsDone if (GetNumActiveTransactions(version) > 0) - AddToWaitingList(lastVersionTransactionsDone); + AddToWaitingList(lastVersionTransactionsDone.Task, StateMachineTaskType.LastVersionTransactionsDone); } internal void ResetLastVersion() { - // First null semaphore, then reset version number + // First null TCS, then reset version number lastVersionTransactionsDone = null; lastVersion = 0; } @@ -155,10 +155,10 @@ public long VerifyTransactionVersion(long txnVersion) public void EndTransaction(long txnVersion) => DecrementActiveTransactions(txnVersion); - internal void AddToWaitingList(SemaphoreSlim waiter) + internal void AddToWaitingList(Task waiter, StateMachineTaskType type) { if (waiter != null) - waitingList.Add(waiter); + waitingList.Add((waiter, type)); } public bool Register(IStateMachine stateMachine, CancellationToken token = default) @@ -238,7 +238,7 @@ void GlobalStateMachineStep(SystemState expectedState) waitForTransitionOut = new SemaphoreSlim(0); waitForTransitionIn = new SemaphoreSlim(0); - logger?.LogTrace("Moved to {0}, {1}", nextState.Phase, nextState.Version); + logger?.LogTrace("SMD: Moved to {0}, {1}", nextState.Phase, nextState.Version); Debug.Assert(!epoch.ThisInstanceProtected()); try @@ -262,7 +262,7 @@ public async Task WaitForStateChange(SystemState currentState) var _waitForTransitionOut = waitForTransitionOut; if (SystemState.Equal(currentState, systemState)) { - await _waitForTransitionOut.WaitAsync(); + await _waitForTransitionOut.WaitAsync().ConfigureAwait(false); } } @@ -273,12 +273,12 @@ public async Task WaitForStateChange(SystemState currentState) /// public async Task WaitForCompletion(SystemState currentState) { - await WaitForStateChange(currentState); + await WaitForStateChange(currentState).ConfigureAwait(false); currentState = systemState; var _waitForTransitionIn = waitForTransitionIn; if (SystemState.Equal(currentState, systemState)) { - await _waitForTransitionIn.WaitAsync(); + await _waitForTransitionIn.WaitAsync().ConfigureAwait(false); } } @@ -304,14 +304,22 @@ void MakeTransitionWorker(SystemState nextState) async Task ProcessWaitingListAsync(CancellationToken token = default) { - await waitForTransitionIn.WaitAsync(token); + await waitForTransitionIn.WaitAsync(token).ConfigureAwait(false); if (waitForTransitionInException != null) { throw waitForTransitionInException; } - foreach (var waiter in waitingList) + foreach (var (task, type) in waitingList) { - await waiter.WaitAsync(token); + try + { + await task.WaitAsync(token).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger?.LogError(ex, "State machine task '{type}' faulted", type); + throw; + } } waitingList.Clear(); } @@ -324,7 +332,7 @@ async Task RunStateMachine(CancellationToken token = default) do { GlobalStateMachineStep(systemState); - await ProcessWaitingListAsync(token); + await ProcessWaitingListAsync(token).ConfigureAwait(false); } while (systemState.Phase != Phase.REST); } catch (Exception e) diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineTaskType.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineTaskType.cs new file mode 100644 index 00000000000..00fe5c25e29 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StateMachineTaskType.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + /// + /// Identifies the type of waiter added to the state machine driver waiting list, + /// including the originating state machine task. + /// + internal enum StateMachineTaskType + { + /// + /// Waiting for all transactions in the last version to complete. + /// + LastVersionTransactionsDone, + + /// + /// Waiting for the main index checkpoint to complete (IndexCheckpointSMTask). + /// + IndexCheckpointSMTaskMainIndexCheckpoint, + + /// + /// Waiting for the overflow buckets checkpoint to complete (IndexCheckpointSMTask). + /// + IndexCheckpointSMTaskOverflowBucketsCheckpoint, + + /// + /// Waiting for the hybrid log flush to complete (FoldOverSMTask). + /// + FoldOverSMTaskHybridLogFlushed, + + /// + /// Waiting for the hybrid log flush to complete (SnapshotCheckpointSMTask). + /// + SnapshotCheckpointSMTaskHybridLogFlushed, + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotCheckpointSMTask.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotCheckpointSMTask.cs index bdaed75e42f..526c88a0e73 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotCheckpointSMTask.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotCheckpointSMTask.cs @@ -13,11 +13,11 @@ namespace Tsavorite.core /// it does not require a snapshot of the index. Recovery is achieved by replaying the yielded log /// of key-value pairs and inserting each record into an empty database. /// - sealed class StreamingSnapshotCheckpointSMTask : HybridLogCheckpointSMTask - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + sealed class StreamingSnapshotCheckpointSMTask : HybridLogCheckpointSMTask + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public StreamingSnapshotCheckpointSMTask(TsavoriteKV store, Guid guid) + public StreamingSnapshotCheckpointSMTask(TsavoriteKV store, Guid guid) : base(store, guid) { isStreaming = true; @@ -30,7 +30,6 @@ public override void GlobalBeforeEnteringState(SystemState next, StateMachineDri { case Phase.PREPARE: base.GlobalBeforeEnteringState(next, stateMachineDriver); - store._lastSnapshotCheckpoint.Dispose(); store._hybridLogCheckpointToken = guid; store.InitializeHybridLogCheckpoint(store._hybridLogCheckpointToken, next.Version); store._hybridLogCheckpoint.info.nextVersion = next.Version + 1; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotTsavoriteKV.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotTsavoriteKV.cs index a6dcda90a8d..4119b413e98 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotTsavoriteKV.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/StreamingSnapshotTsavoriteKV.cs @@ -6,28 +6,28 @@ namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; + IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; long scannedUntilAddressCursor; long numberOfRecords; - class StreamingSnapshotSessionFunctions : SessionFunctionsBase + class StreamingSnapshotSessionFunctions : SessionFunctionsBase { } - class ScanPhase1Functions : IScanIteratorFunctions + class ScanPhase1Functions : IScanIteratorFunctions { - readonly IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; + readonly IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; readonly Guid checkpointToken; readonly long currentVersion; readonly long nextVersion; public long numberOfRecords; - public ScanPhase1Functions(IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions, Guid checkpointToken, long currentVersion, long nextVersion) + public ScanPhase1Functions(IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions, Guid checkpointToken, long currentVersion, long nextVersion) { this.streamingSnapshotIteratorFunctions = streamingSnapshotIteratorFunctions; this.checkpointToken = checkpointToken; @@ -36,16 +36,13 @@ public ScanPhase1Functions(IStreamingSnapshotIteratorFunctions str } /// - public bool SingleReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; - return streamingSnapshotIteratorFunctions.Reader(ref key, ref value, recordMetadata, numberOfRecords); + return streamingSnapshotIteratorFunctions.Reader(in srcLogRecord, recordMetadata, numberOfRecords); } - /// - public bool ConcurrentReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - /// public void OnException(Exception exception, long numberOfRecords) => streamingSnapshotIteratorFunctions.OnException(exception, numberOfRecords); @@ -68,7 +65,7 @@ internal void StreamingSnapshotScanPhase1() // Iterate all the read-only records in the store scannedUntilAddressCursor = Log.SafeReadOnlyAddress; var scanFunctions = new ScanPhase1Functions(streamingSnapshotIteratorFunctions, _hybridLogCheckpointToken, _hybridLogCheckpoint.info.version, _hybridLogCheckpoint.info.nextVersion); - using var s = NewSession(new()); + using var s = NewSession(new()); long cursor = 0; _ = s.ScanCursor(ref cursor, long.MaxValue, scanFunctions, scannedUntilAddressCursor); this.numberOfRecords = scanFunctions.numberOfRecords; @@ -80,28 +77,25 @@ internal void StreamingSnapshotScanPhase1() } } - class ScanPhase2Functions : IScanIteratorFunctions + class ScanPhase2Functions : IScanIteratorFunctions { - readonly IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; + readonly IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions; readonly long phase1NumberOfRecords; - public ScanPhase2Functions(IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions, long acceptedRecordCount) + public ScanPhase2Functions(IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions, long acceptedRecordCount) { this.streamingSnapshotIteratorFunctions = streamingSnapshotIteratorFunctions; this.phase1NumberOfRecords = acceptedRecordCount; } /// - public bool SingleReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord srcLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; - return streamingSnapshotIteratorFunctions.Reader(ref key, ref value, recordMetadata, numberOfRecords); + return streamingSnapshotIteratorFunctions.Reader(in srcLogRecord, recordMetadata, numberOfRecords); } - /// - public bool ConcurrentReader(ref TKey key, ref TValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - /// public void OnException(Exception exception, long numberOfRecords) => streamingSnapshotIteratorFunctions.OnException(exception, numberOfRecords); @@ -120,7 +114,7 @@ internal void StreamingSnapshotScanPhase2(long untilAddress) { // Iterate all the (v) records in the store var scanFunctions = new ScanPhase2Functions(streamingSnapshotIteratorFunctions, this.numberOfRecords); - using var s = NewSession(new()); + using var s = NewSession(new()); _ = s.ScanCursor(ref scannedUntilAddressCursor, long.MaxValue, scanFunctions, endAddress: untilAddress, maxAddress: untilAddress); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/TsavoriteStateMachineProperties.cs b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/TsavoriteStateMachineProperties.cs index c5c92eb0019..dbb5f470d5b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/TsavoriteStateMachineProperties.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Checkpointing/TsavoriteStateMachineProperties.cs @@ -3,9 +3,9 @@ namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal long lastVersion; @@ -25,11 +25,6 @@ public partial class TsavoriteKV : Ts /// public long LastCheckpointedVersion => lastVersion; - /// - /// Size (tail address) of current incremental snapshot delta log - /// - public long IncrementalSnapshotTailAddress => _lastSnapshotCheckpoint.deltaLog?.TailAddress ?? 0; - /// /// Current version number of the store /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/CompletedOutput.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/CompletedOutput.cs index 9bdbb4e9562..c0698c03531 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/CompletedOutput.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/CompletedOutput.cs @@ -6,22 +6,22 @@ namespace Tsavorite.core { /// - /// A list of for completed outputs from a pending operation. + /// A list of for completed outputs from a pending operation. /// /// The session holds this list and returns an enumeration to the caller of an appropriate CompletePending overload. The session will handle /// disposing and clearing this list, but it is best if the caller calls Dispose() after processing the results, so the key, input, and heap containers /// are released as soon as possible. - public sealed class CompletedOutputIterator : IDisposable + public sealed class CompletedOutputIterator : IDisposable { internal const int kInitialAlloc = 32; internal const int kReallocMultuple = 2; - internal CompletedOutput[] vector = new CompletedOutput[kInitialAlloc]; + internal CompletedOutput[] vector = new CompletedOutput[kInitialAlloc]; internal int maxIndex = -1; internal int currentIndex = -1; - internal void TransferFrom(ref TsavoriteKV.PendingContext pendingContext, Status status) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal void TransferFrom(ref TsavoriteKV.PendingContext pendingContext, Status status) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Note: vector is never null if (maxIndex >= vector.Length - 1) @@ -52,7 +52,7 @@ public bool Next() /// if there is no current element, either because Next() has not been called or it has advanced /// past the last element of the array /// - public ref CompletedOutput Current => ref vector[currentIndex]; + public ref CompletedOutput Current => ref vector[currentIndex]; /// public void Dispose() @@ -69,20 +69,20 @@ public void Dispose() /// The session holds a list of these that it returns to the caller of an appropriate CompletePending overload. The session will handle disposing /// and clearing, and will manage Dispose(), but it is best if the caller calls Dispose() after processing the results, so the key, input, and heap containers /// are released as soon as possible. - public struct CompletedOutput + public struct CompletedOutput { - private IHeapContainer keyContainer; + private ConditionallyHoistedKey keyContainer; private IHeapContainer inputContainer; /// /// The key for this pending operation. /// - public ref TKey Key => ref keyContainer.Get(); + public readonly ConditionallyHoistedKey Key => keyContainer; /// /// The input for this pending operation. /// - public ref TInput Input => ref inputContainer.Get(); + public readonly ref TInput Input => ref inputContainer.Get(); /// /// The output for this pending operation. It is the caller's responsibility to dispose this if necessary; will not try to dispose this member. @@ -104,19 +104,19 @@ public struct CompletedOutput /// public Status Status; - internal void TransferFrom(ref TsavoriteKV.PendingContext pendingContext, Status status) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal void TransferFrom(ref TsavoriteKV.PendingContext pendingContext, Status status) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Transfers the containers from the pendingContext, then null them; this is called before pendingContext.Dispose(). - keyContainer = pendingContext.key; - pendingContext.key = null; + keyContainer = pendingContext.requestKey; + pendingContext.requestKey = default; inputContainer = pendingContext.input; - pendingContext.input = null; + pendingContext.input = default; Output = pendingContext.output; Context = pendingContext.userContext; - RecordMetadata = new(pendingContext.recordInfo, pendingContext.logicalAddress); + RecordMetadata = new(pendingContext.logicalAddress); Status = status; } @@ -124,7 +124,7 @@ internal void Dispose() { var tempKeyContainer = keyContainer; keyContainer = default; - tempKeyContainer?.Dispose(); + tempKeyContainer.Dispose(); var tempInputContainer = inputContainer; inputContainer = default; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/ConcurrentCounter.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/ConcurrentCounter.cs index 8938c017940..ec310dabb68 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/ConcurrentCounter.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/ConcurrentCounter.cs @@ -12,7 +12,7 @@ namespace Tsavorite.core internal unsafe struct ConcurrentCounter { [StructLayout(LayoutKind.Explicit, Size = Constants.kCacheLineBytes)] - private unsafe struct Counter + private struct Counter { [FieldOffset(0)] internal long value; @@ -43,10 +43,11 @@ public ConcurrentCounter() /// The value to increment the counter by. internal void Increment(long incrValue) { - if (incrValue == 0) return; - - var partition = Environment.CurrentManagedThreadId % partitionCount; - Interlocked.Add(ref partitionsPtr[partition].value, incrValue); + if (incrValue != 0) + { + var partition = Environment.CurrentManagedThreadId % partitionCount; + _ = Interlocked.Add(ref partitionsPtr[partition].value, incrValue); + } } /// Gets the total value of the counter. @@ -55,11 +56,9 @@ internal long Total get { // return sum of all partitioned counter values - long total = 0; - - for (int i = 0; i < partitionCount; i++) + var total = 0L; + for (var i = 0; i < partitionCount; i++) total += partitionsPtr[i].value; - return total; } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/ExecutionContext.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/ExecutionContext.cs index 8853ca01d88..d6f046d0aa0 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/ExecutionContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/ExecutionContext.cs @@ -8,9 +8,9 @@ namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal sealed class TsavoriteExecutionContext { @@ -27,19 +27,19 @@ internal sealed class TsavoriteExecutionContext public long totalPending; public readonly Dictionary> ioPendingRequests; public readonly AsyncCountDown pendingReads; - public readonly AsyncQueue> readyResponses; + public readonly AsyncQueue readyResponses; public int asyncPendingCount; internal RevivificationStats RevivificationStats = new(); - public bool isAcquiredLockable; + public bool isAcquiredTransactional; public TsavoriteExecutionContext(int sessionID) { SessionState = SystemState.Make(Phase.REST, 1); this.sessionID = sessionID; - readyResponses = new AsyncQueue>(); + readyResponses = new AsyncQueue(); ioPendingRequests = new Dictionary>(); pendingReads = new AsyncCountDown(); - isAcquiredLockable = false; + isAcquiredTransactional = false; } public int SyncIoPendingCount => ioPendingRequests.Count - asyncPendingCount; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/KVSettings.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/KVSettings.cs index bfb6d9468a1..3ed23ec287e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/KVSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/KVSettings.cs @@ -10,7 +10,7 @@ namespace Tsavorite.core /// /// Configuration settings for hybrid log. Use Utility.ParseSize to specify sizes in familiar string notation (e.g., "4k" and "4 MB"). /// - public sealed class KVSettings : IDisposable + public sealed class KVSettings : IDisposable { readonly bool disposeDevices = false; readonly bool deleteDirOnDispose = false; @@ -32,24 +32,32 @@ public sealed class KVSettings : IDisposable public IDevice ObjectLogDevice; /// - /// Size of a page, in bytes + /// Size of a main-log page, in bytes /// public long PageSize = 1 << 25; /// - /// Size of a segment (group of pages), in bytes. Rounds down to power of 2. + /// Main-log circular-buffer size if nonzero; rounds down to a power of 2 and errors if that does not allow PageSize. + /// + /// + /// If zero, calculate it from and , which is also the max if both this and are nonzero. + /// + public int PageCount = 0; + + /// + /// Size of a main log segment (group of pages), in bytes. Rounds down to power of 2. /// public long SegmentSize = 1L << 30; /// - /// Total size of in-memory part of log, in bytes. Rounds down to power of 2. + /// Size of an object log segment (group of pages), in bytes. Rounds down to power of 2. /// - public long MemorySize = 1L << 34; + public long ObjectLogSegmentSize = 1L << 30; /// - /// Controls how many pages should be empty to account for non-power-of-two-sized log + /// Total size of in-memory part of main log, in bytes. Rounds down to power of 2. /// - public int MinEmptyPageCount = 0; + public long LogMemorySize = 1L << 34; /// /// Fraction of log marked as mutable (in-place updates). Rounds down to power of 2. @@ -71,20 +79,26 @@ public sealed class KVSettings : IDisposable /// public bool ReadCacheEnabled = false; + /// + /// Total size of readcache log if readcache is enabled, in bytes. Rounds down to power of 2. + /// + public long ReadCacheMemorySize = 1L << 32; + /// /// Size of a read cache page, in bytes. Rounds down to power of 2. /// public long ReadCachePageSize = 1 << 25; /// - /// Total size of read cache, in bytes. Rounds down to power of 2. + /// Main-log circular-buffer size if nonzero; rounds down to a power of 2 and errors if that does not allow ReadCachePageSize. /// - public long ReadCacheMemorySize = 1L << 34; + /// + /// If zero, calculate it from and , which is also the max if both this and are nonzero. + /// + public int ReadCachePageCount = 0; /// - /// Fraction of log head (in memory) used for second chance - /// copy to tail. This is (1 - MutableFraction) for the - /// underlying log. + /// Fraction of log head (in memory) used for second chance copy to tail. This is (1 - MutableFraction) for the underlying log. /// public double ReadCacheSecondChanceFraction = 0.1; @@ -129,6 +143,16 @@ public sealed class KVSettings : IDisposable /// public StateMachineDriver StateMachineDriver = null; + /// + /// Maximum size of a key stored inline in the in-memory portion of the main log for both allocators. + /// + public int MaxInlineKeySize = 1 << LogSettings.kDefaultMaxInlineKeySizeBits; + + /// + /// Maximum size of a value stored inline in the in-memory portion of the main log for . + /// + public int MaxInlineValueSize = 1 << LogSettings.kDefaultMaxInlineValueSizeBits; + /// /// Create default configuration settings for TsavoriteKV. You need to create and specify LogDevice /// explicitly with this API. @@ -147,6 +171,7 @@ public KVSettings() { } /// /// Base directory (without trailing path separator) /// Whether to delete base directory on dispose. This option prevents later recovery. + /// /// public KVSettings(string baseDir, bool deleteDirOnDispose = false, ILoggerFactory loggerFactory = null, ILogger logger = null) { @@ -157,9 +182,6 @@ public KVSettings(string baseDir, bool deleteDirOnDispose = false, ILoggerFactor this.baseDir = baseDir; LogDevice = baseDir == null ? new NullDevice() : Devices.CreateLogDevice(baseDir + "/hlog.log", deleteOnClose: deleteDirOnDispose); - if (!Utility.IsBlittable() || !Utility.IsBlittable()) - ObjectLogDevice = baseDir == null ? new NullDevice() : Devices.CreateLogDevice(baseDir + "/hlog.obj.log", deleteOnClose: deleteDirOnDispose); - CheckpointDir = baseDir == null ? null : baseDir + "/checkpoints"; } @@ -180,7 +202,7 @@ public void Dispose() /// public override string ToString() { - var retStr = $"index: {Utility.PrettySize(IndexSize)}; log memory: {Utility.PrettySize(MemorySize)}; log page: {Utility.PrettySize(PageSize)}; log segment: {Utility.PrettySize(SegmentSize)}"; + var retStr = $"index: {Utility.PrettySize(IndexSize)}; log memory: {Utility.PrettySize(LogMemorySize)}; log page: {Utility.PrettySize(PageSize)}; log segment: {Utility.PrettySize(SegmentSize)}"; retStr += $"; log device: {(LogDevice == null ? "null" : LogDevice.GetType().Name)}"; retStr += $"; obj log device: {(ObjectLogDevice == null ? "null" : ObjectLogDevice.GetType().Name)}"; retStr += $"; mutable fraction: {MutableFraction};"; @@ -205,33 +227,33 @@ internal static long SetIndexSizeFromCacheLines(long cacheLines) => cacheLines * 64; internal LogSettings GetLogSettings() - { - return new LogSettings + => new() { ReadCopyOptions = ReadCopyOptions, LogDevice = LogDevice, ObjectLogDevice = ObjectLogDevice, - MemorySizeBits = Utility.NumBitsPreviousPowerOf2(MemorySize), + MemorySize = LogMemorySize, PageSizeBits = Utility.NumBitsPreviousPowerOf2(PageSize), + PageCount = PageCount, SegmentSizeBits = Utility.NumBitsPreviousPowerOf2(SegmentSize), + ObjectLogSegmentSizeBits = Utility.NumBitsPreviousPowerOf2(ObjectLogSegmentSize), MutableFraction = MutableFraction, - MinEmptyPageCount = MinEmptyPageCount, PreallocateLog = PreallocateLog, - ReadCacheSettings = GetReadCacheSettings() + ReadCacheSettings = GetReadCacheSettings(), + MaxInlineKeySizeBits = Utility.NumBitsPreviousPowerOf2(MaxInlineKeySize), + MaxInlineValueSizeBits = Utility.NumBitsPreviousPowerOf2(MaxInlineValueSize) }; - } private ReadCacheSettings GetReadCacheSettings() - { - return ReadCacheEnabled ? - new ReadCacheSettings + => ReadCacheEnabled ? + new() { - MemorySizeBits = Utility.NumBitsPreviousPowerOf2(ReadCacheMemorySize), + MemorySize = ReadCacheMemorySize, PageSizeBits = Utility.NumBitsPreviousPowerOf2(ReadCachePageSize), + PageCount = ReadCachePageCount, SecondChanceFraction = ReadCacheSecondChanceFraction } : null; - } internal CheckpointSettings GetCheckpointSettings() { diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogAddress.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogAddress.cs new file mode 100644 index 00000000000..18cadd62662 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogAddress.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#pragma warning disable IDE1006 // Naming Styles: Must begin with uppercase letter + +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// Static utility class for manipulating logical addresses. + public static class LogAddress + { + /// Address is 48 bits, with the top bit being the readcache indicator. + public const int kAddressBits = 48; + /// Mask off the address from a long; e.g. is 48 bits, with the top bit being the readcache indicator. + public const long kAddressBitMask = (1L << kAddressBits) - 1; + + // Get the absolute address by masking out the address type bits. + internal const long kAbsoluteAddressBitMask = ((1L << kAddressBits) - 1) & ~RecordInfo.kIsReadCacheBitMask; + + /// Invalid record logical address; used for initialization. Zero means an IsNull RecordInfo is Invalid. + public const long kInvalidAddress = 0L; + /// Invalid record logical address used for some specific initializations. + public const long kTempInvalidAddress = 1L; + + /// First valid address in the log; ensures space for page header and that 0 and 1 are never valid addresses. + public const long FirstValidAddress = PageHeader.Size; + + /// The max valid address is the in-memory mask (which is greater than the on-disk mask) and the full absolute address range. + public const long MaxValidAddress = kAbsoluteAddressBitMask; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool IsReadCache(long address) => ((ulong)address & RecordInfo.kIsReadCacheBitMask) == RecordInfo.kIsReadCacheBitMask; + + /// Get the absolute address (no readcache bit) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long AbsoluteAddress(long address) => address & kAbsoluteAddressBitMask; + + /// Utility shared between AllocatorBase and ScanIteratorBase + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long GetPageOfAddress(long logicalAddress, int logPageSizeBits) => AbsoluteAddress(logicalAddress) >> logPageSizeBits; + + /// Utility shared between AllocatorBase and ScanIteratorBase + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long GetLogicalAddressOfStartOfPage(long page, int logPageSizeBits) => page << logPageSizeBits; + + /// Pretty-print the address + public static string AddressString(long address) + { + var absoluteAddress = AbsoluteAddress(address); + if (IsReadCache(address)) + return $"rc:{absoluteAddress}"; + if (address == kInvalidAddress) + return "kInvalid"; + if (address == kTempInvalidAddress) + return "kTempInvalid"; + return $"log:{address}"; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSettings.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSettings.cs index 2e5114e4208..129127738a5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSettings.cs @@ -11,20 +11,47 @@ namespace Tsavorite.core internal class LogSettings { /// Minimum number of bits for a page size - public const int kMinPageSizeBits = 6; + public const int kMinPageSizeBits = 6; // 64B /// Maximum number of bits for a page size - public const int kMaxPageSizeBits = 30; + public const int kMaxPageSizeBits = 30; // 1TB - /// Minimum number of bits for a segment (segments consist of one or more pages) - public const int kMinSegmentSizeBits = kMinPageSizeBits; - /// Maximum number of bits for a page size (segments consist of one or more pages) + /// Minimum number of bits for a main-log segment (segments consist of one or more pages) + public const int kMinMainLogSegmentSizeBits = kMinPageSizeBits; + /// Minimum number of bits for a segment (segments consist of one or more pages). This minimum size is also the size of the buffer, + /// so the segment must be a multiple of this (which is guaranteed as both are powers of 2, as long as this minimum is observed). + /// During flush we may create multiple buffers, depending on the degree of parallelism allowed by page concurrency and . + public const int kMinObjectLogSegmentSizeBits = 22; // 4MB + /// Maximum number of bits for a main-log or object-log segment (segments consist of one or more pages). This is also the size of the read/write buffers + /// for object serialization to the object log. public const int kMaxSegmentSizeBits = 62; /// Minimum number of bits for the size of the in-memory portion of the log - public const int kMinMemorySizeBits = kMinSegmentSizeBits; + public const int kMinMemorySizeBits = kMinPageSizeBits; /// Maximum number of bits for the size of the in-memory portion of the log public const int kMaxMemorySizeBits = kMaxSegmentSizeBits; + /// Minimum per flush operation. Must be a power of 2 + public const int kMinFlushBuffers = 2; + /// Maximum per flush operation. Must be a power of 2 + public const int kMaxFlushBuffers = 64; + + /// Minimum per flush operation. Must be a power of 2 + public const int kMinDeserializationBuffers = 2; + /// Maximum per flush operation. Must be a power of 2 + public const int kMaxDeserializationBuffers = 64; + + /// Maximum size of a string (key or value) is 512MB + public const int kMaxStringSizeBits = 29; // 512MB + + /// Default number of bits for the size of an inline (not overflow) key + public const int kDefaultMaxInlineKeySizeBits = kLowestMaxInlineSizeBits + 1; // 128B + + /// Default number of bits for the size of an inline (not overflow) value, for + public const int kDefaultMaxInlineValueSizeBits = kMinPageSizeBits + 6; // 4KB + + /// Minimum number of bits for the size of an overflow (int inline) key or value + public const int kLowestMaxInlineSizeBits = kMinPageSizeBits; // 64B + /// /// Device used for main hybrid log /// @@ -36,24 +63,30 @@ internal class LogSettings public IDevice ObjectLogDevice; /// - /// Size of a segment (group of pages), in bits + /// Total size of in-memory part of log, in bytes. Does not need to be a power of 2 + /// + public long MemorySize = 1L << 34; + + /// + /// Size of a page in bits /// public int PageSizeBits = 25; /// - /// Size of a segment (group of pages), in bits + /// Number of pages in the circular buffer, rounded down to nearest power of 2. /// - public int SegmentSizeBits = 30; + /// If 0, it is calculated from and , which is also the max if both this and are nonzero. + public int PageCount = 0; /// - /// Total size of in-memory part of log, in bits + /// Size of a segment (group of pages) in the main log, in bits /// - public int MemorySizeBits = 34; + public int SegmentSizeBits = 30; // 1GB /// - /// Controls how many pages should be empty to account for non-power-of-two-sized log + /// Size of a segment (group of pages) in the object log, in bits /// - public int MinEmptyPageCount = 0; + public int ObjectLogSegmentSizeBits = 33; // 8GB /// /// Fraction of log marked as mutable (in-place updates) @@ -66,8 +99,7 @@ internal class LogSettings public ReadCopyOptions ReadCopyOptions; /// - /// Settings for optional read cache - /// Overrides the "copy reads to tail" setting + /// Settings for optional read cache. Overrides the "copy reads to tail" setting. /// public ReadCacheSettings ReadCacheSettings = null; @@ -75,5 +107,29 @@ internal class LogSettings /// Whether to preallocate the entire log (pages) in memory /// public bool PreallocateLog = false; + + /// + /// Maximum size of a key stored inline in the in-memory portion of the main log for both allocators. + /// + public int MaxInlineKeySizeBits = kDefaultMaxInlineKeySizeBits; + + /// + /// Maximum size of a value stored inline in the in-memory portion of the main log for . + /// + public int MaxInlineValueSizeBits = kDefaultMaxInlineValueSizeBits; + + /// + /// Number of page buffers during a Flush operation on a page or portion of a page. There may be multiple sets of buffers at any given time, + /// depending on page parallelism. Must be a power of 2. + /// + /// Validated for all allocators, but only used by . + public int NumberOfFlushBuffers = 4; + + /// + /// Number of page buffers during a Flush operation on a page or portion of a page. There may be multiple sets of buffers at any given time, + /// depending on page parallelism. Must be a power of 2. + /// + /// Validated for all allocators, but only used by . + public int NumberOfDeserializationBuffers = 4; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs index d24f311ea66..beaefe8cf1b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs @@ -6,196 +6,254 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using static Tsavorite.core.Utility; namespace Tsavorite.core { - /// Interface for calculating the size of the log - /// Type of key - /// Type of value - public interface ILogSizeCalculator + /// + /// Type-free base class for hybrid log memory allocator. Contains utility methods that do not need type args and are not performance-critical + /// so can be virtual. + /// + public class LogSizeTracker { - /// Calculates the size of a log record - /// Information about the record - /// The key - /// The value - /// The size of the record - long CalculateRecordSize(RecordInfo recordInfo, TKey key, TValue value); - } - - public enum LogOperationType - { - Deserialize - } - - public class LogOperationObserver : IObserver> - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - where TLogSizeCalculator : ILogSizeCalculator - { - private readonly LogSizeTracker logSizeTracker; - private readonly LogOperationType logOperationType; - - public LogOperationObserver(LogSizeTracker logSizeTracker, LogOperationType logOperationType) - { - this.logSizeTracker = logSizeTracker; - this.logOperationType = logOperationType; - } - - public void OnCompleted() { } - - public void OnError(Exception error) { } - - public void OnNext(ITsavoriteScanIterator records) - { - long size = 0; - while (records.GetNext(out RecordInfo info, out TKey key, out TValue value)) - { - Debug.Assert(key != null); - - size += logSizeTracker.LogSizeCalculator.CalculateRecordSize(info, key, value); - } + /// + /// The number of seconds to timeout the wait on . Useful for ensuring that we don't + /// miss a check due to non-atomicity of updating size, determining it is beyond budget, and signaling the event. + /// + public static readonly int ResizeTaskDelaySeconds = 10; - if (size == 0) - return; + /// Target size must be at least this many pages; this gives us (at least a little) room for heap allocations in a minimum of + /// pages. + public const int MinTargetPageCount = MinResizeTargetPageCount * 2; - if (logOperationType == LogOperationType.Deserialize) - { - logSizeTracker.IncrementSize(size); - } - } + /// When resizing we must preserve at least this many pages + public const int MinResizeTargetPageCount = 2; } /// Tracks and controls size of log - /// Type of key - /// Type of value /// /// - /// Type of the log size calculator - public class LogSizeTracker : IObserver> - where TLogSizeCalculator : ILogSizeCalculator - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public sealed class LogSizeTracker : LogSizeTracker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public static readonly int ResizeTaskDelaySeconds = 10; + /// + /// The event to be signaled when an call detects we're over budget. + /// + private CompletionEvent resizeTaskEvent; + + /// The current heap size of the log + private ConcurrentCounter heapSize; - private ConcurrentCounter logSize; - private long lowTargetSize; - private long highTargetSize; - public TLogSizeCalculator LogSizeCalculator; private readonly ILogger logger; - internal LogAccessor logAccessor; + /// Memory usage at which to trigger trimming + private long highTargetSize; + /// Memory usage at which to stop trimming once started + private long lowTargetSize; + + internal LogAccessor logAccessor; /// Indicates whether resizer task has been stopped - public volatile bool Stopped; + enum RunState : int { NotStarted, Running, StopRequested, Stopped }; + /// The integer value of the current , for Interlocked operations. Indicates whether resizer task has been stopped + volatile int runState; - internal Action PostEmptyPageCountIncrease { get; set; } = (int count) => { }; + /// Indicates whether resizer task has been stopped + public bool IsStopped => runState == (int)RunState.Stopped; - internal Action PostEmptyPageCountDecrease { get; set; } = (int count) => { }; + /// + /// Callback for when we have trimmed memory, such as by shifting headAddress to close records and/or evicting pages. + /// Passes the current number of allocated log pages and the headAddress. + /// + /// Currently used for tests. + internal Action PostMemoryTrim { get; set; } = (allocatedPageCount, headAddress) => { }; /// Total size occupied by log, including heap - public long TotalSizeBytes => logAccessor.MemorySizeBytes + logSize.Total; + public long TotalSize => logAccessor.MemorySizeBytes + heapSize.Total; - /// Size of log heap memory - public long LogHeapSizeBytes => logSize.Total; + /// Size of log heap memory only + public long LogHeapSizeBytes => heapSize.Total; /// Target size for the hybrid log memory utilization - public long TargetSize => (highTargetSize + lowTargetSize) / 2; + public long TargetSize { get; private set; } + + /// High and low deltas for + public (long high, long low) TargetDeltaRange => (highTargetSize, lowTargetSize); + + /// + public override string ToString() + { + return $"{runState}; TargetSize: [{TargetSize}, hi: {highTargetSize}, lo: {lowTargetSize}]; TotalSize: [{TotalSize}, Heap: {heapSize.Total}];" + + $" isOver: [{IsBeyondSizeLimit}, canEvict {IsBeyondSizeLimitAndCanEvict}]; AllocPgCt: {logAccessor.AllocatedPageCount}; PgSize {logAccessor.allocatorBase.PageSize}"; + } + + /// Return true if the total size is outside the target plus delta + public bool IsBeyondSizeLimit => TotalSize > highTargetSize; + + /// Return true if the total size is outside the target plus delta *and* we have pages we can (partially or completely) evict + /// If true, we are allocating a new page. Otherwise, we are called when adding or growing a new + /// This should be used only for non-Recovery, because Recovery does not set up HeadAddress and TailAddress before this is called. + public bool IsBeyondSizeLimitAndCanEvict(bool addingPage = false) + { + var headPage = logAccessor.allocatorBase.GetPage(logAccessor.allocatorBase.HeadAddress); + var tailPage = logAccessor.allocatorBase.GetPage(logAccessor.allocatorBase.UnstableGetTailAddress(out _)); + + // The number of pages we have is untilPage - headPage + 1. If we're called here when allocating a new page, see if the new page + // would put us over the maximum count. + var numPages = (int)(tailPage - headPage + 1); + if (addingPage && numPages == logAccessor.allocatorBase.MaxAllocatedPageCount) + return true; + + // Otherwise, we need at least MinResizeTargetPageCount to be able to evict anything. + return (TotalSize > highTargetSize) && numPages > MinResizeTargetPageCount; + } + + /// Return true if the total size plus the size needed for the requested number of pages to read is outside the target plus delta *and* + /// we have pages we can (partially or completely) evict + /// This is called by Recovery. + public bool IsBeyondSizeLimitToReadPages(int numPagesToRead) => TotalSize + (numPagesToRead * logAccessor.allocatorBase.PageSize) > highTargetSize; /// Creates a new log size tracker /// Hybrid log accessor - /// Size calculator /// Target size for the hybrid log memory utilization - /// Delta from target size to maintain memory utilization + /// Delta above the target size at which to trigger hybrid log memory usage trimming + /// Delta below the target size at which to stop trimming hybrid log memory usage once started /// - public LogSizeTracker(LogAccessor logAccessor, TLogSizeCalculator logSizeCalculator, long targetSize, long delta, ILogger logger) + public LogSizeTracker(LogAccessor logAccessor, long targetSize, long highDelta, long lowDelta, ILogger logger) { Debug.Assert(logAccessor != null); - Debug.Assert(logSizeCalculator != null); - Debug.Assert(delta >= 0); - Debug.Assert(targetSize > delta); this.logAccessor = logAccessor; - logSize = new ConcurrentCounter(); - this.UpdateTargetSize(targetSize, delta); - this.LogSizeCalculator = logSizeCalculator; + heapSize = new ConcurrentCounter(); + resizeTaskEvent = new(); this.logger = logger; - Stopped = false; + runState = (int)RunState.NotStarted; + UpdateTargetSize(targetSize, highDelta, lowDelta); } - /// - /// Starts the log size tracker - /// NOTE: Not thread safe to start multiple times - /// - /// - public void Start(CancellationToken token) + /// Starts the log size tracker + /// NOTE: Not thread safe to start multiple times + /// + public void Start(CancellationToken cancellationToken) { - Debug.Assert(Stopped == false); - Task.Run(() => ResizerTask(token)); + Debug.Assert(runState == (int)RunState.NotStarted, "Cannot restart LogSizeTracker"); + resizeTaskEvent.Initialize(); + runState = (int)RunState.Running; + _ = Task.Run(() => ResizerTask(cancellationToken), cancellationToken); + } + + /// Stop the resizer task + public void Stop(bool wait = false) + { + var prevState = Interlocked.CompareExchange(ref runState, (int)RunState.StopRequested, (int)RunState.Running); + if (prevState == (int)RunState.Running) + { + // This Set() will wake up the task and it will detect StopRequested and call OnStopped(). + resizeTaskEvent.Set(); + while (wait && !IsStopped) + _ = Thread.Yield(); + } + } + + void OnStopped() + { + _ = Interlocked.Exchange(ref runState, (int)RunState.Stopped); + resizeTaskEvent.Dispose(); + resizeTaskEvent = default; } /// /// Update target size for the hybrid log memory utilization /// - /// The target size - /// Delta from the target size - public void UpdateTargetSize(long targetSize, long delta) + /// The target size + /// Delta above the target size at which to trigger trimming + /// Delta below the target size at which to stop trimming once started + public void UpdateTargetSize(long newTargetSize, long highDelta, long lowDelta) { - Debug.Assert(delta >= 0); - Debug.Assert(targetSize > delta); - lowTargetSize = targetSize - delta; - highTargetSize = targetSize + delta; - logger?.LogInformation("Target size updated to {targetSize} with delta {delta}", targetSize, delta); + Debug.Assert(highDelta >= 0); + Debug.Assert(lowDelta >= 0); + Debug.Assert(newTargetSize > highDelta); + Debug.Assert(newTargetSize > lowDelta); + + if (newTargetSize < logAccessor.allocatorBase.PageSize * MinTargetPageCount) + throw new TsavoriteException($"Target size must be at least {MinTargetPageCount} pages"); + + var shrink = newTargetSize < TargetSize; + TargetSize = newTargetSize; + highTargetSize = newTargetSize + highDelta; + lowTargetSize = newTargetSize - lowDelta; + logger?.LogInformation("Target size updated to {targetSize} with highDelta {highDelta}, lowDelta {lowDelta}", newTargetSize, highDelta, lowDelta); + + // Only signal if we are shrinking; growth is handled normally as we add pages and records. + if (shrink) + resizeTaskEvent.Set(); } - public bool IsSizeBeyondLimit => TotalSizeBytes > highTargetSize; - - /// Callback on allocator completion - public void OnCompleted() { } - - /// Callback on allocator error - public void OnError(Exception error) { } - - /// Callback on allocator evicting a page to disk - public void OnNext(ITsavoriteScanIterator records) + /// Adds size to the tracked total count + public void IncrementSize(long size) { - long size = 0; - while (records.GetNext(out RecordInfo info, out TKey key, out TValue value)) + if (size != 0) { - Debug.Assert(key != null); - - size += LogSizeCalculator.CalculateRecordSize(info, key, value); + heapSize.Increment(size); + if (size > 0 && IsBeyondSizeLimitAndCanEvict()) + resizeTaskEvent.Set(); + Debug.Assert(size > 0 || heapSize.Total >= 0, $"HeapSize.Total should be >= 0 but is {heapSize.Total} in Resize"); } - - if (size == 0) return; - - IncrementSize(-size); // Reduce size as records are being evicted } - /// Adds size to the tracked total count - /// Size to add - public void IncrementSize(long size) + /// Adds the size to the tracked total count. + public void UpdateSize(in TSourceLogRecord logRecord, bool add) + where TSourceLogRecord : ISourceLogRecord { - logSize.Increment(size); + var size = MemoryUtils.CalculateHeapMemorySize(in logRecord); + if (size != 0) + { + if (add) + { + heapSize.Increment(size); + if (IsBeyondSizeLimitAndCanEvict()) + resizeTaskEvent.Set(); + } + else + { + // Nothing needed if we are decreasing. + heapSize.Increment(-size); + Debug.Assert(heapSize.Total >= 0, $"HeapSize.Total should be >= 0 but is {heapSize.Total} in UpdateSize"); + } + } } + /// Called when the caller has determined we are over budget, to signal the event. + public void Signal() => resizeTaskEvent.Set(); + /// /// Performs resizing by waiting for an event that is signaled whenever memory utilization changes. /// This is invoked on the threadpool to avoid blocking calling threads during the resize operation. /// - async Task ResizerTask(CancellationToken token) + async Task ResizerTask(CancellationToken cancellationToken) { while (true) { try { - await Task.Delay(TimeSpan.FromSeconds(ResizeTaskDelaySeconds), token); - ResizeIfNeeded(token); + // Note: CompletionEvent functions as an AutoResetEvent, so any signals that arrive between + // these calls to WaitAsync will be lost. ResizeIfNeeded retries as long as we are over budget, + // but there is still a chance we'll miss a growth+signal between that check and the next WaitAsync. + // The timeout mitigates this but it would be better to find an awaitable ManualResetEvent. + await resizeTaskEvent.WaitAsync(TimeSpan.FromSeconds(ResizeTaskDelaySeconds), cancellationToken).ConfigureAwait(false); + if (runState == (int)RunState.Running) + ResizeIfNeeded(cancellationToken); + if (runState != (int)RunState.Running) + { + OnStopped(); + return; + } } catch (OperationCanceledException) { logger?.LogTrace("Log resize task has been cancelled."); - Stopped = true; + OnStopped(); return; } catch (Exception e) @@ -205,50 +263,129 @@ async Task ResizerTask(CancellationToken token) } } - /// - /// Adjusts the log size to maintain its size within the range of +/- delta of the target size. - /// It does so by adjusting the number of empty pages in the underlying log. Also, it does this by - /// incrementing/decrementing the empty page count by 1 at a time to avoid large jumps in memory utilization. - /// - private void ResizeIfNeeded(CancellationToken token) + private bool DetermineEvictionRange(long currentSize, CancellationToken cancellationToken, out long headAddress, + ref int allocatedPageCount, out long estimatedHeapTrimmedSize) { - // Monitor the heap size - if (logSize.Total > highTargetSize) + // We know we are oversize so we calculate how much we need to trim to get to lowTargetSize. + var overSize = currentSize - lowTargetSize; + estimatedHeapTrimmedSize = 0L; + + var allocator = logAccessor.allocatorBase; + headAddress = allocator.HeadAddress; + var headPage = allocator.GetPage(headAddress); + var untilAddress = allocator.UnstableGetTailAddress(out _); + var untilPage = allocator.GetPage(untilAddress); + + // The number of pages we have is untilPage - headPage + 1. + if (untilPage - headPage + 1 <= MinResizeTargetPageCount) + return false; + untilAddress = allocator.GetLogicalAddressOfStartOfPage(untilPage - MinResizeTargetPageCount + 1); + + // If there is nothing to trim from the heap, we can just do math to advance HA. + if (heapSize.Total == 0) { - logger?.LogDebug("Heap size {totalLogSize} > target {highTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, highTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); - while (logSize.Total > highTargetSize && - logAccessor.EmptyPageCount < logAccessor.MaxEmptyPageCount) + var evictableSize = untilAddress - headAddress; + var isComplete = overSize <= evictableSize; + if (!isComplete) + overSize = evictableSize; + headAddress = RoundUp(headAddress + overSize, Constants.kRecordAlignment); + + // Scan from head of page to snap headAddress to the next record boundary. + var pageIndex = allocator.GetPage(headAddress); + var pageStartAddress = allocator.GetLogicalAddressOfStartOfPage(pageIndex); + var offset = headAddress - pageStartAddress; + if (offset <= PageHeader.Size) + headAddress = pageStartAddress; + else { - token.ThrowIfCancellationRequested(); - - if (logAccessor.AllocatedPageCount > logAccessor.BufferSize - logAccessor.EmptyPageCount + 1) + var currentAddress = pageStartAddress + PageHeader.Size; + var physicalAddress = allocator.GetPhysicalAddress(currentAddress); + while (currentAddress < headAddress) { - return; // wait for allocation to stabilize + var allocatedSize = new LogRecord(physicalAddress).AllocatedSize; + currentAddress += allocatedSize; + physicalAddress += allocatedSize; } - - logAccessor.EmptyPageCount++; - PostEmptyPageCountIncrease(logAccessor.EmptyPageCount); - logger?.LogDebug("Increasing empty page count to {EmptyPageCount}", logAccessor.EmptyPageCount); } + + allocatedPageCount -= (int)(allocator.GetPage(headAddress) - headPage); + return isComplete; } - else if (logSize.Total < lowTargetSize) + + // This will iterate until iterator.CurrentAddress == untilAddress + using var iterator = logAccessor.Scan(headAddress, untilAddress); + allocatedPageCount = allocator.AllocatedPageCount; + var pageTrimmedSize = 0L; + while (estimatedHeapTrimmedSize + pageTrimmedSize < overSize && iterator.GetNext() && !IsStopped) { - logger?.LogDebug("Heap size {totalLogSize} < target {lowTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, lowTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); - while (logSize.Total < lowTargetSize && - logAccessor.EmptyPageCount > logAccessor.MinEmptyPageCount) + cancellationToken.ThrowIfCancellationRequested(); + estimatedHeapTrimmedSize += iterator.CalculateHeapMemorySize(); + + // If we've crossed a page boundary, we can subtract the pagesize as well. + var currentPage = allocator.GetPage(iterator.CurrentAddress); + if (currentPage > headPage) { - token.ThrowIfCancellationRequested(); + headPage = currentPage; + --allocatedPageCount; + pageTrimmedSize += allocator.PageSize; + } + } - if (logAccessor.AllocatedPageCount < logAccessor.BufferSize - logAccessor.EmptyPageCount - 1) - { - return; // wait for allocation to stabilize - } + // iterator.NextAddress is the end of the last-processed record; if we did not advance far enough to clear all the oversize space + // it is the start of the next record we would have processed (and probably equal to untilAddress). In both cases it is how far we + // can evict to, and because it is the next address we've not yet evaluated whether it's crossed the page boundary; do that here. + headAddress = iterator.NextAddress; - logAccessor.EmptyPageCount--; - PostEmptyPageCountDecrease(logAccessor.EmptyPageCount); - logger?.LogDebug("Decreasing empty page count to {EmptyPageCount}", logAccessor.EmptyPageCount); - } + // Return whether we could satisfy the resize request; for Recovery, we may need to wait on flush. + return estimatedHeapTrimmedSize + pageTrimmedSize >= overSize; + } + + /// + /// Adjusts the log size to maintain its size within the range of highTargetSize and lowTargetSize. + /// + /// True if resize not needed or was complete, else false (need to wait for evictions, possible with flushes before that) + private void ResizeIfNeeded(CancellationToken cancellationToken) + { + // Loop to decrease size. These variables retain the values they acquired during the last loop iteration. + var currentSize = TotalSize; + if (currentSize <= highTargetSize) + return; + + long headAddress, estimatedHeapTrimmedSize, readOnlyAddress; + var isComplete = false; + var allocatedPageCount = logAccessor.AllocatedPageCount; + logger?.LogDebug("Heap size {totalLogSize} > target {highTargetSize}. Alloc: {AllocatedPageCount} BufferSize: {BufferSize}", heapSize.Total, highTargetSize, allocatedPageCount, logAccessor.BufferSize); + + // Acquire the epoch long enough to calculate eviction ranges. + logAccessor.allocatorBase.epoch.Resume(); + try + { + // See how much we can evict from HeadAddress onwards. Ignore the return value that indicates whether this is complete; + // we calculate the new ROA up to MinTargetPageCount pages before TailAddress, and that's as far as we can go. + isComplete = DetermineEvictionRange(currentSize, cancellationToken, out headAddress, ref allocatedPageCount, out estimatedHeapTrimmedSize); + if (runState != (int)RunState.Running) + return; + + // Calculate new ReadOnlyAddress; if it hasn't changed then the ShiftReadOnlyAddress in logAccessor.ShiftHeadAddress will do nothing. + readOnlyAddress = logAccessor.allocatorBase.CalculateReadOnlyAddress(logAccessor.TailAddress, headAddress); + } + finally + { + logAccessor.allocatorBase.epoch.Suspend(); } + + // Release the epoch before calling this because ShiftAddresses will wait for the ROA flush to complete. We need that wait because + // ShiftHeadAddress caps the new HeadAddress at FlushedUntilAddress. Wait until the SHA eviction is complete to avoid going further over budget. + logAccessor.ShiftAddresses(readOnlyAddress, headAddress, waitForEviction: true); + + // Heap size subtraction is handled by the OnNext eviction callback (called during ShiftAddresses), + // which subtracts each record's CURRENT HeapMemorySize at eviction time. + Debug.Assert(heapSize.Total >= 0, $"HeapSize.Total should be >= 0 but is {heapSize.Total} in Resize"); + + // Calculate the number of trimmed pages and report the new expected AllocatedPageCount here, since our last iteration (which may have been the only one) + // would have returned isComplete and thus we didn't wait for the actual eviction. + PostMemoryTrim(allocatedPageCount, headAddress); + logger?.LogDebug("Decreased Allocated page count to {allocatedPageCount} and HeadAddress to {headAddress}; isComplete {isComplete}", allocatedPageCount, headAddress, isComplete); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationOptions.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationOptions.cs index 7482f377d2f..57f5426514d 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationOptions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationOptions.cs @@ -8,7 +8,7 @@ namespace Tsavorite.core /// /// Identifies which log regions records will be copied from to . This specification is /// evaluated in hierarchical order, from that on the TsavoriteKV ctor, which may be overridden by those in - /// .NewSession(), which may be overridden + /// .NewSession(), which may be overridden /// by those at the individual Read() level. /// public enum ReadCopyFrom : byte @@ -86,12 +86,12 @@ public struct ReadOptions /// /// Options for automatically copying immutable records on Read(). /// - public ReadCopyOptions CopyOptions { get; internal set; } + public ReadCopyOptions CopyOptions { get; set; } /// /// The hashcode of the key for this operation /// - public long? KeyHash { get; internal set; } + public long? KeyHash { get; set; } /// public override readonly string ToString() => $"copyOptions {{{CopyOptions}}}, keyHash {Utility.GetHashString(KeyHash)}"; @@ -105,7 +105,7 @@ public struct RMWOptions /// /// The hashcode of the key for this operation /// - public long? KeyHash { get; internal set; } + public long? KeyHash { get; set; } /// public override readonly string ToString() => $"keyHash {Utility.GetHashString(KeyHash)}"; @@ -119,7 +119,7 @@ public struct UpsertOptions /// /// The hashcode of the key for this operation /// - public long? KeyHash { get; internal set; } + public long? KeyHash { get; set; } /// public override readonly string ToString() => $"keyHash {Utility.GetHashString(KeyHash)}"; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationStatus.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationStatus.cs index 5a15ea78d6e..23a386feead 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationStatus.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/OperationStatus.cs @@ -37,11 +37,16 @@ internal enum OperationStatus /// CANCELED = StatusCode.Canceled, + /// + /// Operation was rejected because the record has a different type than expected. + /// + WRONG_TYPE = StatusCode.WrongType, + /// /// The maximum range that directly maps to the enumeration; the operation completed. /// This is an internal code to reserve ranges in the enumeration. /// - MAX_MAP_TO_COMPLETED_STATUSCODE = CANCELED, + MAX_MAP_TO_COMPLETED_STATUSCODE = WRONG_TYPE, // Not-completed Status codes @@ -58,8 +63,8 @@ internal enum OperationStatus RETRY_LATER, /// - /// I/O has been enqueued and the caller must go through or - /// , + /// I/O has been enqueued and the caller must go through or + /// , /// or one of the Async forms. /// RECORD_ON_DISK, @@ -72,7 +77,7 @@ internal enum OperationStatus /// /// Allocation failed, due to a need to flush pages. Clients do not see this status directly; they see . /// - /// For Sync operations we retry this as part of . + /// For Sync operations we retry this as part of . /// For Async operations we retry this as part of the ".Complete(...)" or ".CompleteAsync(...)" operation on the appropriate "*AsyncResult{}" object. /// /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/PendingContext.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/PendingContext.cs index bcb10637121..0b87b8d022f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/PendingContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/PendingContext.cs @@ -1,41 +1,77 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Diagnostics; using System.Runtime.CompilerServices; namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + using static Utility; + + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal struct PendingContext { // User provided information internal OperationType type; - internal IHeapContainer key; - internal IHeapContainer value; + internal readonly bool IsConditionalOp => type is OperationType.CONDITIONAL_INSERT or OperationType.CONDITIONAL_SCAN_PUSH; + + /// + /// DiskLogRecord carries a log record image. It is used for pending ConditionalCopy operations, where it is one of: + /// + /// A created by serializing from an in-memory + /// A retrieved from the disk, for operations such as Compact + /// + /// + internal DiskLogRecord diskLogRecord; + + /// The Key that was sent to this operation if it was RUMD. + internal ConditionallyHoistedKey requestKey; + /// The hash of if it is present. + internal long keyHash; + + /// The Input that was sent to this operation if it was RUMD. internal IHeapContainer input; + /// The Output to be returned from this operation if it was RUM. internal TOutput output; + /// The user Context that was sent to this operation if it was RUMD. internal TContext userContext; - internal long keyHash; - // Some additional information about the previous attempt + /// The id of this operation in the queue. internal long id; + + /// The logical address of the found record, if any; used to create . internal long logicalAddress; - internal long InitialLatestLogicalAddress; + + /// The logical address of the original record. Used by: + /// + /// ConditionalScanPush — retains the address of the record we will push to the caller + /// if it is not found later in the log; must be the one we + /// pass to request. + /// TryCopyToTail's PostCopyToTail trigger — when the source record is from disk + /// (HasMainLogSrc=false), this field carries the source logical address from the + /// compaction / CopyReadsToTail / ContinuePending caller. + /// + internal long originalAddress; + + /// The initial highest logical address of the search; used to limit search ranges when the pending operation completes (e.g. to see if a duplicate was inserted). + internal long initialLatestLogicalAddress; // operationFlags values internal ushort operationFlags; +#pragma warning disable IDE1006 // Naming Styles internal const ushort kNoOpFlags = 0; - internal const ushort kNoKey = 0x0001; - internal const ushort kIsAsync = 0x0002; - internal const ushort kIsReadAtAddress = 0x0004; + internal const ushort kIsNoKey = 0x0001; + internal const ushort kIsReadAtAddress = 0x0002; +#pragma warning restore IDE1006 // Naming Styles internal ReadCopyOptions readCopyOptions; // Two byte enums - internal WriteReason writeReason; // for ConditionalCopyToTail; one byte enum - internal RecordInfo recordInfo; internal long minAddress; internal long maxAddress; @@ -45,63 +81,169 @@ internal struct PendingContext // For RMW if an allocation caused the source record for a copy to go from readonly to below HeadAddress, or for any operation with CAS failure. internal long retryNewLogicalAddress; - internal ScanCursorState scanCursorState; + // Address of the initial entry in the hash chain upon start of Internal(RUMD). + internal long initialEntryAddress; + + internal ScanCursorState scanCursorState; + + /// + public override readonly string ToString() + { + var keyStr = !requestKey.IsEmpty ? SpanByte.ToShortString(requestKey.KeyBytes, 12) : ""; + var keyHashStr = GetHashString(keyHash); + return $"Type={type}, id={id}, reqKey={keyStr}, keyHash={keyHashStr}, IsSet={diskLogRecord.IsSet}, LA={logicalAddress}, InitLLA={initialLatestLogicalAddress}, MinA={minAddress}, MaxA={maxAddress}, ReadCopyOpt={readCopyOptions}"; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal PendingContext(long keyHash) => this.keyHash = keyHash; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal PendingContext(ReadCopyOptions sessionReadCopyOptions, ref ReadOptions readOptions, bool isAsync = false, bool noKey = false) + internal PendingContext(ReadCopyOptions sessionReadCopyOptions, ref ReadOptions readOptions) { - // The async flag is often set when the PendingContext is created, so preserve that. - operationFlags = (ushort)((noKey ? kNoKey : kNoOpFlags) | (isAsync ? kIsAsync : kNoOpFlags)); + operationFlags = kNoOpFlags; readCopyOptions = ReadCopyOptions.Merge(sessionReadCopyOptions, readOptions.CopyOptions); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal PendingContext(ReadCopyOptions readCopyOptions, bool isAsync = false, bool noKey = false) + internal PendingContext(ReadCopyOptions readCopyOptions) { - // The async flag is often set when the PendingContext is created, so preserve that. - operationFlags = (ushort)((noKey ? kNoKey : kNoOpFlags) | (isAsync ? kIsAsync : kNoOpFlags)); + operationFlags = kNoOpFlags; this.readCopyOptions = readCopyOptions; } - internal bool NoKey + internal readonly bool IsNoKey => (operationFlags & kIsNoKey) != 0; + internal void SetIsNoKey() => operationFlags |= kIsNoKey; + + internal readonly bool HasMinAddress => minAddress != kInvalidAddress; + + internal readonly bool IsReadAtAddress => (operationFlags & kIsReadAtAddress) != 0; + internal void SetIsReadAtAddress() => operationFlags |= kIsReadAtAddress; + + public void Dispose() { - readonly get => (operationFlags & kNoKey) != 0; - set => operationFlags = value ? (ushort)(operationFlags | kNoKey) : (ushort)(operationFlags & ~kNoKey); + if (diskLogRecord.IsSet) + diskLogRecord.Dispose(); + diskLogRecord = default; + requestKey = default; + input?.Dispose(); + input = default; } - internal readonly bool HasMinAddress => minAddress != Constants.kInvalidAddress; + #region Serialized Record Creation + /// + /// Serialize for Read and RMW operations; no Value is passed + /// + /// Record key + /// Input to the operation + /// Output from the operation + /// User context for the operation + /// Session functions wrapper for the operation + /// Allocator for backing storage + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void CopyInputsForReadOrRMW(TKey key, ref TInput input, ref TOutput output, TContext userContext, + TSessionFunctionsWrapper sessionFunctions, SectorAlignedBufferPool bufferPool) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + { + CopyKey(key, bufferPool, sessionFunctions); + + if (this.input == default) + { + if (typeof(TInput) == typeof(PinnedSpanByte)) + this.input = new SpanByteHeapContainer(Unsafe.As(ref input), sessionFunctions.Store.hlogBase.bufferPool) as IHeapContainer; + else + this.input = new StandardHeapContainer(ref input); + } + this.output = output; + sessionFunctions.ConvertOutputToHeap(ref input, ref this.output); + this.userContext = userContext; + } - internal bool IsAsync + /// Copy the passed key into our + internal void CopyKey(TKey key, SectorAlignedBufferPool bufferPool, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - readonly get => (operationFlags & kIsAsync) != 0; - set => operationFlags = value ? (ushort)(operationFlags | kIsAsync) : (ushort)(operationFlags & ~kIsAsync); + if (requestKey.IsEmpty) + requestKey = ConditionallyHoistedKey.Create(key, bufferPool); + else + Debug.Assert(sessionFunctions.Store.StoreFunctions.KeysEqual(requestKey, key), "pendingContext.requestKey should not change keys"); } - internal bool IsReadAtAddress + /// + /// Does an in-memory transfer of a record into the pending context. The transfer operates based on the implementation of the : + /// + /// If it is a , it comes from the log or possibly an iterator; we copy the inline record data into a + /// and then reassign its ObjectIds to the . + /// Otherwise it is a and we can transfer its into the local . + /// It will already have a in its contained . + /// + /// + /// The log record to be copied into the . This may be either in-memory or from disk IO + /// The memory pool used for allocating the space for inline data to be copied + /// The objectIdMap to reassign the objectIds to + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void CopyFrom(in TSourceLogRecord srcLogRecord, SectorAlignedBufferPool bufferPool, ObjectIdMap transientObjectIdMap) + where TSourceLogRecord : ISourceLogRecord { - readonly get => (operationFlags & kIsReadAtAddress) != 0; - set => operationFlags = value ? (ushort)(operationFlags | kIsReadAtAddress) : (ushort)(operationFlags & ~kIsReadAtAddress); + Debug.Assert(!diskLogRecord.IsSet, "Should not try to reset PendingContext.diskLogRecord"); + if (srcLogRecord.IsMemoryLogRecord) + { + ref var memoryLogRecord = ref srcLogRecord.AsMemoryLogRecordRef(); + diskLogRecord = DiskLogRecord.CopyFrom(in memoryLogRecord, bufferPool, transientObjectIdMap); + return; + } + + Debug.Assert(srcLogRecord.IsDiskLogRecord, $"Unknown SrcLogRecord implementation: {srcLogRecord}"); + ref var inputDiskLogRecord = ref srcLogRecord.AsDiskLogRecordRef(); + // If the inputDiskLogRecord owns its memory this will efficiently transfer it to the local diskLogRecord; otherwise it will deep copy. + diskLogRecord = DiskLogRecord.TransferFrom(ref inputDiskLogRecord, bufferPool); } - // RecordInfo is not used as such during the pending phase, so we reuse the space here. - internal long InitialEntryAddress + internal void TransferFrom(ref DiskLogRecord inputDiskLogRecord, SectorAlignedBufferPool bufferPool) { - readonly get => recordInfo.PreviousAddress; - set => recordInfo.PreviousAddress = value; + Debug.Assert(!diskLogRecord.IsSet, "Should not try to reset PendingContext.diskLogRecord"); + diskLogRecord = DiskLogRecord.TransferFrom(ref inputDiskLogRecord, bufferPool); } - public void Dispose() + #endregion // Serialized Record Creation + + #region Shortcuts to contained DiskLogRecord + public readonly DiskLogRecord DiskLogRecord { - key?.Dispose(); - key = default; - value?.Dispose(); - value = default; - input?.Dispose(); - input = default; + get + { + Debug.Assert(diskLogRecord.IsSet, "PendingContext.diskLogRecord must be set for 'DiskLogRecord'"); + return diskLogRecord; + } + } + + /// + public readonly RecordInfo Info + { + get + { + Debug.Assert(diskLogRecord.IsSet, "PendingContext.diskLogRecord must be set for 'Info'"); + return diskLogRecord.Info; + } + } + + /// + public readonly ReadOnlySpan Key + { + get + { + Debug.Assert(diskLogRecord.IsSet, "PendingContext.diskLogRecord must be set for 'Key'"); + return diskLogRecord.Key; + } } + #endregion Shortcuts to contained DiskLogRecord } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/ReadCacheSettings.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/ReadCacheSettings.cs index 9d16c2b2483..09f17a8ead4 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/ReadCacheSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/ReadCacheSettings.cs @@ -11,19 +11,23 @@ namespace Tsavorite.core internal class ReadCacheSettings { /// - /// Size of a segment (group of pages), in bits + /// Total size of in-memory part of log, in bytes + /// + public long MemorySize = 1L << 34; + + /// + /// Size of a page in bits /// public int PageSizeBits = 25; /// - /// Total size of in-memory part of log, in bits + /// Number of pages in the circular buffer, rounded down to nearest power of 2 /// - public int MemorySizeBits = 34; + /// If 0, it is calculated from and , which is also the max if both this and are nonzero. + public int PageCount = 25; /// - /// Fraction of log head (in memory) used for second chance - /// copy to tail. This is (1 - MutableFraction) for the - /// underlying log + /// Fraction of log head (in memory) used for second chance copy to tail. This is (1 - MutableFraction) for the underlying log /// public double SecondChanceFraction = 0.1; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs index 180dfbb0259..bfd7fa201b1 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordInfo.cs @@ -6,53 +6,86 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; -using static Tsavorite.core.Utility; namespace Tsavorite.core { - // RecordInfo layout (64 bits total): - // [VectorSet][Modified][InNewVersion][Filler][Dirty][ETag][Sealed][Valid][Tombstone][LLLLLLL] [RAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] - // where L = leftover, R = readcache, A = address +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + + // RecordInfo layout (64 bits total, high to low): + // Unused/Reserved bits (consume from Unused5 upward, so remaining stay contiguous): + // [Unused1][Unused2][Unused3][Unused4][Unused5] + // RecordInfo bits: + // [Sealed][Modified][InNewVersion][Valid][Tombstone] + // [HasExpiration][HasETag][ValueIsObject][ValueIsInline][KeyIsInline][HasFiller] + // LogAddress bits (where A = address): + // [R][AAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [AAAAAAAA] [StructLayout(LayoutKind.Explicit, Size = 8)] public struct RecordInfo { - const int kTotalSizeInBytes = 8; - const int kTotalBits = kTotalSizeInBytes * 8; - - // Previous address - internal const int kPreviousAddressBits = 48; - internal const long kPreviousAddressMaskInWord = (1L << kPreviousAddressBits) - 1; - - // Leftover bits (that were reclaimed from locking) - const int kLeftoverBitCount = 7; - - // Other marker bits. Unused* means bits not yet assigned; use the highest number when assigning - const int kTombstoneBitOffset = kPreviousAddressBits + kLeftoverBitCount; + public const int Size = sizeof(ulong); + +#pragma warning disable IDE1006 // Naming Styles: Must begin with uppercase letter + const int kTotalBits = Size * 8; + + // Other marker bits. Unused* means bits not yet assigned + const int kIsReadCacheBitOffset = kAddressBits - 1; + const int kHasFillerBitOffset = kIsReadCacheBitOffset + 1; + const int kKeyIsInlineBitOffset = kHasFillerBitOffset + 1; + const int kValueIsInlineBitOffset = kKeyIsInlineBitOffset + 1; + const int kValueIsObjectBitOffset = kValueIsInlineBitOffset + 1; + const int kHasETagBitOffset = kValueIsObjectBitOffset + 1; + const int kHasExpirationBitOffset = kHasETagBitOffset + 1; + const int kTombstoneBitOffset = kHasExpirationBitOffset + 1; const int kValidBitOffset = kTombstoneBitOffset + 1; - const int kSealedBitOffset = kValidBitOffset + 1; - const int kEtagBitOffset = kSealedBitOffset + 1; - const int kDirtyBitOffset = kEtagBitOffset + 1; - const int kFillerBitOffset = kDirtyBitOffset + 1; - const int kInNewVersionBitOffset = kFillerBitOffset + 1; + const int kInNewVersionBitOffset = kValidBitOffset + 1; const int kModifiedBitOffset = kInNewVersionBitOffset + 1; - const int kVectorSetBitOffset = kModifiedBitOffset + 1; - + const int kSealedBitOffset = kModifiedBitOffset + 1; + const int kUnused5BitOffset = kSealedBitOffset + 1; + const int kUnused4BitOffset = kUnused5BitOffset + 1; + const int kUnused3BitOffset = kUnused4BitOffset + 1; + const int kUnused2BitOffset = kUnused3BitOffset + 1; + const int kUnused1BitOffset = kUnused2BitOffset + 1; + + internal const long kIsReadCacheBitMask = 1L << kIsReadCacheBitOffset; + const long kHasFillerBitMask = 1L << kHasFillerBitOffset; + const long kKeyIsInlineBitMask = 1L << kKeyIsInlineBitOffset; + const long kValueIsInlineBitMask = 1L << kValueIsInlineBitOffset; + const long kValueIsObjectBitMask = 1L << kValueIsObjectBitOffset; + const long kHasETagBitMask = 1L << kHasETagBitOffset; + const long kHasExpirationBitMask = 1L << kHasExpirationBitOffset; const long kTombstoneBitMask = 1L << kTombstoneBitOffset; const long kValidBitMask = 1L << kValidBitOffset; - const long kSealedBitMask = 1L << kSealedBitOffset; - const long kETagBitMask = 1L << kEtagBitOffset; - const long kDirtyBitMask = 1L << kDirtyBitOffset; - const long kFillerBitMask = 1L << kFillerBitOffset; const long kInNewVersionBitMask = 1L << kInNewVersionBitOffset; const long kModifiedBitMask = 1L << kModifiedBitOffset; - const long kVectorSetBitMask = 1L << kVectorSetBitOffset; + const long kSealedBitMask = 1L << kSealedBitOffset; + const long kUnused5BitMask = 1L << kUnused5BitOffset; + const long kUnused4BitMask = 1L << kUnused4BitOffset; + const long kUnused3BitMask = 1L << kUnused3BitOffset; + const long kUnused2BitMask = 1L << kUnused2BitOffset; + const long kUnused1BitMask = 1L << kUnused1BitOffset; +#pragma warning restore IDE1006 // Naming Styles [FieldOffset(0)] private long word; // Used by routines to initialize a local recordInfo variable to serve as an initial source for srcRecordInfo, before we have // an in-memory address (or even know if the key will be found in-memory). - internal static RecordInfo InitialValid = new() { Valid = true, PreviousAddress = Constants.kTempInvalidAddress }; + internal static RecordInfo InitialValid = new(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordInfo() + { + Valid = true; + SetKeyAndValueInline(); + PreviousAddress = kTempInvalidAddress; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordInfo(long word) + { + this.word = word; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] public void WriteInfo(bool inNewVersion, long previousAddress) @@ -62,7 +95,7 @@ public void WriteInfo(bool inNewVersion, long previousAddress) // Otherwise, Scan could return partial records (e.g. a checkpoint was taken that flushed midway through the record update). // - Revivification sets Sealed; we need to preserve it here. // We'll clear both on successful CAS. - InitializeToSealedAndInvalid(); + InitializeNewRecord(); PreviousAddress = previousAddress; if (inNewVersion) SetIsInNewVersion(); @@ -73,14 +106,16 @@ public void WriteInfo(bool inNewVersion, long previousAddress) public void ClearBitsForDiskImages() { // A Sealed record may become current again during recovery if the RCU-inserted record was not written to disk during a crash. So clear that bit here. - word &= ~(kDirtyBitMask | kSealedBitMask); + // Preserve Key/ValueIsInline as they are always inline for DiskLogRecord. Preserve ValueIsObject to indicate whether a value object should be deserialized + // or if the value should remain inline (and possibly overflow if copied to a LogRecord). + word &= ~kSealedBitMask; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsClosedWord(long word) => (word & (kValidBitMask | kSealedBitMask)) != kValidBitMask; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool IsClosedOrTombstoned(ref OperationStatus internalStatus) + internal readonly bool IsClosedOrTombstoned(ref OperationStatus internalStatus) { if ((word & (kValidBitMask | kSealedBitMask | kTombstoneBitMask)) != kValidBitMask) { @@ -111,7 +146,7 @@ public bool TrySeal(bool invalidate) // If this fails for any reason it means another record is trying to modify (perhaps revivify) it, so return false to RETRY_LATER. // If invalidate, we in a situation such as revivification freelisting where we want to make sure that removing Seal will not leave // it eligible to be Scanned after Recovery. - long expected_word = word; + var expected_word = word; if (IsClosedWord(expected_word)) return false; var new_word = expected_word | kSealedBitMask; @@ -127,9 +162,10 @@ public bool TrySeal(bool invalidate) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryResetModifiedAtomic() { - for (int spinCount = Constants.kMaxLockSpins; ; Thread.Yield()) + var spinCount = Constants.kMaxLockSpins; + while (true) { - long expected_word = word; + var expected_word = word; if (IsClosedWord(expected_word)) return false; if ((expected_word & kModifiedBitMask) == 0) @@ -138,6 +174,7 @@ internal bool TryResetModifiedAtomic() return true; if (--spinCount <= 0) return false; + _ = Thread.Yield(); } } @@ -145,14 +182,14 @@ internal bool TryResetModifiedAtomic() public bool TryUpdateAddress(long expectedPrevAddress, long newPrevAddress) { var expected_word = word; - RecordInfo newRI = new() { word = expected_word }; + RecordInfo newRI = new(expected_word); if (newRI.PreviousAddress != expectedPrevAddress) return false; newRI.PreviousAddress = newPrevAddress; return expected_word == Interlocked.CompareExchange(ref word, newRI.word, expected_word); } - public readonly bool IsNull() => word == 0; + public readonly bool IsNull => word == 0; public readonly bool Tombstone { @@ -178,21 +215,6 @@ public bool Valid } } - public void ClearDirtyAtomic() - { - for (; ; Thread.Yield()) - { - long expected_word = word; // TODO: Interlocked.And is not supported in netstandard2.1 - if (expected_word == Interlocked.CompareExchange(ref word, expected_word & ~kDirtyBitMask, expected_word)) - break; - } - } - - public readonly bool Dirty - { - get => (word & kDirtyBitMask) > 0; - } - public bool Modified { readonly get => (word & kModifiedBitMask) > 0; @@ -203,18 +225,6 @@ public bool Modified } } - public readonly bool HasFiller - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => (word & kFillerBitMask) > 0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetHasFiller() => word |= kFillerBitMask; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ClearHasFiller() => word &= ~kFillerBitMask; - public readonly bool IsInNewVersion { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -224,13 +234,16 @@ public readonly bool IsInNewVersion [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SetIsInNewVersion() => word |= kInNewVersionBitMask; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetDirtyAndModified() => word |= kDirtyBitMask | kModifiedBitMask; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetDirty() => word |= kDirtyBitMask; + public void SetModified() => word |= kModifiedBitMask; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SetInvalid() => word &= ~kValidBitMask; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeToSealedAndInvalid() => word = kSealedBitMask; // Does not include kValidBitMask + public void InitializeNewRecord() + { + // Initialize to Sealed and Invalid (do not include kValidBitMask) and to Inline Key and Value so no Oversize or ObjectId is expected. + word = kSealedBitMask | kKeyIsInlineBitMask | kValueIsInlineBitMask; + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void UnsealAndValidate() => word = (word & ~kSealedBitMask) | kValidBitMask; [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -241,11 +254,12 @@ public readonly bool IsInNewVersion [MethodImpl(MethodImplOptions.AggressiveInlining)] public void SetInvalidAtomic() { - for (; ; Thread.Yield()) + while (true) { - long expected_word = word; // TODO: Interlocked.And is not supported in netstandard2.1 + var expected_word = word; if (expected_word == Interlocked.CompareExchange(ref word, expected_word & ~kValidBitMask, expected_word)) return; + _ = Thread.Yield(); } } @@ -260,36 +274,107 @@ public readonly bool Invalid public long PreviousAddress { [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get { return word & kPreviousAddressMaskInWord; } + readonly get { return word & kAddressBitMask; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - set { word = (word & ~kPreviousAddressMaskInWord) | (value & kPreviousAddressMaskInWord); } + set { word = (word & ~kAddressBitMask) | (value & kAddressBitMask); } } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetLength() => kTotalSizeInBytes; + public readonly bool HasETag => (word & kHasETagBitMask) != 0; + public void SetHasETag() => word |= kHasETagBitMask; + public void ClearHasETag() => word &= ~kHasETagBitMask; + + public readonly bool HasExpiration => (word & kHasExpirationBitMask) != 0; + public void SetHasExpiration() => word |= kHasExpirationBitMask; + public void ClearHasExpiration() => word &= ~kHasExpirationBitMask; + + public readonly bool HasOptionalFields => (word & (kHasETagBitMask | kHasExpirationBitMask)) != 0; + public readonly bool HasOptionalOrObjectFields => (word & (kKeyIsInlineBitMask | kValueIsInlineBitMask | kHasETagBitMask | kHasExpirationBitMask)) != (kKeyIsInlineBitMask | kValueIsInlineBitMask); + + // Note: KeyIsOveflow bit is not needed as it is the negation of KeyIsInline + public readonly bool KeyIsInline => (word & kKeyIsInlineBitMask) != 0; + public void SetKeyIsInline() => word |= kKeyIsInlineBitMask; + public void ClearKeyIsInline() => word &= ~kKeyIsInlineBitMask; + public readonly bool KeyIsOverflow => !KeyIsInline; + public void SetKeyIsOverflow() => word &= ~kKeyIsInlineBitMask; + + // Note: a ValueIsOverflow bit is not needed as it is the negation of (ValueIsInline | ValueIsObject) + public readonly bool ValueIsInline => (word & kValueIsInlineBitMask) != 0; + public void SetValueIsInline() => word = (word & ~kValueIsObjectBitMask) | kValueIsInlineBitMask; + public void ClearValueIsInline() => word &= ~kValueIsInlineBitMask; + + public readonly bool ValueIsObject => (word & kValueIsObjectBitMask) != 0; + public void SetValueIsObject() => word = (word & ~kValueIsInlineBitMask) | kValueIsObjectBitMask; + + public readonly bool HasFiller => (word & kHasFillerBitMask) != 0; + public void SetHasFiller() => word |= kHasFillerBitMask; + public void ClearHasFiller() => word &= ~kHasFillerBitMask; + + // Value "Overflow" is determined by lack of Inline and lack of Object + public readonly bool ValueIsOverflow => !ValueIsInline && !ValueIsObject; + public void SetValueIsOverflow() => word &= ~(kValueIsInlineBitMask | kValueIsObjectBitMask); + + public void SetKeyAndValueInline() => word = (word & ~kValueIsObjectBitMask) | kKeyIsInlineBitMask | kValueIsInlineBitMask; + + public readonly bool RecordIsInline => (word & (kKeyIsInlineBitMask | kValueIsInlineBitMask)) == (kKeyIsInlineBitMask | kValueIsInlineBitMask); - public bool VectorSet + public readonly bool RecordHasObjects => (word & (kKeyIsInlineBitMask | kValueIsInlineBitMask)) != (kKeyIsInlineBitMask | kValueIsInlineBitMask); + + internal bool IsReadCache + { + readonly get => (word & kIsReadCacheBitMask) != 0; + set => word = value ? word | kIsReadCacheBitMask : word & ~kIsReadCacheBitMask; + } + + internal bool Unused1 { - readonly get => (word & kVectorSetBitMask) != 0; - set => word = value ? word | kVectorSetBitMask : word & ~kVectorSetBitMask; + readonly get => (word & kUnused1BitMask) != 0; + set => word = value ? word | kUnused1BitMask : word & ~kUnused1BitMask; } - public bool ETag + internal bool Unused2 { - readonly get => (word & kETagBitMask) != 0; - set => word = value ? word | kETagBitMask : word & ~kETagBitMask; + readonly get => (word & kUnused2BitMask) != 0; + set => word = value ? word | kUnused2BitMask : word & ~kUnused2BitMask; } - public void SetHasETag() => word |= kETagBitMask; - public void ClearHasETag() => word &= ~kETagBitMask; + internal bool Unused3 + { + readonly get => (word & kUnused3BitMask) != 0; + set => word = value ? word | kUnused3BitMask : word & ~kUnused3BitMask; + } + + internal bool Unused4 + { + readonly get => (word & kUnused4BitMask) != 0; + set => word = value ? word | kUnused4BitMask : word & ~kUnused4BitMask; + } + + internal bool Unused5 + { + readonly get => (word & kUnused5BitMask) != 0; + set => word = value ? word | kUnused5BitMask : word & ~kUnused5BitMask; + } + + internal readonly int GetOptionalSize() + { + var size = HasETag ? LogRecord.ETagSize : 0; + if (HasExpiration) + size += LogRecord.ExpirationSize; + if (!RecordIsInline) + size += LogRecord.ObjectLogPositionSize; + return size; + } public override readonly string ToString() { - var paRC = IsReadCache(PreviousAddress) ? "(rc)" : string.Empty; static string bstr(bool value) => value ? "T" : "F"; - return $"prev {AbsoluteAddress(PreviousAddress)}{paRC}, valid {bstr(Valid)}, tomb {bstr(Tombstone)}, seal {bstr(IsSealed)}," - + $" mod {bstr(Modified)}, dirty {bstr(Dirty)}, fill {bstr(HasFiller)}, etag {bstr(ETag)}, vset {bstr(VectorSet)}"; + var keyString = KeyIsInline ? "inl" : "ovf"; + var valString = ValueIsInline ? "inl" : (ValueIsObject ? "obj" : "ovf"); + return $"prev {AddressString(PreviousAddress)}, valid {bstr(Valid)}, tomb {bstr(Tombstone)}, seal {bstr(IsSealed)}, rc {bstr(IsReadCache)}," + + $" mod {bstr(Modified)}, inv {bstr(IsInNewVersion)}, Key::{keyString}, Val::{valString}," + + $" ETag {bstr(HasETag)}, Expir {bstr(HasExpiration)}, Filler {bstr(HasFiller)}," + + $" Un1 {bstr(Unused1)}, Un2 {bstr(Unused2)}, Un3 {bstr(Unused3)}, Un4 {bstr(Unused4)}, Un5 {bstr(Unused5)}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordMetadata.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordMetadata.cs index 397d78bf108..a4bcdd144c4 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordMetadata.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/RecordMetadata.cs @@ -3,28 +3,24 @@ namespace Tsavorite.core { + using static LogAddress; + /// /// A structure carrying metadata about a record in the log. /// public readonly struct RecordMetadata { - /// - /// The header of the record. - /// - public readonly RecordInfo RecordInfo; - /// /// The logical address of the record. /// public readonly long Address; - internal RecordMetadata(RecordInfo recordInfo, long address = Constants.kInvalidAddress) + internal RecordMetadata(long address = kInvalidAddress) { - RecordInfo = recordInfo; Address = address; } /// - public override string ToString() => $"ri {RecordInfo}, addr {Address}"; + public override string ToString() => $"addr {AddressString(Address)}"; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/CallbackInfos.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/CallbackInfos.cs index 634a745eb1c..9e54f1d019e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/CallbackInfos.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/CallbackInfos.cs @@ -1,10 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Diagnostics; -using System.Runtime.CompilerServices; -using static Tsavorite.core.Utility; - namespace Tsavorite.core { /// @@ -17,6 +13,11 @@ public enum UpsertAction : byte /// Default, + /// + /// Stop the operation immediately with a "wrong type" error + /// + WrongType, + /// /// Stop the operation immediately and return. /// @@ -24,7 +25,7 @@ public enum UpsertAction : byte } /// - /// Information passed to record-update callbacks. + /// Information passed to record-update callbacks. /// public struct UpsertInfo { @@ -48,24 +49,6 @@ public struct UpsertInfo /// public int SessionID { get; internal set; } - /// - /// The header of the record. - /// - public RecordInfo RecordInfo { get; private set; } - - internal void SetRecordInfo(ref RecordInfo recordInfo) => RecordInfo = recordInfo; - - /// - /// The length of data in the value that is in use. Incoming, it is set by Tsavorite. - /// If an application wants to allow data to shrink and then grow again within the same record, it must set this to the correct length on output. - /// - public int UsedValueLength { get; set; } - - /// - /// The allocated length of the record value. - /// - public int FullValueLength { get; internal set; } - /// /// What actions Tsavorite should perform on a false return from the ISessionFunctions method /// @@ -87,65 +70,6 @@ public UpsertInfo(ref RMWInfo rmwInfo) KeyHash = rmwInfo.KeyHash; Action = UpsertAction.Default; } - - /// - /// Retrieve the extra value length from the record, if present, and then clear it to ensure consistent log scan during in-place update. - /// - /// Reference to the record value - /// The currently-used length of the record value - /// The record header - /// The type of the value - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly unsafe void ClearExtraValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength) - { - Debug.Assert(usedValueLength == UsedValueLength, $"UpsertInfo: usedValueLength ({usedValueLength}) != this.UsedValueLength ({UsedValueLength})"); - StaticClearExtraValueLength(ref recordInfo, ref recordValue, usedValueLength); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void StaticClearExtraValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength) - { - if (!recordInfo.HasFiller) - return; - - var valueAddress = (long)Unsafe.AsPointer(ref recordValue); - int* extraLengthPtr = (int*)(valueAddress + RoundUp(usedValueLength, sizeof(int))); - - *extraLengthPtr = 0; - recordInfo.ClearHasFiller(); - } - - /// - /// Set the extra value length, if any, into the record past the used value length. - /// - /// The record header - /// Reference to the record value - /// The currently-used length of the record value - /// The type of the value - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe void SetUsedValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength) - { - StaticSetUsedValueLength(ref recordInfo, ref recordValue, usedValueLength, FullValueLength); - UsedValueLength = usedValueLength; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void StaticSetUsedValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength, int fullValueLength) - { - // Note: This is only called for variable-length types, and for those we have ensured the location of recordValue is pinned. - long valueAddress = (long)Unsafe.AsPointer(ref recordValue); - Debug.Assert(!recordInfo.HasFiller, "Filler should have been cleared by ClearExtraValueLength()"); - - usedValueLength = RoundUp(usedValueLength, sizeof(int)); - int extraValueLength = fullValueLength - usedValueLength; - if (extraValueLength >= sizeof(int)) - { - int* extraValueLengthPtr = (int*)(valueAddress + usedValueLength); - Debug.Assert(*extraValueLengthPtr == 0 || *extraValueLengthPtr == extraValueLength, "existing ExtraValueLength should be 0 or the same value"); - *extraValueLengthPtr = extraValueLength; - recordInfo.SetHasFiller(); - } - } } /// @@ -168,6 +92,11 @@ public enum RMWAction : byte /// ExpireAndStop, + /// + /// Stop the operation immediately with a "wrong type" error + /// + WrongType, + /// /// Stop the operation immediately and return. /// @@ -175,7 +104,7 @@ public enum RMWAction : byte } /// - /// Information passed to record-update callbacks. + /// Information passed to record-update callbacks. /// public struct RMWInfo { @@ -185,11 +114,24 @@ public struct RMWInfo public long Version { get; internal set; } /// - /// The logical address of the record being operated on. For CopyUpdater, this is the source address, - /// or if the source is the read cache. + /// The logical address of the record being operated on. For CopyUpdater, this is the source address + /// when runs, then becomes + /// the destination address by the time + /// and are called. Use + /// to access the source address from PostCopyUpdater. + /// Set to if the source is the read cache. /// public long Address { get; internal set; } + /// + /// For CopyUpdater (and the post-CAS PostCopyUpdater hook): the logical address of the source record + /// that was copied from. Populated alongside at the start of the RMW so it remains + /// available even after is reassigned to the destination. + /// Equals if the source is the read cache or there is no source + /// (initial update path). + /// + public long SourceAddress { get; internal set; } + /// /// Hash code of key being operated on /// @@ -200,36 +142,15 @@ public struct RMWInfo /// public int SessionID { get; internal set; } - /// - /// The header of the record. - /// - public RecordInfo RecordInfo { get; private set; } - - internal void SetRecordInfo(ref RecordInfo recordInfo) => RecordInfo = recordInfo; - internal void ClearRecordInfo() => RecordInfo = default; - - /// - /// The length of data in the value that is in use. Incoming, it is set by Tsavorite. - /// If an application wants to allow data to shrink and then grow again within the same record, it must set this to the correct length on output. - /// - public int UsedValueLength { get; set; } - - /// - /// The allocated length of the record value. - /// - public int FullValueLength { get; internal set; } - - public int FullRecordLength { get; internal set; } - /// /// If set true by CopyUpdater, the source record for the RCU will not be elided from the tag chain even if this is otherwise possible. /// public bool PreserveCopyUpdaterSourceRecord { get; set; } /// - /// Whether the call is from sync or async (pending) path + /// If set true by RMW and there is a source ValueObject it will be cleared immediately (to manage object size tracking most effectively). /// - public bool IsFromPending { get; internal set; } + public bool ClearSourceValueObject { get; set; } /// /// What actions Tsavorite should perform on a false return from the ISessionFunctions method @@ -240,34 +161,6 @@ public struct RMWInfo /// User-defined byte of data associated with the operation /// public byte UserData { get; set; } - - /// - /// Retrieve the extra value length from the record, if present, and then clear it to ensure consistent log scan during in-place update. - /// - /// Reference to the record header - /// Reference to the record value - /// The currently-used length of the record value - /// The type of the value - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ClearExtraValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength) - { - Debug.Assert(usedValueLength == UsedValueLength, $"RMWInfo: usedValueLength ({usedValueLength}) != this.UsedValueLength ({UsedValueLength})"); - UpsertInfo.StaticClearExtraValueLength(ref recordInfo, ref recordValue, usedValueLength); - } - - /// - /// Set the extra value length, if any, into the record past the used value length. - /// - /// Reference to the record header - /// Reference to the record value - /// The currently-used length of the record value - /// The type of the value - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe void SetUsedValueLength(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength) - { - UpsertInfo.StaticSetUsedValueLength(ref recordInfo, ref recordValue, usedValueLength, FullValueLength); - UsedValueLength = usedValueLength; - } } /// @@ -286,7 +179,7 @@ public enum DeleteAction : byte CancelOperation } /// - /// Information passed to record-update callbacks. + /// Information passed to record-update callbacks. /// public struct DeleteInfo { @@ -310,24 +203,6 @@ public struct DeleteInfo /// public int SessionID { get; internal set; } - /// - /// The header of the record. - /// - public RecordInfo RecordInfo { get; private set; } - - internal void SetRecordInfo(ref RecordInfo recordInfo) => RecordInfo = recordInfo; - - /// - /// The length of data in the value that is in use. Incoming, it is set by Tsavorite to the result. - /// If an application wants to allow data to shrink and then grow again within the same record, it must set this to the correct length on output. - /// - public int UsedValueLength { get; set; } - - /// - /// The allocated length of the record value. - /// - public int FullValueLength { get; internal set; } - /// /// What actions Tsavorite should perform on a false return from the ISessionFunctions method /// @@ -342,7 +217,7 @@ public struct DeleteInfo /// /// What actions to take following the RMW ISessionFunctions method call, such as cancellation or record expiration. /// - public enum ReadAction + public enum ReadAction : byte { /// /// Execute the default action for the method 'false' return. @@ -354,6 +229,11 @@ public enum ReadAction /// Expire, + /// + /// Stop the operation immediately with a "wrong type" error + /// + WrongType, + /// /// Stop the operation immediately and return. /// @@ -361,7 +241,7 @@ public enum ReadAction } /// - /// Information passed to record-read callbacks. + /// Information passed to record-read callbacks. /// public struct ReadInfo { @@ -375,13 +255,6 @@ public struct ReadInfo /// public long Address { get; internal set; } - /// - /// The header of the record. - /// - public RecordInfo RecordInfo { get; private set; } - - internal void SetRecordInfo(ref RecordInfo recordInfo) => RecordInfo = recordInfo; - /// /// Whether the call is from sync or async (pending) path /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/DataContractObjectSerializer.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/DataContractObjectSerializer.cs deleted file mode 100644 index 2dddae94f34..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/DataContractObjectSerializer.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using System.Runtime.Serialization; -using System.Xml; - -namespace Tsavorite.core -{ - /// - /// Serializer (for class types) based on DataContract - /// - /// - public sealed class DataContractObjectSerializer : BinaryObjectSerializer - { - private static readonly DataContractSerializer serializer = new DataContractSerializer(typeof(T)); - - /// - /// Deserialize - /// - /// - public override void Deserialize(out T obj) - { - int count = reader.ReadInt32(); - var byteArray = reader.ReadBytes(count); - using var ms = new MemoryStream(byteArray); - using var _reader = XmlDictionaryReader.CreateBinaryReader(ms, XmlDictionaryReaderQuotas.Max); - obj = (T)serializer.ReadObject(_reader); - } - - /// - /// Serialize - /// - /// - public override void Serialize(ref T obj) - { - using var ms = new MemoryStream(); - using (var _writer = XmlDictionaryWriter.CreateBinaryWriter(ms, null, null, false)) - serializer.WriteObject(_writer, obj); - writer.Write((int)ms.Position); - writer.Write(ms.ToArray()); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKey.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKey.cs new file mode 100644 index 00000000000..325aad2e902 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKey.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Tsavorite.core +{ + /// + /// Defines a key type for Tsavorite operations, providing hashing and equality comparison. + /// + public interface IKey + { + /// + /// True if the and other memory exposed by this can be safely assumed to not move. + /// + /// This includes for the duration of any pending operations, through their explicit completion. + /// This means things like variables or need to stay unchanged and in place if wrapped, provided this returns true. + /// + bool IsPinned { get; } + + /// + /// True if the is truly empty - not zero bytes, but uninitialized and conceptually bereft of data. + /// + /// This should be false for almost all implementors. + /// + bool IsEmpty => false; + + /// + /// The raw bytes of this key. + /// + [UnscopedRef] + ReadOnlySpan KeyBytes { get; } + + /// + /// True if this has a namespace associated with it. + /// + /// Namespaces are not visible parts of a key, but are used in hashing and equality. + /// + bool HasNamespace { get; } + + /// + /// If returns true, called to get the contents of the namespace. + /// + /// The special value [0] is reserved and should never be returned. + /// + [UnscopedRef] + ReadOnlySpan NamespaceBytes { get; } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKeyComparer.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKeyComparer.cs index 1d5299b0b24..260031a57ba 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKeyComparer.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IKeyComparer.cs @@ -8,20 +8,33 @@ namespace Tsavorite.core /// /// Defines methods to support the comparison of Tsavorite keys for equality. /// - /// The type of keys to compare. - /// This comparer differs from the built-in in that it implements a 64-bit hash code - public interface IKeyComparer + /// This comparer differs from the built-in in that it implements a 64-bit hash code + public interface IKeyComparer { /// /// Get 64-bit hash code /// - long GetHashCode64(ref T key); + long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; /// /// Equality comparison /// /// Left side /// Right side - bool Equals(ref T k1, ref T k2); + bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IObjectSerializer.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IObjectSerializer.cs index 42770386641..8aaf61ae1a0 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IObjectSerializer.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IObjectSerializer.cs @@ -22,7 +22,7 @@ public interface IObjectSerializer /// Serialize object /// /// - void Serialize(ref T obj); + void Serialize(T obj); /// /// End serialization to given stream @@ -69,7 +69,7 @@ public abstract class BinaryObjectSerializer : IObjectSerializer public void BeginSerialize(Stream stream) => writer = new BinaryWriter(stream, new UTF8Encoding(), true); /// Serialize - public abstract void Serialize(ref T obj); + public abstract void Serialize(T obj); /// End serialize public void EndSerialize() => writer.Dispose(); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IReadArgBatch.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IReadArgBatch.cs index 07dd5afa7fd..e903e419191 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IReadArgBatch.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/IReadArgBatch.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Tsavorite.core { /// @@ -10,12 +12,21 @@ namespace Tsavorite.core /// Type of input /// Type of output public interface IReadArgBatch + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { /// /// Count of keys/args/outputs. /// int Count { get; } + /// + /// Raw parameters for the batch. + /// + ReadOnlySpan Parameters { get; } + /// /// Get th key. /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs index a014e35f385..c7d11b41e74 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctions.cs @@ -1,105 +1,177 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Tsavorite.core { /// /// Callback functions to Tsavorite /// - /// - /// - /// - /// - /// - public interface ISessionFunctions + public interface ISessionFunctions : IVariableLengthInput { #region Reads /// - /// Non-concurrent reader. + /// Read the record by copying all or part of it to . /// - /// The key for the record to be read - /// The user input for computing from - /// The value for the record being read - /// The location where is to be copied + /// The log record being read + /// The user input for computing from the record value + /// Receives the output of the operation, if any /// Information about this read operation and its context /// True if the value was available, else false (e.g. the value was expired) - bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo); - - /// - /// Concurrent reader - /// - /// The key for the record to be read - /// The user input for computing from - /// The value for the record being read - /// The location where is to be copied - /// Information about this read operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification - /// True if the value was available, else false (e.g. the value was expired) - bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo); + bool Reader(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord; /// /// Read completion /// - /// The key for this record + /// The log record that was read from disk /// The user input that was used in the read operation /// The result of the read operation; if this is a struct, then it will be a temporary and should be copied to /// The application context passed through the pending operation /// The result of the pending operation - /// Metadata for the record; may be used to obtain .PreviousAddress when doing iterative reads - void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); + /// Metadata for the record + void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); #endregion reads #region Upserts /// - /// Non-concurrent writer; called on an Upsert that does not find the key so does an insert or finds the key's record in the immutable region so does a read/copy/update (RCU). + /// Non-concurrent writer for Span value; called on an Upsert that does not find the key so does an insert or finds the key's record in the immutable region so does a read/copy/update (RCU). + /// + /// The destination log record + /// The size information for this record's fields + /// The user input to be used for computing + /// The input Span to be copied to the record value + /// The location where the result of the update may be placed + /// Information about this update operation and its context + /// True if the write was performed, else false (e.g. cancellation) + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + + /// + /// Non-concurrent writer for Object values; called on an Upsert that does not find the key so does an insert or finds the key's record in the immutable region so does a read/copy/update (RCU). + /// + /// The destination log record + /// The size information for this record's fields + /// The user input to be used for computing + /// The input Object to be copied to the record value + /// The location where the result of the update may be placed + /// Information about this update operation and its context + /// True if the write was performed, else false (e.g. cancellation) + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + + /// + /// Non-concurrent writer for Object values; called on an Upsert that does not find the key so does an insert or finds the key's record in the immutable region so does a read/copy/update (RCU). /// - /// The key for this record - /// The user input to be used for computing - /// The previous value to be copied/updated - /// The destination to be updated; because this is an copy to a new location, there is no previous value there. + /// The destination log record + /// The size information for this record's fields + /// The user input to be used for computing + /// The log record passed to Upsert, to be copied to the destination record /// The location where the result of the update may be placed /// Information about this update operation and its context - /// The operation for which this write is being done - /// A reference to the RecordInfo for the record; used for variable-length record length modification /// True if the write was performed, else false (e.g. cancellation) - bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo); + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; /// - /// Called after SingleWriter when a record containing an upsert of a new key has been successfully inserted at the tail of the log. + /// Called after InitialWriter when a record has been successfully inserted at the tail of the log. /// - /// The key for this record - /// The user input that was used to compute - /// The previous value to be copied/updated - /// The destination to be updated; because this is an copy to a new location, there is no previous value there. + /// The destination log record + /// The size information for this record's fields + /// The user input that was used to compute + /// The input Span that was to be copied to the record value /// The location where the result of the update may be placed /// Information about this update operation and its context - /// The operation for which this write is being done - void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason); + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo); /// - /// Concurrent writer; called on an Upsert that finds the record in the mutable range. + /// Called after InitialWriter when a record has been successfully inserted at the tail of the log. /// - /// The key for the record to be written - /// The user input to be used for computing - /// The value to be copied to - /// The location where is to be copied; because this method is called only for in-place updates, there is a previous value there. + /// The destination log record + /// The size information for this record's fields + /// The user input that was used to compute + /// The input Object that was to be copied to the record value /// The location where the result of the update may be placed /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + + /// + /// Called after InitialWriter when a record has been successfully inserted at the tail of the log. + /// + /// The destination log record + /// The size information for this record's fields + /// The user input that was used to compute + /// The input LogRecord that was to be copied to the record value + /// The location where the result of the update may be placed + /// Information about this update operation and its context + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; + + /// + /// Concurrent writer; called on an Upsert that is in-place updating a record in the mutable range. + /// + /// The destination log record + /// The user input to be used for computing the destination record's value + /// The Span value passed to Upsert, to be copied to the destination record + /// The location where the result of the update may be placed + /// Information about this update operation and its context + /// /// True if the value was written, else false /// If the value is shrunk in-place, the caller must first zero the data that is no longer used, to ensure log-scan correctness. - bool ConcurrentWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo); + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan newValue, ref TOutput output, ref UpsertInfo upsertInfo); /// - /// Called after the Upsert operation, but before we unlock the record (if it was ephemerally locked). + /// Concurrent writer; called on an Upsert that is in-place updating a record in the mutable range. /// - /// - /// - /// - /// - /// - /// - void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epoch) + /// The destination log record + /// The user input to be used for computing the destination record's value + /// The value passed to Upsert, to be copied to the destination record + /// The location where the result of the update may be placed + /// Information about this update operation and its context + /// + /// True if the value was written, else false + /// If the value is shrunk in-place, the caller must first zero the data that is no longer used, to ensure log-scan correctness. + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject newValue, ref TOutput output, ref UpsertInfo upsertInfo); + + /// + /// Concurrent writer; called on an Upsert that is in-place updating a record in the mutable range. The caller should be aware of ETag and Expiration in the source record. + /// + /// The destination log record + /// The user input to be used for computing the destination record's value + /// The log record passed to Upsert, to be copied to the destination record + /// The location where the result of the update may be placed + /// Information about this update operation and its context + /// + /// True if the value was written, else false + /// If the value is shrunk in-place, the caller must first zero the data that is no longer used, to ensure log-scan correctness. + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; + + /// + /// Called after the Upsert operation but before we unlock the record (if it was ephemerally locked). + /// + /// + /// This is always called after the operation whether it succeeds or not (including when it has gone pending), so must have information indicating whether + /// the action is to be performed (such as by checking + /// + void PostUpsertOperation(TKey key, ref TInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor; + + /// + /// Called after the Upsert operation but before we unlock the record (if it was ephemerally locked). + /// + /// + /// This is always called after the operation whether it succeeds or not, so must have information indicating whether + /// the action is to be performed (such as by checking + /// + void PostUpsertOperation(TKey key, ref TInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; #endregion Upserts @@ -109,169 +181,161 @@ void PostUpsertOperation(ref TKey key, ref TInput input, ref TVa /// /// Whether we need to invoke initial-update for RMW /// - /// The key for this record + /// The key for this record; this is the key passed to Upsert as we don't have a log record yet. /// The user input to be used for computing the updated value /// The location where the result of the operation is to be copied /// Information about this update operation and its context - bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); + bool NeedInitialUpdate(TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; /// /// Initial update for RMW (insert at the tail of the log). /// - /// The key for this record - /// The user input to be used for computing the updated - /// The destination to be updated; because this is an insert, there is no previous value there. - /// The location where the result of the operation on is to be copied + /// The destination log record + /// The size information for this record's fields + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification /// True if the write was performed, else false (e.g. cancellation) - bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); + bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); /// /// Called after a record containing an initial update for RMW has been successfully inserted at the tail of the log. /// - /// The key for this record - /// The user input to be used for computing the updated - /// The destination to be updated; because this is an insert, there is no previous value there. - /// The location where the result of the operation on is to be copied + /// The log record that was created + /// The size information for this record's fields + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context - void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo); + void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); #endregion InitialUpdater #region CopyUpdater /// /// Whether we need to invoke copy-update for RMW /// - /// The key for this record + /// The source record being copied from /// The user input to be used for computing the updated value - /// The existing value that would be copied. - /// The location where the result of the operation on is to be copied + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context - bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo); + bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; /// /// Copy-update for RMW (RCU (Read-Copy-Update) to the tail of the log) /// - /// The key for this record - /// The user input to be used for computing from - /// The previous value to be copied/updated - /// The destination to be updated; because this is an copy to a new location, there is no previous value there. - /// The location where is to be copied + /// The source record being copied from + /// The destination log record being created + /// The size information for this record's fields + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification /// True if the write was performed, else false (e.g. cancellation) - bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); + bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; /// /// Called after a record containing an RCU (Read-Copy-Update) for RMW has been successfully inserted at the tail of the log. /// - /// The key for this record - /// The user input to be used for computing from - /// The previous value to be copied/updated; may also be disposed here if appropriate - /// The destination to be updated; because this is an copy to a new location, there is no previous value there. - /// The location where is to be copied + /// The source record being copied from + /// The destination log record being created + /// The size information for this record's fields + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context /// This is the only Post* method that returns non-void. The bool functions the same as CopyUpdater; this is because we do not want to modify /// objects in-memory until we know the "insert at tail" is successful. Therefore, we allow a false return as a signal to inspect /// and handle . - bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo); + bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; #endregion CopyUpdater #region InPlaceUpdater /// /// In-place update for RMW /// - /// The key for this record - /// The user input to be used for computing the updated - /// The destination to be updated; because this is an in-place update, there is a previous value there. - /// The location where the result of the operation on is to be copied + /// The log record that is being updated + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification + /// /// True if the value was successfully updated, else false (e.g. the value was expired) /// If the value is shrunk in-place, the caller must first zero the data that is no longer used, to ensure log-scan correctness. - bool InPlaceUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); + bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); #endregion InPlaceUpdater - #region Variable-length value size - /// - /// Length of resulting value object when performing RMW modification of value using given input - /// - int GetRMWModifiedValueLength(ref TValue value, ref TInput input); - - /// - /// Initial expected length of value object when populated by RMW using given input - /// - int GetRMWInitialValueLength(ref TInput input); - - /// - /// Length of resulting value object when performing Upsert of value using given input - /// - int GetUpsertValueLength(ref TValue value, ref TInput input); - #endregion Variable-length value size - /// /// Called after the RMW operation, but before we unlock the record (if it was ephemerally locked). /// - /// - /// - /// - /// - /// - void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epoch) + /// + /// This is always called after the operation whether it succeeds or not (including when it has gone pending), so must have information indicating whether + /// the action is to be performed (such as by checking + /// + void PostRMWOperation(TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; /// /// RMW completion /// - /// The key for this record - /// The user input that was used to perform the modification - /// The result of the RMW operation; if this is a struct, then it will be a temporary and should be copied to + /// The log record that was read from disk + /// The user input to be used to create the destination record's value + /// The location where the output of the operation, if any, is to be copied /// The application context passed through the pending operation /// The result of the pending operation /// The metadata of the modified or inserted record - void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); + void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); #endregion RMWs #region Deletes /// /// Single deleter; called on a Delete that does not find the record in the mutable range and so inserts a new record. /// - /// The key for the record to be deleted - /// The value for the record being deleted; because this method is called only for in-place updates, there is a previous value there. Usually this is ignored or assigned 'default'. + /// The log record that is being created with a tombstone /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification - /// For Object Value types, Dispose() can be called here. If recordInfo.Invalid is true, this is called after the record was allocated and populated, but could not be appended at the end of the log. + /// For Object Value types, Dispose() can be called here. If recordInfo.Invalid is true, this is called after the record was allocated and populated, + /// but could not be appended at the end of the log. /// True if the deleted record should be added, else false (e.g. cancellation) - bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo); + bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); /// /// Called after a record marking a Delete (with Tombstone set) has been successfully inserted at the tail of the log. /// - /// The key for the record that was deleted + /// The log record that was created with a tombstone /// Information about this update operation and its context /// This does not have the address of the record that contains the value at 'key'; Delete does not retrieve records below HeadAddress, so /// the last record we have in the 'key' chain may belong to 'key' or may be a collision. - void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo); + void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); /// /// Concurrent deleter; called on a Delete that finds the record in the mutable range. /// - /// The key for the record to be deleted - /// The value for the record being deleted; because this method is called only for in-place updates, there is a previous value there. Usually this is ignored or assigned 'default'. + /// The log record that is being deleted in-place /// Information about this update operation and its context - /// A reference to the RecordInfo for the record; used for variable-length record length modification - /// For Object Value types, Dispose() can be called here. If recordInfo.Invalid is true, this is called after the record was allocated and populated, but could not be appended at the end of the log. + /// For Object Value types, Dispose() can be called here. If logRecord.Info.Invalid is true, this is called after the record was allocated and populated, + /// but could not be appended at the end of the log. /// True if the value was successfully deleted, else false (e.g. the record was sealed) - bool ConcurrentDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo); + bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); /// - /// Called after the Delete operation, but before we unlock the record (if it was ephemerally locked). + /// Called after the Delete operation but before we unlock the record (if it was ephemerally locked). /// - /// - /// - /// - /// - void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epoch) + /// + /// This is always called after the operation whether it succeeds or not, so must have information indicating whether + /// the action is to be performed (such as by checking + /// + void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; #endregion Deletes @@ -280,27 +344,45 @@ void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo /// Called by Tsavorite when the operation goes pending, so the app can signal to itself that any pinned /// buffer in the Output is no longer valid and a heap-based buffer must be created. /// + /// /// void ConvertOutputToHeap(ref TInput input, ref TOutput output); + + /// + /// Called before reading a single key to verify key freshness and enforce prefix consistency. + /// + /// The key hash about to be read + void BeforeConsistentReadCallback(long hash); + + /// + /// Called after a single key read to update the session timestamp. + /// + void AfterConsistentReadKeyCallback(); + + /// + /// Called before reading a batch of keys to verify their freshness and enforce prefix consistency. + /// + /// + void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters); + + /// + /// Called after reading a batch of keys to update the session timestamp. + /// + bool AfterConsistentReadKeyBatchCallback(int keyCount); #endregion Utilities } /// /// Callback functions to Tsavorite (two-param version) /// - /// - /// - public interface ISessionFunctions : ISessionFunctions + public interface ISessionFunctions : ISessionFunctions { } /// /// Callback functions to Tsavorite (two-param version with context) /// - /// - /// - /// - public interface ISessionFunctions : ISessionFunctions + public interface ISessionFunctions : ISessionFunctions { } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctionsWrapper.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctionsWrapper.cs index ca8bb147f3a..a0789e94e4b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctionsWrapper.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionFunctionsWrapper.cs @@ -1,60 +1,97 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Tsavorite.core { /// /// Provides thread management and all callbacks. A wrapper for IFunctions and additional methods called by TsavoriteImpl; the wrapped /// IFunctions methods provide additional parameters to support the wrapper functionality, then call through to the user implementations. /// - internal interface ISessionFunctionsWrapper : IVariableLengthInput - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal interface ISessionFunctionsWrapper : IVariableLengthInput + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - bool IsManualLocking { get; } - TsavoriteKV Store { get; } + bool IsTransactionalLocking { get; } + TsavoriteKV Store { get; } #region Reads - bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo); - bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo); - void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); + bool Reader(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput dst, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord; + void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); #endregion reads #region Upserts - bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo); - void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo); - bool ConcurrentWriter(long physicalAddress, ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo); - void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo); + bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord; + void PostUpsertOperation(TKey key, ref TInput input, ReadOnlySpan srcValueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor; + void PostUpsertOperation(TKey key, ref TInput input, IHeapObject srcValueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; #endregion Upserts #region RMWs #region InitialUpdater - bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); - bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); - void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rMWInfo, ref RecordInfo recordInfo); + bool NeedInitialUpdate(TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; + bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo); + void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rMWInfo); #endregion InitialUpdater #region CopyUpdater - bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo); - bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); - bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo); + bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; + bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; + bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord; #endregion CopyUpdater #region InPlaceUpdater - bool InPlaceUpdater(long physicalAddress, ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status, ref RecordInfo recordInfo); + bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, out OperationStatus status); #endregion InPlaceUpdater - void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + void PostRMWOperation(TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; - void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); + void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata); #endregion RMWs #region Deletes - bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo); - void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo); - bool ConcurrentDeleter(long physicalAddress, ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo, out int fullRecordLength); - void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); + void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); + bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo); + void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor; #endregion Deletes @@ -63,22 +100,20 @@ void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo void ConvertOutputToHeap(ref TInput input, ref TOutput output); #endregion Utilities - #region Transient locking - bool TryLockTransientExclusive(ref TKey key, ref OperationStackContext stackCtx); - bool TryLockTransientShared(ref TKey key, ref OperationStackContext stackCtx); - void UnlockTransientExclusive(ref TKey key, ref OperationStackContext stackCtx); - void UnlockTransientShared(ref TKey key, ref OperationStackContext stackCtx); - #endregion + #region Ephemeral locking + bool TryLockEphemeralExclusive(ref OperationStackContext stackCtx); + bool TryLockEphemeralShared(ref OperationStackContext stackCtx); + void UnlockEphemeralExclusive(ref OperationStackContext stackCtx); + void UnlockEphemeralShared(ref OperationStackContext stackCtx); + #endregion #region Epoch control void UnsafeResumeThread(); void UnsafeSuspendThread(); #endregion - bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false); - - TsavoriteKV.TsavoriteExecutionContext Ctx { get; } + bool CompletePendingWithOutputs(out CompletedOutputIterator completedOutputs, bool wait = false, bool spinWaitForCommit = false); - IHeapContainer GetHeapContainer(ref TInput input); + TsavoriteKV.TsavoriteExecutionContext Ctx { get; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionLocker.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionLocker.cs index 8c83129da63..120dbe85089 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionLocker.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ISessionLocker.cs @@ -10,106 +10,106 @@ namespace Tsavorite.core /// Provides thread management and all callbacks. A wrapper for ISessionFunctions and additional methods called by TsavoriteImpl; the wrapped /// ISessionFunctions methods provide additional parameters to support the wrapper functionality, then call through to the user implementations. /// - public interface ISessionLocker - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public interface ISessionLocker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - bool IsManualLocking { get; } + bool IsTransactionalLocking { get; } - bool TryLockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx); - bool TryLockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx); - void UnlockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx); - void UnlockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx); + bool TryLockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx); + bool TryLockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx); + void UnlockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx); + void UnlockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx); } /// - /// Basic (non-lockable) sessions must do transient locking. + /// Basic (non-transactional) sessions must do Ephemeral locking. /// /// /// This struct contains no data fields; SessionFunctionsWrapper redirects with its ClientSession. /// - internal struct BasicSessionLocker : ISessionLocker - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal struct BasicSessionLocker : ISessionLocker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public bool IsManualLocking => false; + public bool IsTransactionalLocking => false; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) + public bool TryLockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) { if (!store.LockTable.TryLockExclusive(ref stackCtx.hei)) return false; - stackCtx.recSrc.SetHasTransientXLock(); + stackCtx.recSrc.SetHasEphemeralXLock(); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx) + public bool TryLockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx) { if (!store.LockTable.TryLockShared(ref stackCtx.hei)) return false; - stackCtx.recSrc.SetHasTransientSLock(); + stackCtx.recSrc.SetHasEphemeralSLock(); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) + public void UnlockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) { store.LockTable.UnlockExclusive(ref stackCtx.hei); - stackCtx.recSrc.ClearHasTransientXLock(); + stackCtx.recSrc.ClearHasEphemeralXLock(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx) + public void UnlockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx) { store.LockTable.UnlockShared(ref stackCtx.hei); - stackCtx.recSrc.ClearHasTransientSLock(); + stackCtx.recSrc.ClearHasEphemeralSLock(); } } /// - /// Lockable sessions are manual locking and thus must have already locked the record prior to an operation on it, so assert that. + /// Transactional sessions must have already locked the record prior to an operation on it, so assert that. /// - internal struct LockableSessionLocker : ISessionLocker - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal struct TransactionalSessionLocker : ISessionLocker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - public bool IsManualLocking => true; + public bool IsTransactionalLocking => true; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) + public bool TryLockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) { Debug.Assert(store.LockTable.IsLockedExclusive(ref stackCtx.hei), - $"Attempting to use a non-XLocked key in a Lockable context (requesting XLock):" + $"Attempting to use a non-XLocked key in a Transactional context (requesting XLock):" + $" XLocked {store.LockTable.IsLockedExclusive(ref stackCtx.hei)}," + $" Slocked {store.LockTable.IsLockedShared(ref stackCtx.hei)}"); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryLockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx) + public bool TryLockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx) { Debug.Assert(store.LockTable.IsLocked(ref stackCtx.hei), - $"Attempting to use a non-Locked (S or X) key in a Lockable context (requesting SLock):" + $"Attempting to use a non-Locked (S or X) key in a Transactional context (requesting SLock):" + $" XLocked {store.LockTable.IsLockedExclusive(ref stackCtx.hei)}," + $" Slocked {store.LockTable.IsLockedShared(ref stackCtx.hei)}"); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) + public void UnlockEphemeralExclusive(TsavoriteKV store, ref OperationStackContext stackCtx) { Debug.Assert(store.LockTable.IsLockedExclusive(ref stackCtx.hei), - $"Attempting to unlock a non-XLocked key in a Lockable context (requesting XLock):" + $"Attempting to unlock a non-XLocked key in a Transactional context (requesting XLock):" + $" XLocked {store.LockTable.IsLockedExclusive(ref stackCtx.hei)}," + $" Slocked {store.LockTable.IsLockedShared(ref stackCtx.hei)}"); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnlockTransientShared(TsavoriteKV store, ref OperationStackContext stackCtx) + public void UnlockEphemeralShared(TsavoriteKV store, ref OperationStackContext stackCtx) { Debug.Assert(store.LockTable.IsLockedShared(ref stackCtx.hei), - $"Attempting to use a non-XLocked key in a Lockable context (requesting XLock):" + $"Attempting to use a non-XLocked key in a Transactional context (requesting XLock):" + $" XLocked {store.LockTable.IsLockedExclusive(ref stackCtx.hei)}," + $" Slocked {store.LockTable.IsLockedShared(ref stackCtx.hei)}"); } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/KeyComparers.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/KeyComparers.cs deleted file mode 100644 index 206aaaf7baa..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/KeyComparers.cs +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; - -namespace Tsavorite.core -{ - internal static class KeyComparers - { - public static IKeyComparer Get() - { - if (typeof(T) == typeof(string)) - return (IKeyComparer)(object)StringKeyComparer.Instance; - else if (typeof(T) == typeof(byte[])) - return (IKeyComparer)(object)ByteArrayKeyComparer.Instance; - else if (typeof(T) == typeof(long)) - return (IKeyComparer)(object)LongKeyComparer.Instance; - else if (typeof(T) == typeof(int)) - return (IKeyComparer)(object)IntKeyComparer.Instance; - else if (typeof(T) == typeof(Guid)) - return (IKeyComparer)(object)GuidKeyComparer.Instance; - else if (typeof(T) == typeof(SpanByte)) - return (IKeyComparer)(object)SpanByteComparer.Instance; - else - { - Debug.WriteLine("***WARNING*** Creating default Tsavorite key equality comparer based on potentially slow EqualityComparer.Default." - + "To avoid this, provide a comparer (ITsavoriteEqualityComparer) as an argument to Tsavorite's constructor, or make Key implement the interface ITsavoriteEqualityComparer"); - return DefaultKeyComparer.Instance; - } - } - } - - /// - /// Deterministic equality comparer for strings - /// - public sealed class StringKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly StringKeyComparer Instance = new(); - - /// - public bool Equals(ref string key1, ref string key2) - { - // Use locals in case the record space is cleared. - string k1 = key1, k2 = key2; - return (k1 is null || k2 is null) ? false : k1 == k2; - } - - /// - public unsafe long GetHashCode64(ref string key) - { - // Use locals in case the record space is cleared. - string k = key; - if (k is null) - return 0; - - fixed (char* c = k) - { - return Utility.HashBytes((byte*)c, key.Length * sizeof(char)); - } - } - } - - /// - /// Deterministic equality comparer for longs - /// - public sealed class LongKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly LongKeyComparer Instance = new(); - - /// - public bool Equals(ref long k1, ref long k2) => k1 == k2; - - /// - public long GetHashCode64(ref long k) => Utility.GetHashCode(k); - } - - /// - /// Deterministic equality comparer for longs - /// - public sealed class IntKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly IntKeyComparer Instance = new(); - - /// - public bool Equals(ref int k1, ref int k2) => k1 == k2; - - /// - public long GetHashCode64(ref int k) => Utility.GetHashCode(k); - } - - /// - /// Deterministic equality comparer for longs - /// - public sealed class GuidKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly GuidKeyComparer Instance = new(); - - /// - public bool Equals(ref Guid k1, ref Guid k2) => k1 == k2; - - /// - public unsafe long GetHashCode64(ref Guid k) - { - var _k = k; - var pGuid = (long*)&_k; - return pGuid[0] ^ pGuid[1]; - } - } - - /// - /// Deterministic equality comparer for byte[] - /// - public sealed class ByteArrayKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly ByteArrayKeyComparer Instance = new(); - - /// - public bool Equals(ref byte[] key1, ref byte[] key2) => key1.AsSpan().SequenceEqual(key2); - - /// - public unsafe long GetHashCode64(ref byte[] key) - { - // Use locals in case the record space is cleared. - byte[] k = key; - if (k is null) - return 0; - - fixed (byte* b = k) - { - return Utility.HashBytes(b, k.Length); - } - } - } - - /// - /// No-op equality comparer for Empty (used by TsavoriteLog) - /// - public sealed class EmptyKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly EmptyKeyComparer Instance = new(); - - /// - public bool Equals(ref Empty key1, ref Empty key2) => throw new NotImplementedException(); - - /// - public long GetHashCode64(ref Empty key) => throw new NotImplementedException(); - } - - /// - /// Low-performance Tsavorite equality comparer wrapper around EqualityComparer.Default - /// - /// - internal sealed class DefaultKeyComparer : IKeyComparer - { - /// - /// The default instance. - /// - /// Used to avoid allocating new comparers. - public static readonly DefaultKeyComparer Instance = new(); - - private static readonly EqualityComparer DefaultEC = EqualityComparer.Default; - - /// - public bool Equals(ref T k1, ref T k2) => DefaultEC.Equals(k1, k2); - - /// - public long GetHashCode64(ref T k) => Utility.GetHashCode(DefaultEC.GetHashCode(k)); - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ObjectSerializer.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ObjectSerializer.cs deleted file mode 100644 index 7802a82bf78..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/ObjectSerializer.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; - -namespace Tsavorite.core -{ - internal static class ObjectSerializer - { - public static Func> Get() - { - if (typeof(T) == typeof(string)) - return () => (IObjectSerializer)new StringBinaryObjectSerializer(); - else if (typeof(T) == typeof(byte[])) - return () => (IObjectSerializer)new ByteArrayBinaryObjectSerializer(); - else - return () => new DataContractObjectSerializer(); - } - } - - internal class StringBinaryObjectSerializer : BinaryObjectSerializer - { - public override void Deserialize(out string obj) => obj = reader.ReadString(); - public override void Serialize(ref string obj) => writer.Write(obj); - } - - internal class ByteArrayBinaryObjectSerializer : BinaryObjectSerializer - { - public override void Deserialize(out byte[] obj) => obj = reader.ReadBytes(reader.ReadInt32()); - public override void Serialize(ref byte[] obj) - { - writer.Write(obj.Length); - writer.Write(obj); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs index f3954586f6c..219eb5bbe76 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/SessionFunctionsBase.cs @@ -10,129 +10,193 @@ namespace Tsavorite.core /// /// Default empty functions base class to make it easy for users to provide their own implementation of ISessionFunctions /// - /// - /// - /// - /// - /// - public abstract class SessionFunctionsBase : ISessionFunctions + public abstract class SessionFunctionsBase : ISessionFunctions { /// - public virtual bool ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) => true; + public virtual bool Reader(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref ReadInfo readInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + /// - public virtual bool SingleReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst, ref ReadInfo readInfo) => true; + public virtual bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration, which will come from TInput in fuller implementations. + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(key: logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); + } /// - public virtual bool ConcurrentWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) { dst = src; return true; } + public virtual bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration, which will come from TInput in fuller implementations. + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(key: logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + } + + public virtual bool InPlaceWriter(ref LogRecord dstLogRecord, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + // This includes ETag and Expiration + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(key: dstLogRecord, inputLogRecord, ref input) }; + dstLogRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + /// - public virtual bool SingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) { dst = src; return true; } + public virtual bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration, which will come from TInput in fuller implementations. + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); + } + /// - public virtual void PostSingleWriter(ref TKey key, ref TInput input, ref TValue src, ref TValue dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason) { } + public virtual bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) + { + // This does not try to set ETag or Expiration, which will come from TInput in fuller implementations. + return dstLogRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + } /// - public virtual void PostUpsertOperation(ref TKey key, ref TInput input, ref TValue src, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + public virtual bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { + // This includes ETag and Expiration + return dstLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); + } + + public virtual void PostUpsertOperation(TKey key, ref TInput input, ReadOnlySpan valueSpan, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TEpochAccessor : IEpochAccessor + { } + public virtual void PostUpsertOperation(TKey key, ref TInput input, IHeapObject valueObject, ref UpsertInfo upsertInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { } /// - public virtual bool InitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - /// - public virtual void PostInitialUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo) { } + public virtual void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ReadOnlySpan srcValue, ref TOutput output, ref UpsertInfo upsertInfo) { } /// - public virtual bool NeedInitialUpdate(ref TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; + public virtual void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, IHeapObject srcValue, ref TOutput output, ref UpsertInfo upsertInfo) { } /// - public virtual bool NeedCopyUpdate(ref TKey key, ref TInput input, ref TValue oldValue, ref TOutput output, ref RMWInfo rmwInfo) => true; + public virtual void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo) + where TSourceLogRecord : ISourceLogRecord + { } + /// - public virtual bool CopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; + public virtual bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; /// - public virtual bool PostCopyUpdater(ref TKey key, ref TInput input, ref TValue oldValue, ref TValue newValue, ref TOutput output, ref RMWInfo rmwInfo) => true; + public virtual void PostInitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) { } /// - public virtual bool InPlaceUpdater(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; + public virtual bool NeedInitialUpdate(TKey key, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => true; /// - public virtual bool SingleDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) { value = default; return true; } - public virtual void PostSingleDeleter(ref TKey key, ref DeleteInfo deleteInfo) { } - public virtual bool ConcurrentDeleter(ref TKey key, ref TValue value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; + public virtual bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; /// - public virtual void PostDeleteOperation(ref TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + public virtual void PostRMWOperation(TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { } - public virtual void ReadCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } + /// + public virtual bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + /// + public virtual bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) + where TSourceLogRecord : ISourceLogRecord + => true; + /// + public virtual bool InPlaceUpdater(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo) => true; /// - public virtual void PostRMWOperation(ref TKey key, ref TInput input, ref RMWInfo rmwInfo, TEpochAccessor epochAccessor) + public virtual bool InitialDeleter(ref LogRecord dstLogRecord, ref DeleteInfo deleteInfo) + { + dstLogRecord.ClearValueIfHeap(); + return true; + } + public virtual void PostInitialDeleter(ref LogRecord dstLogRecord, ref DeleteInfo deleteInfo) { } + public virtual bool InPlaceDeleter(ref LogRecord dstLogRecord, ref DeleteInfo deleteInfo) => true; + + /// + public virtual void PostDeleteOperation(TKey key, ref DeleteInfo deleteInfo, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif where TEpochAccessor : IEpochAccessor { } + + public virtual void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } /// - public virtual void RMWCompletionCallback(ref TKey key, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } + public virtual void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref TInput input, ref TOutput output, TContext ctx, Status status, RecordMetadata recordMetadata) { } + // *FieldInfo require an implementation that knows what is in IInput /// - public virtual int GetRMWModifiedValueLength(ref TValue value, ref TInput input) => throw new TsavoriteException("GetRMWModifiedValueLength is only available for SpanByte Functions"); + public virtual RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TInput input) + where TSourceLogRecord : ISourceLogRecord + => throw new NotImplementedException("GetRMWModifiedFieldInfo requires knowledge of TInput"); /// - public virtual int GetRMWInitialValueLength(ref TInput input) => throw new TsavoriteException("GetRMWInitialValueLength is only available for SpanByte Functions"); + public virtual RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => throw new NotImplementedException("GetRMWInitialFieldInfo requires knowledge of TInput"); /// - public virtual int GetUpsertValueLength(ref TValue value, ref TInput input) => throw new TsavoriteException("GetUpsertValueLength is only available for SpanByte Functions"); - + public virtual RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + // TODO: Namespace! + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length, ValueIsObject = false }; /// - public virtual void ConvertOutputToHeap(ref TInput input, ref TOutput output) { } - } - - /// - /// Default empty functions base class to make it easy for users to provide their own implementation of FunctionsBase - /// - /// - /// - /// - public class SimpleSessionFunctions : SessionFunctionsBase - { - private readonly Func merger; - public SimpleSessionFunctions() => merger = (l, r) => l; - public SimpleSessionFunctions(Func merger) => this.merger = merger; - + public virtual RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + // TODO: Namespace! + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; /// - public override bool ConcurrentReader(ref TKey key, ref TValue input, ref TValue value, ref TValue dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst = value; - return true; - } + public virtual RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + // TODO: Namespace! + => new() { KeySize = key.KeyBytes.Length, ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, ValueIsObject = inputLogRecord.Info.ValueIsObject }; /// - public override bool SingleReader(ref TKey key, ref TValue input, ref TValue value, ref TValue dst, ref ReadInfo readInfo) - { - dst = value; - return true; - } + public virtual void ConvertOutputToHeap(ref TInput input, ref TOutput output) { } - public override bool SingleWriter(ref TKey key, ref TValue input, ref TValue src, ref TValue dst, ref TValue output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - var result = base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); - if (result) - output = dst; - return result; - } + public virtual void BeforeConsistentReadCallback(long hash) { } - public override bool ConcurrentWriter(ref TKey key, ref TValue input, ref TValue src, ref TValue dst, ref TValue output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - var result = base.ConcurrentWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, ref recordInfo); - if (result) - output = dst; - return result; - } + public virtual void AfterConsistentReadKeyCallback() { } - /// - public override bool InitialUpdater(ref TKey key, ref TValue input, ref TValue value, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { value = output = input; return true; } - /// - public override bool CopyUpdater(ref TKey key, ref TValue input, ref TValue oldValue, ref TValue newValue, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { newValue = output = merger(input, oldValue); return true; } - /// - public override bool InPlaceUpdater(ref TKey key, ref TValue input, ref TValue value, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { value = output = merger(input, value); return true; } - } + public virtual void BeforeConsistentReadKeyBatchCallback(ReadOnlySpan parameters) { } - public class SimpleSimpleFunctions : SimpleSessionFunctions - { - public SimpleSimpleFunctions() : base() { } - public SimpleSimpleFunctions(Func merger) : base(merger) { } + public virtual bool AfterConsistentReadKeyBatchCallback(int keyCount) => true; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/TryAddFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/TryAddFunctions.cs deleted file mode 100644 index fb40ef86555..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Interfaces/TryAddFunctions.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -namespace Tsavorite.core -{ - /// - /// Functions that make RMW behave as an atomic TryAdd operation, where Input is the value being added. - /// Return Status.NotFound => TryAdd succeededed (item added). - /// Return Status.Found => TryAdd failed (item not added, key was already present). - /// - /// - /// - /// - public class TryAddFunctions : SimpleSessionFunctions - { - /// - public override bool InPlaceUpdater(ref TKey key, ref TValue input, ref TValue value, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => true; - /// - public override bool NeedCopyUpdate(ref TKey key, ref TValue input, ref TValue oldValue, ref TValue output, ref RMWInfo rmwInfo) => false; - } - - /// - /// Functions that make RMW behave as an atomic TryAdd operation, where Input is the value being added. - /// Return Status.NotFound => TryAdd succeededed (item added) - /// Return Status.Found => TryAdd failed (item not added, key was already present) - /// - /// - /// - public class TryAddFunctions : TryAddFunctions { } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Checkpoint.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Checkpoint.cs index 1e9d14476ec..ea3cd6e1a99 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Checkpoint.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Checkpoint.cs @@ -26,9 +26,9 @@ internal static class EpochPhaseIdx public const int CheckpointCompletionCallback = 4; } - public partial class TsavoriteKV - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal TaskCompletionSource checkpointTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -36,7 +36,6 @@ public partial class TsavoriteKV internal Guid _indexCheckpointToken; internal Guid _hybridLogCheckpointToken; internal HybridLogCheckpointInfo _hybridLogCheckpoint; - internal HybridLogCheckpointInfo _lastSnapshotCheckpoint; internal Task CheckpointTask => checkpointTcs.Task; @@ -54,21 +53,11 @@ internal void WriteHybridLogMetaInfo() internal void CleanupLogCheckpoint() { + if (!checkpointManager.PerformAutomaticCleanup) return; checkpointManager.CleanupLogCheckpoint(_hybridLogCheckpointToken); Log.ShiftBeginAddress(_hybridLogCheckpoint.info.beginAddress, truncateLog: true); } - internal void WriteHybridLogIncrementalMetaInfo(DeltaLog deltaLog) - { - _hybridLogCheckpoint.info.cookie = checkpointManager.GetCookie(); - checkpointManager.CommitLogIncrementalCheckpoint(_hybridLogCheckpointToken, _hybridLogCheckpoint.info.ToByteArray(), deltaLog); - } - - internal void CleanupLogIncrementalCheckpoint() - { - checkpointManager.CleanupLogIncrementalCheckpoint(_hybridLogCheckpointToken); - } - internal void WriteIndexMetaInfo() { checkpointManager.CommitIndexCheckpoint(_indexCheckpointToken, _indexCheckpoint.info.ToByteArray()); @@ -76,6 +65,7 @@ internal void WriteIndexMetaInfo() internal void CleanupIndexCheckpoint() { + if (!checkpointManager.PerformAutomaticCleanup) return; checkpointManager.CleanupIndexCheckpoint(_indexCheckpointToken); } @@ -89,12 +79,6 @@ internal void InitializeHybridLogCheckpoint(Guid hybridLogToken, long version) _hybridLogCheckpoint.Initialize(hybridLogToken, version, checkpointManager); } - internal long Compact(ISessionFunctions functions, CompactionFunctions compactionFunctions, long untilAddress, CompactionType compactionType) - where CompactionFunctions : ICompactionFunctions - { - throw new NotImplementedException(); - } - // #endregion } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/DeltaLog.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/DeltaLog.cs deleted file mode 100644 index 9890d791ad2..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/DeltaLog.cs +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; - -namespace Tsavorite.core -{ - /// - /// The type of a record in the delta (incremental) log - /// - public enum DeltaLogEntryType : int - { - /// - /// The entry is a delta record - /// - DELTA, - - /// - /// The entry is checkpoint metadata - /// - CHECKPOINT_METADATA - } - - [StructLayout(LayoutKind.Explicit, Size = DeltaLog.HeaderSize)] - struct DeltalogHeader - { - [FieldOffset(0)] - public ulong Checksum; - [FieldOffset(8)] - public int Length; - [FieldOffset(12)] - public DeltaLogEntryType Type; - } - - /// - /// Scan iterator for hybrid log - /// - public sealed class DeltaLog : ScanIteratorBase, IDisposable - { - /// - /// Header size - /// - public const int HeaderSize = 16; - - readonly IDevice deltaLogDevice; - readonly int LogPageSizeBits; - readonly int PageSize; - readonly int PageSizeMask; - readonly int AlignedPageSizeBytes; - readonly int sectorSize; - BlittableFrame frame; - bool disposed = false; - - // Fields to support writes - SectorAlignedBufferPool memory; - long tailAddress; - long flushedUntilAddress; - - SemaphoreSlim completedSemaphore; - int issuedFlush; - SectorAlignedMemory buffer; - - /// - /// Tail address - /// - public long TailAddress => tailAddress; - - /// - /// Constructor - /// - public DeltaLog(IDevice deltaLogDevice, int logPageSizeBits, long tailAddress, ILogger logger = null) - : base(0, tailAddress >= 0 ? tailAddress : deltaLogDevice.GetFileSize(0), ScanBufferingMode.SinglePageBuffering, false, default, logPageSizeBits, false, logger: logger) - { - LogPageSizeBits = logPageSizeBits; - PageSize = 1 << LogPageSizeBits; - PageSizeMask = PageSize - 1; - this.deltaLogDevice = deltaLogDevice; - this.tailAddress = flushedUntilAddress = endAddress; - sectorSize = (int)deltaLogDevice.SectorSize; - AlignedPageSizeBytes = (int)Align(PageSize); - issuedFlush = 1; - completedSemaphore = new SemaphoreSlim(0); - } - - /// - public override void InitializeForReads() - { - base.InitializeForReads(); - if (frameSize > 0 && (endAddress > 0 || tailAddress > 0)) - frame = new BlittableFrame(frameSize, 1 << LogPageSizeBits, sectorSize); - } - - /// - /// Dispose the iterator - /// - public override void Dispose() - { - if (!disposed) - { - base.Dispose(); - - // Dispose/unpin the frame from memory - frame?.Dispose(); - // Wait for ongoing page flushes - if (Interlocked.Decrement(ref issuedFlush) == 0) - completedSemaphore.Release(); - completedSemaphore.Wait(); - // Dispose flush buffer - buffer?.Dispose(); - disposed = true; - } - } - - internal override void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) - { - IDevice usedDevice = deltaLogDevice; - completed = new CountdownEvent(numPages); - for (long readPage = readPageStart; readPage < (readPageStart + numPages); readPage++) - { - int pageIndex = (int)(readPage % frame.frameSize); - if (frame.frame[pageIndex] == null) - { - frame.Allocate(pageIndex); - } - else - { - frame.Clear(pageIndex); - } - var asyncResult = new PageAsyncReadResult() - { - page = readPage, - context = context, - handle = completed, - frame = frame - }; - - ulong offsetInFile = (ulong)(AlignedPageSizeBytes * readPage); - - uint readLength = (uint)AlignedPageSizeBytes; - long adjustedUntilAddress = (AlignedPageSizeBytes * (untilAddress >> LogPageSizeBits) + (untilAddress & PageSizeMask)); - - if (adjustedUntilAddress > 0 && ((adjustedUntilAddress - (long)offsetInFile) < PageSize)) - { - readLength = (uint)(adjustedUntilAddress - (long)offsetInFile); - readLength = (uint)(Align(readLength)); - } - - if (device != null) - offsetInFile = (ulong)(AlignedPageSizeBytes * (readPage - devicePageOffset)); - - usedDevice.ReadAsync(offsetInFile, (IntPtr)frame.pointers[pageIndex], readLength, AsyncReadPagesCallback, asyncResult); - } - } - - private static unsafe ref DeltalogHeader GetHeader(long physicalAddress) => ref Unsafe.AsRef((void*)physicalAddress); - - private unsafe void AsyncReadPagesCallback(uint errorCode, uint numBytes, object context) - { - try - { - var result = (PageAsyncReadResult)context; - - if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncReadPagesCallback)} error: {{errorCode}}", errorCode); - result.cts?.Cancel(); - } - Debug.Assert(result.freeBuffer1 == null); - - if (errorCode == 0) - result.handle?.Signal(); - - Interlocked.MemoryBarrier(); - } - catch when (disposed) { } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private long Align(long length) - { - return (length + sectorSize - 1) & ~(sectorSize - 1); - } - - /// - /// Get next entry - /// - /// - /// - /// - /// - public unsafe bool GetNext(out long physicalAddress, out int entryLength, out DeltaLogEntryType type) - { - while (true) - { - physicalAddress = 0; - entryLength = 0; - currentAddress = nextAddress; - type = DeltaLogEntryType.DELTA; - - var _currentPage = currentAddress >> LogPageSizeBits; - var _currentFrame = _currentPage % frameSize; - var _currentOffset = currentAddress & PageSizeMask; - var _headAddress = long.MaxValue; - - if (disposed) - return false; - - var _endAddress = endAddress; - if (tailAddress > _endAddress) _endAddress = tailAddress; - - if (currentAddress >= _endAddress) - return false; - - - if (BufferAndLoad(currentAddress, _currentPage, _currentFrame, _headAddress, _endAddress)) - continue; - physicalAddress = frame.GetPhysicalAddress(_currentFrame, _currentOffset); - - // Get and check entry length - entryLength = GetHeader(physicalAddress).Length; - type = GetHeader(physicalAddress).Type; - - if (entryLength == 0) - { - if (_currentOffset == 0) - { - // We found a hole at beginning of page, this must imply end of delta log - return false; - } - - // Hole at end of page, skip to next page - currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits; - if (!Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _)) - return false; - else - continue; - } - - int recordSize = (int)(Align(_currentOffset + HeaderSize + entryLength) - _currentOffset); - if (entryLength < 0 || (_currentOffset + recordSize > PageSize)) - { - currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits; - if (!Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _)) - return false; - else - continue; - } - - // Verify checksum - if (!VerifyBlockChecksum((byte*)physicalAddress, entryLength)) - { - currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits; - if (!Utility.MonotonicUpdate(ref nextAddress, currentAddress, out _)) - return false; - else - continue; - } - physicalAddress += HeaderSize; - - if ((currentAddress & PageSizeMask) + recordSize == PageSize) - currentAddress = (1 + (currentAddress >> LogPageSizeBits)) << LogPageSizeBits; - else - currentAddress += recordSize; - - if (Utility.MonotonicUpdate(ref nextAddress, currentAddress, out long oldCurrentAddress)) - { - currentAddress = oldCurrentAddress; - return true; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe bool VerifyBlockChecksum(byte* ptr, int length) - { - var cs = Utility.XorBytes(ptr + 8, length + HeaderSize - 8); - if (cs != GetHeader((long)ptr).Checksum) - { - return false; - } - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe void SetBlockHeader(int length, DeltaLogEntryType type, byte* dest) - { - ref var header = ref GetHeader((long)dest); - header.Length = length; - header.Type = type; - header.Checksum = Utility.XorBytes(dest + 8, length + HeaderSize - 8); - } - - /// - /// Initialize for writes - /// - /// - public void InitializeForWrites(SectorAlignedBufferPool memory) - { - this.memory = memory; - buffer = memory.Get(PageSize); - } - - /// - /// Returns allocated region on delta log to write to - /// - /// Max usable size of allocated region - /// Address for caller to write to - public unsafe void Allocate(out int maxEntryLength, out long physicalAddress) - { - long pageEndAddress = (1 + (tailAddress >> LogPageSizeBits)) << LogPageSizeBits; - long dataStartAddress = tailAddress + HeaderSize; - maxEntryLength = (int)(pageEndAddress - dataStartAddress); - int offset = (int)(dataStartAddress & PageSizeMask); - physicalAddress = (long)buffer.aligned_pointer + offset; - } - - /// - /// Seal allocated region for given size, write header, move tail address - /// - /// Entry length - /// Optional record type - public unsafe void Seal(int entryLength, DeltaLogEntryType type = DeltaLogEntryType.DELTA) - { - if (entryLength > 0) - { - int offset = (int)(tailAddress & PageSizeMask); - SetBlockHeader(entryLength, type, buffer.aligned_pointer + offset); - - long oldTailAddress = tailAddress; - tailAddress += HeaderSize + entryLength; - tailAddress = Align(tailAddress); - - long pageEndAddress = (1 + (tailAddress >> LogPageSizeBits)) << LogPageSizeBits; - if (tailAddress + HeaderSize >= pageEndAddress) - tailAddress = (1 + (tailAddress >> LogPageSizeBits)) << LogPageSizeBits; - - if ((oldTailAddress >> LogPageSizeBits) < (tailAddress >> LogPageSizeBits)) - FlushPage(); - } - else - { - // Unable to use entry, skip to next page - tailAddress = (1 + (tailAddress >> LogPageSizeBits)) << LogPageSizeBits; - FlushPage(); - } - } - - private unsafe void FlushPage() - { - long pageStartAddress = tailAddress & ~PageSizeMask; - int offset = (int)(tailAddress & PageSizeMask); - if (offset == 0) - pageStartAddress = (tailAddress - 1) & ~PageSizeMask; - if (flushedUntilAddress > pageStartAddress) - pageStartAddress = flushedUntilAddress; - int startOffset = (int)(pageStartAddress & PageSizeMask); - - var asyncResult = new PageAsyncFlushResult { count = 1, freeBuffer1 = buffer }; - var alignedBlockSize = Align(tailAddress - pageStartAddress); - Interlocked.Increment(ref issuedFlush); - deltaLogDevice.WriteAsync((IntPtr)buffer.aligned_pointer + startOffset, - (ulong)pageStartAddress, - (uint)alignedBlockSize, AsyncFlushPageToDeviceCallback, asyncResult); - flushedUntilAddress = tailAddress; - buffer = memory.Get(PageSize); - } - - /// - /// Flush - /// - /// - public async Task FlushAsync() - { - // Flush last page if needed - long pageStartAddress = tailAddress & ~PageSizeMask; - if (tailAddress > pageStartAddress) - FlushPage(); - if (Interlocked.Decrement(ref issuedFlush) == 0) - completedSemaphore.Release(); - await completedSemaphore.WaitAsync().ConfigureAwait(false); - Interlocked.Increment(ref issuedFlush); - completedSemaphore = new SemaphoreSlim(0); - } - - /// - /// IOCompletion callback for page flush - /// - /// - /// - /// - private void AsyncFlushPageToDeviceCallback(uint errorCode, uint numBytes, object context) - { - try - { - if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncFlushPageToDeviceCallback)} error: {{errorCode}}", errorCode); - } - - PageAsyncFlushResult result = (PageAsyncFlushResult)context; - if (Interlocked.Decrement(ref result.count) == 0) - { - result.Free(); - } - if (Interlocked.Decrement(ref issuedFlush) == 0) - completedSemaphore.Release(); - } - catch when (disposed) { } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/ICheckpointManager.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/ICheckpointManager.cs index 68c44e8e26d..fe70ca13c96 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/ICheckpointManager.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/ICheckpointManager.cs @@ -30,6 +30,13 @@ namespace Tsavorite.core /// public interface ICheckpointManager : IDisposable { + /// + /// Whether Tsavorite should perform internal cleanup of checkpoint snapshot files and hybrid log segments + /// during the checkpoint state machine. When false, cleanup of hlog segments is avoided and the external + /// layer is responsible for managing checkpoint lifecycle (e.g., cluster mode with reader-safe deletion). + /// + bool PerformAutomaticCleanup { get; } + /// /// Get current cookie /// @@ -91,20 +98,6 @@ public interface ICheckpointManager : IDisposable /// void CheckpointVersionShiftEnd(long oldVersion, long newVersion, bool isStreaming); - /// - /// Commit log incremental checkpoint (incremental snapshot) - /// - /// - /// - /// - void CommitLogIncrementalCheckpoint(Guid logToken, byte[] commitMetadata, DeltaLog deltaLog); - - /// - /// Cleanup log incremental checkpoint (incremental snapshot) - /// - /// - void CleanupLogIncrementalCheckpoint(Guid logToken); - /// /// Retrieve commit metadata for specified index checkpoint /// @@ -116,11 +109,8 @@ public interface ICheckpointManager : IDisposable /// Retrieve commit metadata for specified log checkpoint /// /// Token - /// Delta log - /// whether or not to scan through the delta log to acquire latest entry. make sure the delta log points to the tail address immediately following the returned metadata. - /// version upper bound to scan for in the delta log. Function will return the largest version metadata no greater than the given version. /// Metadata, or null if invalid - byte[] GetLogCheckpointMetadata(Guid logToken, DeltaLog deltaLog, bool scanDelta = false, long recoverTo = -1); + byte[] GetLogCheckpointMetadata(Guid logToken); /// /// Get list of index checkpoint tokens, in order of usage preference @@ -155,13 +145,6 @@ public interface ICheckpointManager : IDisposable /// IDevice GetSnapshotObjectLogDevice(Guid token); - /// - /// Provide device to store incremental (delta) snapshot of log (required only for incremental snapshot checkpoints) - /// - /// - /// - IDevice GetDeltaLogDevice(Guid token); - /// /// Cleanup all data (subfolder) related to the given guid by this manager /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexCheckpoint.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexCheckpoint.cs index ea0400d606a..360b16e71f8 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexCheckpoint.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexCheckpoint.cs @@ -11,9 +11,9 @@ namespace Tsavorite.core { internal unsafe delegate void SkipReadCache(HashBucket* bucket); - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Derived class facing persistence API internal IndexCheckpointInfo _indexCheckpoint; @@ -54,8 +54,8 @@ internal bool IsIndexFuzzyCheckpointCompleted() internal void AddIndexCheckpointWaitingList(StateMachineDriver stateMachineDriver) { - stateMachineDriver.AddToWaitingList(mainIndexCheckpointSemaphore); - stateMachineDriver.AddToWaitingList(overflowBucketsAllocator.GetCheckpointSemaphore()); + stateMachineDriver.AddToWaitingList(mainIndexCheckpointTcs.Task, StateMachineTaskType.IndexCheckpointSMTaskMainIndexCheckpoint); + stateMachineDriver.AddToWaitingList(overflowBucketsAllocator.GetCheckpointTask(), StateMachineTaskType.IndexCheckpointSMTaskOverflowBucketsCheckpoint); } internal async ValueTask IsIndexFuzzyCheckpointCompletedAsync(CancellationToken token = default) @@ -67,18 +67,17 @@ internal async ValueTask IsIndexFuzzyCheckpointCompletedAsync(CancellationToken await t2.ConfigureAwait(false); } - // Implementation of an asynchronous checkpointing scheme // for main hash index of Tsavorite private int mainIndexCheckpointCallbackCount; - private SemaphoreSlim mainIndexCheckpointSemaphore; + private TaskCompletionSource mainIndexCheckpointTcs; private SemaphoreSlim throttleIndexCheckpointFlushSemaphore; internal unsafe void BeginMainIndexCheckpoint(int version, IDevice device, out ulong numBytesWritten, bool useReadCache = false, SkipReadCache skipReadCache = default, int throttleCheckpointFlushDelayMs = -1) { long totalSize = state[version].size * sizeof(HashBucket); numBytesWritten = (ulong)totalSize; - mainIndexCheckpointSemaphore = new SemaphoreSlim(0); + mainIndexCheckpointTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); if (throttleCheckpointFlushDelayMs >= 0) Task.Run(FlushRunner); @@ -87,64 +86,72 @@ internal unsafe void BeginMainIndexCheckpoint(int version, IDevice device, out u void FlushRunner() { - int numChunks = 1; - if (useReadCache && (totalSize > (1L << 25))) - { - numChunks = (int)Math.Ceiling((double)totalSize / (1L << 25)); - numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); - } - else if (totalSize > uint.MaxValue) + try { - numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue); - numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); - } + int numChunks = 1; + if (useReadCache && (totalSize > (1L << 25))) + { + numChunks = (int)Math.Ceiling((double)totalSize / (1L << 25)); + numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); + } + else if (totalSize > uint.MaxValue) + { + numChunks = (int)Math.Ceiling((double)totalSize / (long)uint.MaxValue); + numChunks = (int)Math.Pow(2, Math.Ceiling(Math.Log(numChunks, 2))); + } - uint chunkSize = (uint)(totalSize / numChunks); - mainIndexCheckpointCallbackCount = numChunks; + uint chunkSize = (uint)(totalSize / numChunks); + mainIndexCheckpointCallbackCount = numChunks; - if (throttleCheckpointFlushDelayMs >= 0) - throttleIndexCheckpointFlushSemaphore = new SemaphoreSlim(0); - HashBucket* start = state[version].tableAligned; + if (throttleCheckpointFlushDelayMs >= 0) + throttleIndexCheckpointFlushSemaphore = new SemaphoreSlim(0); + HashBucket* start = state[version].tableAligned; - ulong numBytesWritten = 0; - for (int index = 0; index < numChunks; index++) - { - IntPtr chunkStartBucket = (IntPtr)((byte*)start + (index * chunkSize)); - HashIndexPageAsyncFlushResult result = default; - result.chunkIndex = index; - if (!useReadCache) - { - device.WriteAsync(chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); - } - else + ulong numBytesWritten = 0; + for (int index = 0; index < numChunks; index++) { - result.mem = new SectorAlignedMemory((int)chunkSize, (int)device.SectorSize); - bool prot = false; - if (!epoch.ThisInstanceProtected()) + IntPtr chunkStartBucket = (IntPtr)((byte*)start + (index * chunkSize)); + HashIndexPageAsyncFlushResult result = default; + result.chunkIndex = index; + if (!useReadCache) { - prot = true; - epoch.Resume(); + device.WriteAsync(chunkStartBucket, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); } - Buffer.MemoryCopy((void*)chunkStartBucket, result.mem.aligned_pointer, chunkSize, chunkSize); - for (int j = 0; j < chunkSize; j += sizeof(HashBucket)) + else { - skipReadCache((HashBucket*)(result.mem.aligned_pointer + j)); + result.mem = new SectorAlignedMemory((int)chunkSize, (int)device.SectorSize); + bool prot = false; + if (!epoch.ThisInstanceProtected()) + { + prot = true; + epoch.Resume(); + } + Buffer.MemoryCopy((void*)chunkStartBucket, result.mem.aligned_pointer, chunkSize, chunkSize); + for (int j = 0; j < chunkSize; j += sizeof(HashBucket)) + { + skipReadCache((HashBucket*)(result.mem.aligned_pointer + j)); + } + if (prot) + epoch.Suspend(); + + device.WriteAsync((IntPtr)result.mem.aligned_pointer, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); } - if (prot) - epoch.Suspend(); - - device.WriteAsync((IntPtr)result.mem.aligned_pointer, numBytesWritten, chunkSize, AsyncPageFlushCallback, result); - } - if (throttleCheckpointFlushDelayMs >= 0) - { - throttleIndexCheckpointFlushSemaphore.Wait(); - Thread.Sleep(throttleCheckpointFlushDelayMs); + if (throttleCheckpointFlushDelayMs >= 0) + { + throttleIndexCheckpointFlushSemaphore.Wait(); + Thread.Sleep(throttleCheckpointFlushDelayMs); + } + numBytesWritten += chunkSize; } - numBytesWritten += chunkSize; - } - Debug.Assert(numBytesWritten == (ulong)totalSize); - throttleIndexCheckpointFlushSemaphore = null; + Debug.Assert(numBytesWritten == (ulong)totalSize); + throttleIndexCheckpointFlushSemaphore = null; + } + catch (Exception ex) + { + logger?.LogError(ex, "{method} failed while flushing index checkpoint", nameof(BeginMainIndexCheckpoint)); + mainIndexCheckpointTcs.TrySetException(ex); + } } } @@ -155,12 +162,10 @@ private bool IsMainIndexCheckpointCompleted() private async ValueTask IsMainIndexCheckpointCompletedAsync(CancellationToken token = default) { - var s = mainIndexCheckpointSemaphore; - await s.WaitAsync(token).ConfigureAwait(false); - s.Release(); + await mainIndexCheckpointTcs.Task.WaitAsync(token).ConfigureAwait(false); } - private unsafe void AsyncPageFlushCallback(uint errorCode, uint numBytes, object context) + private void AsyncPageFlushCallback(uint errorCode, uint numBytes, object context) { // Set the page status to flushed var mem = ((HashIndexPageAsyncFlushResult)context).mem; @@ -172,7 +177,7 @@ private unsafe void AsyncPageFlushCallback(uint errorCode, uint numBytes, object } if (Interlocked.Decrement(ref mainIndexCheckpointCallbackCount) == 0) { - mainIndexCheckpointSemaphore.Release(); + mainIndexCheckpointTcs.TrySetResult(true); } throttleIndexCheckpointFlushSemaphore?.Release(); } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexRecovery.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexRecovery.cs index f2a14b6162c..fed7c5ab07b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexRecovery.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/IndexRecovery.cs @@ -54,10 +54,8 @@ private ulong InitializeMainIndexRecovery(ref IndexCheckpointInfo info, bool isA Initialize(info.info.table_size, (int)sectorSize); } - BeginMainIndexRecovery(ht_version, info.main_ht_device, info.info.num_ht_bytes, isAsync); - var alignedIndexSize = (info.info.num_ht_bytes + (sectorSize - 1)) & ~((ulong)sectorSize - 1); return alignedIndexSize; } @@ -147,9 +145,7 @@ private bool IsMainIndexRecoveryCompleted(bool waitUntilComplete = false) private unsafe void AsyncPageReadCallback(uint errorCode, uint numBytes, object overlap) { if (errorCode != 0) - { logger?.LogError($"{nameof(AsyncPageReadCallback)} error: {{errorCode}}", errorCode); - } recoveryCountdown.Decrement(); } @@ -161,19 +157,19 @@ internal unsafe void DeleteTentativeEntries() var table_size_ = state[version].size; var ptable_ = state[version].tableAligned; - for (long bucket = 0; bucket < table_size_; ++bucket) + for (long bucket = 0; bucket < table_size_; bucket++) { HashBucket* b = ptable_ + bucket; while (true) { - for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; ++bucket_entry) + for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; bucket_entry++) { entry.word = b->bucket_entries[bucket_entry]; if (entry.Tentative) b->bucket_entries[bucket_entry] = 0; } // Reset any ephemeral bucket level locks - b->bucket_entries[Constants.kOverflowBucketIndex] &= Constants.kAddressMask; + b->bucket_entries[Constants.kOverflowBucketIndex] &= (long)LogAddress.kAddressBitMask; if (b->bucket_entries[Constants.kOverflowBucketIndex] == 0) break; b = (HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(b->bucket_entries[Constants.kOverflowBucketIndex]); } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Recovery.cs b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Recovery.cs index cefaf8a6898..2939f1b19cf 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Recovery.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Recovery/Recovery.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using static Tsavorite.core.Utility; namespace Tsavorite.core { @@ -15,36 +16,30 @@ internal enum FlushStatus { Pending, Done, Error }; internal sealed class RecoveryStatus { - public long endPage; - public long snapshotEndPage; - public long untilAddress; - public int capacity; - public int usableCapacity; - public CheckpointType checkpointType; - + /// Main log recovery device, obtained from CheckpointManager. public IDevice recoveryDevice; + /// The first page to recover; this is the page index of the snapshotStartAddress and is the page-offset into + /// the address range of the snapshot file (i.e. the page at the snapshot file's offset 0). This field is populated + /// from . public long recoveryDevicePageOffset; + /// Object log recovery device, obtained from CheckpointManager. public IDevice objectLogRecoveryDevice; - // These are circular buffers of 'capacity' size; the indexing wraps due to hlog.GetPageIndexForPage(). + /// Circular status buffer of 'capacity' size; the indexing wraps per hlog.GetPageIndexForPage(). public ReadStatus[] readStatus; + /// Circular status buffer of 'capacity' size; the indexing wraps per hlog.GetPageIndexForPage(). public FlushStatus[] flushStatus; + /// Signals completion of an in-progress page read. private readonly SemaphoreSlim readSemaphore = new(0); + /// Signals completion of an in-progress page flush. private readonly SemaphoreSlim flushSemaphore = new(0); - public RecoveryStatus(int capacity, int emptyPageCount, - long endPage, long untilAddress, CheckpointType checkpointType) + public RecoveryStatus(int bufferSize) { - this.capacity = capacity; - this.usableCapacity = capacity - emptyPageCount; - this.endPage = endPage; - this.untilAddress = untilAddress; - this.checkpointType = checkpointType; - - readStatus = new ReadStatus[capacity]; - flushStatus = new FlushStatus[capacity]; - for (int i = 0; i < capacity; i++) + readStatus = new ReadStatus[bufferSize]; + flushStatus = new FlushStatus[bufferSize]; + for (int i = 0; i < bufferSize; i++) { flushStatus[i] = FlushStatus.Done; readStatus[i] = ReadStatus.Pending; @@ -55,13 +50,13 @@ public RecoveryStatus(int capacity, int emptyPageCount, internal void SignalRead(int pageIndex) { readStatus[pageIndex] = ReadStatus.Done; - readSemaphore.Release(); + _ = readSemaphore.Release(); } internal void SignalReadError(int pageIndex) { readStatus[pageIndex] = ReadStatus.Error; - readSemaphore.Release(); + _ = readSemaphore.Release(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -86,13 +81,13 @@ internal async ValueTask WaitReadAsync(int pageIndex, CancellationToken cancella internal void SignalFlushed(int pageIndex) { flushStatus[pageIndex] = FlushStatus.Done; - flushSemaphore.Release(); + _ = flushSemaphore.Release(); } internal void SignalFlushedError(int pageIndex) { flushStatus[pageIndex] = FlushStatus.Error; - flushSemaphore.Release(); + _ = flushSemaphore.Release(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -117,6 +112,8 @@ internal void Dispose() { recoveryDevice.Dispose(); objectLogRecoveryDevice.Dispose(); + readSemaphore.Dispose(); + flushSemaphore.Dispose(); } } @@ -139,27 +136,31 @@ internal RecoveryOptions(long headAddress, long fuzzyRegionStartAddress, bool un /// public struct LogFileInfo { - /// - /// Snapshot file end address (start address is always 0) - /// + /// Snapshot file end address (start address is always 0). public long snapshotFileEndAddress; - /// - /// Hybrid log file start address - /// + + /// Hybrid log file start address public long hybridLogFileStartAddress; - /// - /// Hybrid log file end address - /// + + /// Hybrid log file end address public long hybridLogFileEndAddress; - /// - /// Delta log tail address - /// - public long deltaLogTailAddress; + + /// True if this snapshot had object log records + public bool hasSnapshotObjects; + + /// Address of ; the start of the lowest object log segment + /// in use by the hybrid log at snapshot PREPARE time + public long hybridLogObjectFileStartAddress; + /// The objectLogTail taken at the start of WAIT_FLUSH, corresponding to the hlog's FlushedUntilAddress at that point + public long hybridLogObjectFileEndAddress; + /// The snapshotEndObjectLogTail taken at PERSISTENCE_CALLBACK, which corresponds to the object log position for the final TailAddress + /// written by the checkpoint. (Start address is always 0.) + public long snapshotObjectFileEndAddress; } - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { private const long NoPageFreed = -1; @@ -182,16 +183,12 @@ public void GetLatestCheckpointTokens(out Guid hlogToken, out Guid indexToken, o } using var current = new HybridLogCheckpointInfo(); - // Make sure we consider delta log in order to compute latest checkpoint version - current.Recover(hlogToken, checkpointManager, hlogBase.LogPageSizeBits, - out var _, true); + current.Recover(hlogToken, checkpointManager, out var _); storeVersion = current.info.nextVersion; GetClosestIndexCheckpointInfo(ref recoveredHlcInfo, out indexToken, out var recoveredICInfo); - if (recoveredICInfo.IsDefault()) - { + if (recoveredICInfo.IsDefault) logger?.LogInformation("No index checkpoint found, returning default index token in GetLatestCheckpointTokens"); - } } finally { @@ -202,7 +199,6 @@ public void GetLatestCheckpointTokens(out Guid hlogToken, out Guid indexToken, o /// /// Get HLog latest version /// - /// public long GetLatestCheckpointVersion() { GetClosestHybridLogCheckpointInfo(-1, out var hlogToken, out var hlcInfo, out var _); @@ -211,31 +207,51 @@ public long GetLatestCheckpointVersion() return -1; using var current = new HybridLogCheckpointInfo(); - // Make sure we consider delta log in order to compute latest checkpoint version - current.Recover(hlogToken, checkpointManager, hlogBase.LogPageSizeBits, - out var _, true); + current.Recover(hlogToken, checkpointManager, out var _); return current.info.nextVersion; } /// /// Get size of snapshot files for token /// - /// - /// - /// - public LogFileInfo GetLogFileSize(Guid token, long version = -1) + public LogFileInfo GetLogFileSize(Guid token) { using var current = new HybridLogCheckpointInfo(); - // We find the latest checkpoint metadata for the given token, including scanning the delta log for the latest metadata - current.Recover(token, checkpointManager, hlogBase.LogPageSizeBits, - out var _, true, version); - long snapshotDeviceOffset = hlogBase.GetPage(current.info.snapshotStartFlushedLogicalAddress) << hlogBase.LogPageSizeBits; + current.Recover(token, checkpointManager, out var _); + var hasSnapshotObjects = current.info.snapshotEndObjectLogTail.HasData; + var snapshotDeviceOffset = hlogBase.GetLogicalAddressOfStartOfPage(hlogBase.GetPage(current.info.snapshotStartFlushedLogicalAddress)); return new LogFileInfo { - snapshotFileEndAddress = current.info.snapshotFinalLogicalAddress - snapshotDeviceOffset, - hybridLogFileStartAddress = hlogBase.GetPage(current.info.beginAddress) << hlogBase.LogPageSizeBits, + // Hybrid (main log file) info: + // - The main log address range is from: + // - BeginAddress at PREPARE to... + // - FlushedUntilAddress at PERSISTENCE_CALLBACK. + // - The snapshot address range starts at 0 in the snapshot files and includes all main-log data until the final TailAddress. In detail, it is from: + // - 0, but the start offset is FlushedUntilAddress taken at the start of WAIT_FLUSH (which is used to calculate this.snapshotDeviceOffset) to... + // - TailAddress taken at the start of WAIT_FLUSH minus the start offset. This TailAddress is the maximum logical address that will be written to the snapshot. + // The overlap between the FlushedUntilAddress for the main log being recorded after the flush completes and the FlushedUntilAddress for the snapshot + // being recorded before the flush starts ensures there is no gap. + hybridLogFileStartAddress = hlogBase.GetLogicalAddressOfStartOfPage(hlogBase.GetPage(current.info.beginAddress)), hybridLogFileEndAddress = current.info.flushedLogicalAddress, - deltaLogTailAddress = current.info.deltaTailAddress, + snapshotFileEndAddress = current.info.snapshotFinalLogicalAddress - snapshotDeviceOffset, + + // Object log file info: + // - The object log address range is from: + // - The start of the in-use object segment corresponding to main-log BeginAddress at PREPARE (matching this.hybridLogFileStartAddress) to... + // - The hLogEndObjectLogTail taken at PERSISTENCE_CALLBACK (matching this.hybridLogFileEndAddress). + // - The snapshot address range starts at 0 in the snapshot file and includes all object-log data until the final TailAddress. In detail, it is from: + // - The objectLogTail taken at the start of WAIT_FLUSH as info.snapshotStartObjectLogTail, corresponding to the main log's FlushedUntilAddress at + // that point (which is used to calculate this.snapshotDeviceOffset) to... + // - The snapshotEndObjectLogTail which is taken at PERSISTENCE_CALLBACK, which corresponds to the main-log TailAddress taken at WAIT_FLUSH. + // The snapshotEndObjectLogTail grows during the Flush, so is not final until PERSISTENCE_CALLBACK; but it will only be written for records + // up to the TailAddress at the start of WAIT_FLUSH. + // Note that there are no object-log segments for the mutable region of the hybrid log; they are not written until ReadOnlyAddress growth triggers + // a main-log Flush. However the snapshot does cause object-log segments for the mutable range to be written. + hasSnapshotObjects = hasSnapshotObjects, + hybridLogObjectFileStartAddress = hasSnapshotObjects ? (long)current.info.beginAddressObjectLogSegment << current.info.hlogEndObjectLogTail.SegmentSizeBits : 0, + hybridLogObjectFileEndAddress = hasSnapshotObjects ? (long)current.info.snapshotStartObjectLogTail.CurrentAddress : 0, + snapshotObjectFileEndAddress = hasSnapshotObjects ? (long)current.info.snapshotEndObjectLogTail.CurrentAddress : 0, + }; } @@ -271,8 +287,7 @@ private void GetClosestHybridLogCheckpointInfo( try { current = new HybridLogCheckpointInfo(); - current.Recover(hybridLogToken, checkpointManager, hlogBase.LogPageSizeBits, - out var currCookie, false); + current.Recover(hybridLogToken, checkpointManager, out var currCookie); var distanceToTarget = (requestedVersion == -1 ? long.MaxValue : requestedVersion) - current.info.version; // This is larger than intended version, cannot recover to this. if (distanceToTarget < 0) continue; @@ -340,54 +355,23 @@ private void GetClosestIndexCheckpointInfo(ref HybridLogCheckpointInfo recovered } } - private void FindRecoveryInfo(long requestedVersion, out HybridLogCheckpointInfo recoveredHlcInfo, - out IndexCheckpointInfo recoveredICInfo) + private void FindRecoveryInfo(long requestedVersion, out HybridLogCheckpointInfo recoveredHlcInfo, out IndexCheckpointInfo recoveredICInfo) { logger?.LogInformation("********* Primary Recovery Information ********"); GetClosestHybridLogCheckpointInfo(requestedVersion, out var closestToken, out recoveredHlcInfo, out recoveredCommitCookie); - - if (recoveredHlcInfo.IsDefault()) + if (recoveredHlcInfo.IsDefault) throw new TsavoriteNoHybridLogException("Unable to find valid HybridLog token"); - if (recoveredHlcInfo.deltaLog != null) - { - recoveredHlcInfo.Dispose(); - // need to actually scan delta log now - recoveredHlcInfo.Recover(closestToken, checkpointManager, hlogBase.LogPageSizeBits, out _, true); - } recoveredHlcInfo.info.DebugPrint(logger); GetClosestIndexCheckpointInfo(ref recoveredHlcInfo, out _, out recoveredICInfo); - - if (recoveredICInfo.IsDefault()) - { + if (recoveredICInfo.IsDefault) logger?.LogInformation("No index checkpoint found, recovering from beginning of log"); - } } private static bool IsCompatible(in IndexRecoveryInfo indexInfo, in HybridLogRecoveryInfo recoveryInfo) - { - var l1 = indexInfo.finalLogicalAddress; - var l2 = recoveryInfo.finalLogicalAddress; - return l1 <= l2; - } - - private long InternalRecover(Guid indexToken, Guid hybridLogToken, int numPagesToPreload, bool undoNextVersion, long recoverTo) - { - GetRecoveryInfo(indexToken, hybridLogToken, out HybridLogCheckpointInfo recoveredHLCInfo, out IndexCheckpointInfo recoveredICInfo); - if (recoverTo != -1 && recoveredHLCInfo.deltaLog == null) - { - throw new TsavoriteException("Recovering to a specific version within a token is only supported for incremental snapshots"); - } - return InternalRecover(recoveredICInfo, recoveredHLCInfo, numPagesToPreload, undoNextVersion, recoverTo); - } - - private ValueTask InternalRecoverAsync(Guid indexToken, Guid hybridLogToken, int numPagesToPreload, bool undoNextVersion, long recoverTo, CancellationToken cancellationToken) - { - GetRecoveryInfo(indexToken, hybridLogToken, out HybridLogCheckpointInfo recoveredHLCInfo, out IndexCheckpointInfo recoveredICInfo); - return InternalRecoverAsync(recoveredICInfo, recoveredHLCInfo, numPagesToPreload, undoNextVersion, recoverTo, cancellationToken); - } + => indexInfo.finalLogicalAddress <= recoveryInfo.finalLogicalAddress; private void GetRecoveryInfo(Guid indexToken, Guid hybridLogToken, out HybridLogCheckpointInfo recoveredHLCInfo, out IndexCheckpointInfo recoveredICInfo) { @@ -398,7 +382,7 @@ private void GetRecoveryInfo(Guid indexToken, Guid hybridLogToken, out HybridLog // Recovery appropriate context information recoveredHLCInfo = new HybridLogCheckpointInfo(); - recoveredHLCInfo.Recover(hybridLogToken, checkpointManager, hlogBase.LogPageSizeBits, out recoveredCommitCookie, true); + recoveredHLCInfo.Recover(hybridLogToken, checkpointManager, out recoveredCommitCookie); recoveredHLCInfo.info.DebugPrint(logger); try { @@ -414,18 +398,11 @@ private void GetRecoveryInfo(Guid indexToken, Guid hybridLogToken, out HybridLog recoveredICInfo = default; } - if (recoveredICInfo.IsDefault()) - { + // Verify that the index and log checkpoints are compatible for recovery + if (recoveredICInfo.IsDefault) logger?.LogInformation("Invalid index checkpoint token, recovering from beginning of log"); - } - else - { - // Check if the two checkpoints are compatible for recovery - if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info)) - { - throw new TsavoriteException("Cannot recover from (" + indexToken.ToString() + "," + hybridLogToken.ToString() + ") checkpoint pair!\n"); - } - } + else if (!IsCompatible(recoveredICInfo.info, recoveredHLCInfo.info)) + throw new TsavoriteException("Cannot recover from (" + indexToken.ToString() + "," + hybridLogToken.ToString() + ") checkpoint pair!\n"); } /// @@ -444,30 +421,38 @@ public void Reset() lastVersion = 0; } + /// Synchronous recovery driver + private long InternalRecover(Guid indexToken, Guid hybridLogToken, int numPagesToPreload, bool undoNextVersion) + { + GetRecoveryInfo(indexToken, hybridLogToken, out var recoveredHLCInfo, out var recoveredICInfo); + return InternalRecover(recoveredICInfo, recoveredHLCInfo, numPagesToPreload, undoNextVersion); + } - private long InternalRecover(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, bool undoNextVersion, long recoverTo) + /// Synchronous recovery driver + private long InternalRecover(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, bool undoNextVersion) { hlogBase.VerifyRecoveryInfo(recoveredHLCInfo, false); - if (hlogBase.GetTailAddress() > hlog.GetFirstValidLogicalAddress(0)) + if (hlogBase.GetTailAddress() > hlogBase.GetFirstValidLogicalAddressOnPage(0)) { logger?.LogInformation("Recovery called on non-empty log - resetting to empty state first. Make sure store is quiesced before calling Recover on a running store."); Reset(); } - if (!RecoverToInitialPage(recoveredICInfo, recoveredHLCInfo, out long recoverFromAddress)) + if (!GetInitialRecoveryAddress(recoveredICInfo, recoveredHLCInfo, out long recoverFromAddress)) RecoverFuzzyIndex(recoveredICInfo); if (!SetRecoveryPageRanges(recoveredHLCInfo, numPagesToPreload, recoverFromAddress, out long tailAddress, out long headAddress, out long scanFromAddress)) return -1; - RecoveryOptions options = new(headAddress, recoveredHLCInfo.info.startLogicalAddress, undoNextVersion); + RecoveryOptions options = new(headAddress, fuzzyRegionStartAddress: recoveredHLCInfo.info.startLogicalAddress, undoNextVersion); - long readOnlyAddress; - long lastFreedPage; // Make index consistent for version v + long readOnlyAddress, lastFreedPage; if (recoveredHLCInfo.info.useSnapshotFile == 0) { - lastFreedPage = RecoverHybridLog(scanFromAddress, recoverFromAddress, recoveredHLCInfo.info.finalLogicalAddress, recoveredHLCInfo.info.nextVersion, CheckpointType.FoldOver, options); + lastFreedPage = RecoverHybridLog(scanFromAddress, recoverFromAddress, untilAddress: recoveredHLCInfo.info.finalLogicalAddress, + recoveredHLCInfo.info.nextVersion, CheckpointType.FoldOver, options); + readOnlyAddress = tailAddress; } else @@ -475,11 +460,17 @@ private long InternalRecover(IndexCheckpointInfo recoveredICInfo, HybridLogCheck if (recoveredHLCInfo.info.flushedLogicalAddress < headAddress) headAddress = recoveredHLCInfo.info.flushedLogicalAddress; - // First recover from index starting point (fromAddress) to snapshot starting point (flushedLogicalAddress) - lastFreedPage = RecoverHybridLog(scanFromAddress, recoverFromAddress, recoveredHLCInfo.info.flushedLogicalAddress, recoveredHLCInfo.info.nextVersion, CheckpointType.Snapshot, options); - // Then recover snapshot into mutable region - var snapshotLastFreedPage = RecoverHybridLogFromSnapshotFile(recoveredHLCInfo.info.flushedLogicalAddress, recoverFromAddress, recoveredHLCInfo.info.finalLogicalAddress, recoveredHLCInfo.info.snapshotStartFlushedLogicalAddress, - recoveredHLCInfo.info.snapshotFinalLogicalAddress, recoveredHLCInfo.info.nextVersion, recoveredHLCInfo.info.guid, options, recoveredHLCInfo.deltaLog, recoverTo); + // First recover from index starting point (fromAddress) to snapshot starting point (flushedLogicalAddress taken at PERSISTENCE_CALLBACK, so it includes + // any flushes to the hybrid log files due to OnPagesMarkedReadOnly while we were flushing to the snapshot files). + lastFreedPage = RecoverHybridLog(scanFromAddress, recoverFromAddress, untilAddress: recoveredHLCInfo.info.flushedLogicalAddress, + recoveredHLCInfo.info.nextVersion, CheckpointType.Snapshot, options); + + // Then recover snapshot into mutable region. Note that the ObjectAllocator will not write object log records for the mutable region; + // that only happens during flushes due to OnPagesMarkedReadOnly. + var snapshotLastFreedPage = RecoverHybridLogFromSnapshotFile(scanFromAddress: recoveredHLCInfo.info.flushedLogicalAddress, + recoverFromAddress, untilAddress: recoveredHLCInfo.info.finalLogicalAddress, + snapshotStartAddress: recoveredHLCInfo.info.snapshotStartFlushedLogicalAddress, snapshotEndAddress: recoveredHLCInfo.info.snapshotFinalLogicalAddress, + recoveredHLCInfo.info.nextVersion, recoveredHLCInfo.info.guid, options); if (snapshotLastFreedPage != NoPageFreed) lastFreedPage = snapshotLastFreedPage; @@ -491,30 +482,38 @@ private long InternalRecover(IndexCheckpointInfo recoveredICInfo, HybridLogCheck return recoveredHLCInfo.info.version; } - private async ValueTask InternalRecoverAsync(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, bool undoNextVersion, long recoverTo, CancellationToken cancellationToken) + /// Aynchronous recovery driver + private ValueTask InternalRecoverAsync(Guid indexToken, Guid hybridLogToken, int numPagesToPreload, bool undoNextVersion, CancellationToken cancellationToken) + { + GetRecoveryInfo(indexToken, hybridLogToken, out var recoveredHLCInfo, out var recoveredICInfo); + return InternalRecoverAsync(recoveredICInfo, recoveredHLCInfo, numPagesToPreload, undoNextVersion, cancellationToken); + } + + /// Asynchronous recovery driver + private async ValueTask InternalRecoverAsync(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, bool undoNextVersion, CancellationToken cancellationToken) { hlogBase.VerifyRecoveryInfo(recoveredHLCInfo, false); - if (hlogBase.GetTailAddress() > hlog.GetFirstValidLogicalAddress(0)) + if (hlogBase.GetTailAddress() > hlogBase.GetFirstValidLogicalAddressOnPage(0)) { logger?.LogInformation("Recovery called on non-empty log - resetting to empty state first. Make sure store is quiesced before calling Recover on a running store."); Reset(); } - if (!RecoverToInitialPage(recoveredICInfo, recoveredHLCInfo, out long recoverFromAddress)) + if (!GetInitialRecoveryAddress(recoveredICInfo, recoveredHLCInfo, out long recoverFromAddress)) await RecoverFuzzyIndexAsync(recoveredICInfo, cancellationToken).ConfigureAwait(false); if (!SetRecoveryPageRanges(recoveredHLCInfo, numPagesToPreload, recoverFromAddress, out long tailAddress, out long headAddress, out long scanFromAddress)) return -1; - RecoveryOptions options = new(headAddress, recoveredHLCInfo.info.startLogicalAddress, undoNextVersion); + RecoveryOptions options = new(headAddress, fuzzyRegionStartAddress: recoveredHLCInfo.info.startLogicalAddress, undoNextVersion); - long readOnlyAddress; - long lastFreedPage; // Make index consistent for version v + long readOnlyAddress, lastFreedPage; if (recoveredHLCInfo.info.useSnapshotFile == 0) { - lastFreedPage = await RecoverHybridLogAsync(scanFromAddress, recoverFromAddress, recoveredHLCInfo.info.finalLogicalAddress, recoveredHLCInfo.info.nextVersion, CheckpointType.FoldOver, - options, cancellationToken).ConfigureAwait(false); + lastFreedPage = await RecoverHybridLogAsync(scanFromAddress, recoverFromAddress, untilAddress: recoveredHLCInfo.info.finalLogicalAddress, + recoveredHLCInfo.info.nextVersion, CheckpointType.FoldOver, options, cancellationToken).ConfigureAwait(false); + readOnlyAddress = tailAddress; } else @@ -522,12 +521,18 @@ private async ValueTask InternalRecoverAsync(IndexCheckpointInfo recovered if (recoveredHLCInfo.info.flushedLogicalAddress < headAddress) headAddress = recoveredHLCInfo.info.flushedLogicalAddress; - // First recover from index starting point (fromAddress) to snapshot starting point (flushedLogicalAddress) - lastFreedPage = await RecoverHybridLogAsync(scanFromAddress, recoverFromAddress, recoveredHLCInfo.info.flushedLogicalAddress, recoveredHLCInfo.info.nextVersion, CheckpointType.Snapshot, - new RecoveryOptions(headAddress, recoveredHLCInfo.info.startLogicalAddress, undoNextVersion), cancellationToken).ConfigureAwait(false); - // Then recover snapshot into mutable region - var snapshotLastFreedPage = await RecoverHybridLogFromSnapshotFileAsync(recoveredHLCInfo.info.flushedLogicalAddress, recoverFromAddress, recoveredHLCInfo.info.finalLogicalAddress, recoveredHLCInfo.info.snapshotStartFlushedLogicalAddress, - recoveredHLCInfo.info.snapshotFinalLogicalAddress, recoveredHLCInfo.info.nextVersion, recoveredHLCInfo.info.guid, options, recoveredHLCInfo.deltaLog, recoverTo, cancellationToken).ConfigureAwait(false); + // First recover from index starting point (fromAddress) to snapshot starting point (flushedLogicalAddress taken at PERSISTENCE_CALLBACK, so it includes + // any flushes to the hybrid log files due to OnPagesMarkedReadOnly while we were flushing to the snapshot files). + lastFreedPage = await RecoverHybridLogAsync(scanFromAddress, recoverFromAddress, untilAddress: recoveredHLCInfo.info.flushedLogicalAddress, + recoveredHLCInfo.info.nextVersion, CheckpointType.Snapshot, + new RecoveryOptions(headAddress, fuzzyRegionStartAddress: recoveredHLCInfo.info.startLogicalAddress, undoNextVersion), cancellationToken).ConfigureAwait(false); + + // Then recover snapshot into mutable region. Note that the ObjectAllocator will not write object log records for the mutable region; + // that only happens during flushes due to OnPagesMarkedReadOnly. + var snapshotLastFreedPage = await RecoverHybridLogFromSnapshotFileAsync(scanFromAddress: recoveredHLCInfo.info.flushedLogicalAddress, + recoverFromAddress, untilAddress: recoveredHLCInfo.info.finalLogicalAddress, + snapshotStartAddress: recoveredHLCInfo.info.snapshotStartFlushedLogicalAddress, snapshotEndAddress: recoveredHLCInfo.info.snapshotFinalLogicalAddress, + recoveredHLCInfo.info.nextVersion, recoveredHLCInfo.info.guid, options, cancellationToken).ConfigureAwait(false); if (snapshotLastFreedPage != NoPageFreed) lastFreedPage = snapshotLastFreedPage; @@ -542,12 +547,12 @@ private async ValueTask InternalRecoverAsync(IndexCheckpointInfo recovered private void DoPostRecovery(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, long tailAddress, ref long headAddress, ref long readOnlyAddress, long lastFreedPage) { // Adjust head and read-only address post-recovery - var _head = (1 + (tailAddress >> hlogBase.LogPageSizeBits) - (hlogBase.GetCapacityNumPages() - hlogBase.MinEmptyPageCount)) << hlogBase.LogPageSizeBits; + var _head = hlogBase.GetFirstValidLogicalAddressOnPage(1 + hlogBase.GetPage(tailAddress) - hlogBase.MaxAllocatedPageCount); - // If additional pages have been freed to accommodate heap memory constraints, adjust head address accordingly + // If additional pages have been freed to accommodate memory constraints, adjust head address accordingly if (lastFreedPage != NoPageFreed) { - var nextAddress = (lastFreedPage + 1) << hlogBase.LogPageSizeBits; + var nextAddress = hlogBase.GetFirstValidLogicalAddressOnPage(lastFreedPage + 1); if (_head < nextAddress) _head = nextAddress; } @@ -558,6 +563,7 @@ private void DoPostRecovery(IndexCheckpointInfo recoveredICInfo, HybridLogCheckp readOnlyAddress = headAddress; hlogBase.RecoveryReset(tailAddress, headAddress, recoveredHLCInfo.info.beginAddress, readOnlyAddress); + hlogBase.SetObjectLogTail(recoveredHLCInfo.info.hlogEndObjectLogTail); checkpointManager.OnRecovery(recoveredICInfo.info.token, recoveredHLCInfo.info.guid); recoveredHLCInfo.Dispose(); } @@ -573,45 +579,42 @@ public void SetVersion(long version) } /// - /// Compute recovery address and determine where to recover to + /// Compute recovery address and determine where to recover from /// /// IndexCheckpointInfo /// HybridLogCheckpointInfo - /// Address from which to perform recovery (undo v+1 records) + /// Address from which to perform recovery (undo v+1 records and append to tag-chain tail) /// Whether we are recovering to the initial page - private bool RecoverToInitialPage(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, out long recoverFromAddress) + private bool GetInitialRecoveryAddress(IndexCheckpointInfo recoveredICInfo, HybridLogCheckpointInfo recoveredHLCInfo, out long recoverFromAddress) { // Set new system state after recovery stateMachineDriver.SetSystemState(SystemState.Make(Phase.REST, recoveredHLCInfo.info.version + 1)); - if (!recoveredICInfo.IsDefault() && recoveryCountdown != null) + if (!recoveredICInfo.IsDefault && recoveryCountdown != null) { Debug.WriteLine("Ignoring index checkpoint as we have already recovered index previously"); recoveredICInfo = default; } - if (recoveredICInfo.IsDefault()) - { - // No index checkpoint - recover from begin of log - recoverFromAddress = recoveredHLCInfo.info.beginAddress; + // Initialize to recover from beginning of log + recoverFromAddress = recoveredHLCInfo.info.beginAddress; - // Unless we recovered previously until some hlog address + if (recoveredICInfo.IsDefault) + { + // No index checkpoint - recover from beginning of log unless we recovered previously until some hlog address if (hlogBase.FlushedUntilAddress > recoverFromAddress) recoverFromAddress = hlogBase.FlushedUntilAddress; - // Start recovery at least from beginning of fuzzy log region - // Needed if we are recovering to the same checkpoint a second time, with undo - // set to true during the second time. + // Start recovery at least from beginning of fuzzy log region. Needed if we are recovering to the same checkpoint + // a second time, with undo set to true during the second time. if (recoveredHLCInfo.info.startLogicalAddress < recoverFromAddress) recoverFromAddress = recoveredHLCInfo.info.startLogicalAddress; } else { - recoverFromAddress = recoveredHLCInfo.info.beginAddress; - if (recoveredICInfo.info.startLogicalAddress > recoverFromAddress) { - // Index checkpoint given - recover to that + // Index checkpoint was given - recover to that recoverFromAddress = recoveredICInfo.info.startLogicalAddress; return false; } @@ -620,7 +623,7 @@ private bool RecoverToInitialPage(IndexCheckpointInfo recoveredICInfo, HybridLog return true; } - private bool SetRecoveryPageRanges(HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, long fromAddress, out long tailAddress, out long headAddress, out long scanFromAddress) + private bool SetRecoveryPageRanges(HybridLogCheckpointInfo recoveredHLCInfo, int numPagesToPreload, long recoverFromAddress, out long tailAddress, out long headAddress, out long scanFromAddress) { if ((recoveredHLCInfo.info.useSnapshotFile == 0) && (recoveredHLCInfo.info.finalLogicalAddress <= hlogBase.GetTailAddress())) { @@ -628,24 +631,18 @@ private bool SetRecoveryPageRanges(HybridLogCheckpointInfo recoveredHLCInfo, int return false; } - // Recover segment offsets for object log - if (recoveredHLCInfo.info.objectLogSegmentOffsets != null) - Array.Copy(recoveredHLCInfo.info.objectLogSegmentOffsets, - hlog.GetSegmentOffsets(), - recoveredHLCInfo.info.objectLogSegmentOffsets.Length); - tailAddress = recoveredHLCInfo.info.finalLogicalAddress; headAddress = recoveredHLCInfo.info.headAddress; if (numPagesToPreload != -1) { - var head = (hlogBase.GetPage(tailAddress) - numPagesToPreload) << hlogBase.LogPageSizeBits; + var head = hlogBase.GetFirstValidLogicalAddressOnPage(hlogBase.GetPage(tailAddress) - numPagesToPreload); if (head > headAddress) headAddress = head; } scanFromAddress = headAddress; - if (fromAddress < scanFromAddress) - scanFromAddress = fromAddress; + if (recoverFromAddress < scanFromAddress) + scanFromAddress = recoverFromAddress; // Adjust head address if we need to anyway preload if (scanFromAddress < headAddress) @@ -660,68 +657,105 @@ private bool SetRecoveryPageRanges(HybridLogCheckpointInfo recoveredHLCInfo, int return true; } + private long ReadPagesWithMemoryConstraint(long endAddress, RecoveryStatus recoveryStatus, long page, long endPage, int numPagesToRead) + { + // Before reading in additional pages, trim memory if needed to make room for the inline space (we can't know the heap size yet) + var freedPage = TrimLogMemorySize(recoveryStatus, tailPage: page, numPagesToRead); + + // Set all page read statuses to Pending + for (var p = page; p < endPage; p++) + recoveryStatus.readStatus[hlogBase.GetPageIndexForPage(p)] = ReadStatus.Pending; + + // Issue request to read pages as much as possible + hlogBase.AsyncReadPagesForRecovery(page, numPagesToRead, endAddress, recoveryStatus, recoveryStatus.recoveryDevicePageOffset, + recoveryStatus.recoveryDevice, recoveryStatus.objectLogRecoveryDevice); + return freedPage; + } + /// - /// This method ensures that before 'pagesToRead' number of pages are read into memory, any previously allocated pages - /// that would cause total number of pages in memory to go beyond usableCapacity are freed. This is to ensure that - /// memory size constraint is maintained during recovery. - /// Illustration with capacity 32, usableCapacity 20, pagesToRead 2: - /// beg: startPage - 32 - /// end: startPage - 18 - /// We free these 14 pages, leaving 18 allocated, and then read 2, which fills up usableCapacity. - /// The beg, end can only be zero on the first pass through the buffer, as the page number continuously increases + /// Called before 'pagesToRead' number of pages are read into memory, this method determines how many previously allocated pages + /// must be (partially or completely) freed to avoid the total memory size to go beyond the specified maximum during recovery. /// - private void FreePagesBeyondUsableCapacity(long startPage, int capacity, int usableCapacity, int pagesToRead, RecoveryStatus recoveryStatus) + /// True if is nonzero, else false + private bool GetEvictionPageRange(long tailPage, int numPagesToRead, CancellationToken cancellationToken, out long startPage, out int minEvictPageCount, out int maxEvictPageCount) { - var beg = Math.Max(0, startPage - capacity); - var end = Math.Max(0, startPage - (usableCapacity - pagesToRead)); + // The caller will iterate from startPage to endPage, so we use that as the basis for our eviction counts (which will start evicting at startPage). + // tailPage is the leading page index and start/endPage are the trailing page indexes: startPage is at the start of a full buffer of pages, + // and endPage is the start of the "usable" buffer capacity (the amount of pages we can actually use within the hlogBase.MaxAllocatedPageCount + // constraint) PLUS the number of pages to read. If hlogBase.MaxAllocatedPageCount is less than hlogBase.BufferSize, the the calling + // TrimLogMemorySize will probably be iterating over freed (non-allocated) pages from startPage to (endPage - numPagesToRead), and then + // will start actually evicting pages. NOTE: Currently numPagesToRead is always 1, but we may be able to optimize that in the future. + startPage = Math.Max(0, tailPage - hlogBase.BufferSize); + var endPage = Math.Max(0, tailPage - hlogBase.MaxAllocatedPageCount + numPagesToRead); + + // TODO: Currently Recovery is still page-level eviction only. hlogBase.HeadAddress etc. are not yet set so we will have to propagate + // the new headAddress back up the path we currently pass the lastFreedPage. + + // MinEvictPageCount is the number of pages we must clear so we can read numPagesToRead without violating the maximum page count constraint. + minEvictPageCount = Math.Max(0, (int)(endPage - startPage)); + maxEvictPageCount = minEvictPageCount; + if (endPage <= startPage) + return false; + + // If no log size tracker, just ensure MaxPageCount is not exceeded. + if (hlogBase.logSizeTracker is null) + return minEvictPageCount > 0; - for (var page = beg; page < end; page++) + // We have a log size tracker, so set minEvictPageCount to zero and maxEvictPageCount to the maximum number of pages we can evict; + // the caller will also test logSizeTracker.IsBeyondSizeLimitToReadPages during the eviction loop and jump out if it drops within budget. + maxEvictPageCount = Math.Max(minEvictPageCount, (int)(tailPage - startPage) - LogSizeTracker.MinResizeTargetPageCount); + return minEvictPageCount > 0 || hlogBase.logSizeTracker.IsBeyondSizeLimitToReadPages(numPagesToRead); + } + + private long TrimLogMemorySize(RecoveryStatus recoveryStatus, long tailPage, int numPagesToRead) + { + var lastFreedPage = NoPageFreed; + if (GetEvictionPageRange(tailPage, numPagesToRead, cancellationToken: default, out long startPage, out int minEvictPageCount, out int maxEvictPageCount)) { - var pageIndex = hlogBase.GetPageIndexForPage(page); - if (hlog.IsAllocated(pageIndex)) + // Evict pages one at a time + for (var ii = 0; ii < maxEvictPageCount; ii++) { - recoveryStatus.WaitFlush(pageIndex); - hlogBase.EvictPage(page); + if (hlogBase.logSizeTracker is not null && ii >= minEvictPageCount && !hlogBase.logSizeTracker.IsBeyondSizeLimitToReadPages(numPagesToRead)) + break; + var page = startPage + ii; + var pageIndex = hlogBase.GetPageIndexForPage(page); + if (hlogBase.IsAllocated(pageIndex)) + { + recoveryStatus.WaitFlush(pageIndex); + hlogBase.EvictPageForRecovery(page); + lastFreedPage = page; + } } } - } - - private void ReadPagesWithMemoryConstraint(long endAddress, int capacity, RecoveryStatus recoveryStatus, long page, long endPage, int numPagesToRead) - { - // Before reading in additional pages, make sure that any previously allocated pages that would violate the memory size - // constraint are freed. - FreePagesBeyondUsableCapacity(startPage: page, capacity: capacity, usableCapacity: capacity - hlogBase.MinEmptyPageCount, pagesToRead: numPagesToRead, recoveryStatus); - // Issue request to read pages as much as possible - for (var p = page; p < endPage; p++) recoveryStatus.readStatus[hlogBase.GetPageIndexForPage(p)] = ReadStatus.Pending; - hlogBase.AsyncReadPagesFromDevice(page, numPagesToRead, endAddress, - hlogBase.AsyncReadPagesCallbackForRecovery, - recoveryStatus, recoveryStatus.recoveryDevicePageOffset, - recoveryStatus.recoveryDevice, recoveryStatus.objectLogRecoveryDevice); + return lastFreedPage; } - private long FreePagesToLimitHeapMemory(RecoveryStatus recoveryStatus, long page) + private async Task TrimLogMemorySizeAsync(RecoveryStatus recoveryStatus, long tailPage, int numPagesToRead, CancellationToken cancellationToken = default) { - long lastFreedPage = NoPageFreed; - if (hlogBase.IsSizeBeyondLimit == null) - return lastFreedPage; - - // free up additional pages, one at a time, to bring memory usage under control starting with the earliest possible page - for (var p = Math.Max(0, page - recoveryStatus.usableCapacity + 1); p < page && hlogBase.IsSizeBeyondLimit(); p++) + var lastFreedPage = NoPageFreed; + if (GetEvictionPageRange(tailPage, numPagesToRead, cancellationToken: default, out long startPage, out int minEvictPageCount, out int maxEvictPageCount)) { - var pageIndex = hlogBase.GetPageIndexForPage(p); - if (hlog.IsAllocated(pageIndex)) + // Evict pages one at a time + for (var ii = 0; ii < maxEvictPageCount; ii++) { - recoveryStatus.WaitFlush(pageIndex); - hlogBase.EvictPage(p); - lastFreedPage = p; + if (hlogBase.logSizeTracker is not null && ii >= minEvictPageCount && !hlogBase.logSizeTracker.IsBeyondSizeLimitToReadPages(numPagesToRead)) + break; + var page = startPage + ii; + var pageIndex = hlogBase.GetPageIndexForPage(page); + if (hlogBase.IsAllocated(pageIndex)) + { + await recoveryStatus.WaitFlushAsync(pageIndex, cancellationToken).ConfigureAwait(false); + hlogBase.EvictPageForRecovery(page); + lastFreedPage = page; + } } } return lastFreedPage; } - private long ReadPagesForRecovery(long untilAddress, RecoveryStatus recoveryStatus, long endPage, int capacity, int numPagesToReadPerIteration, long page) + private (long end, long freedPage) ReadPagesForRecovery(long untilAddress, RecoveryStatus recoveryStatus, long endPage, int numPagesToReadPerIteration, long page) { var readEndPage = Math.Min(page + numPagesToReadPerIteration, endPage); if (page < readEndPage) @@ -730,15 +764,15 @@ private long ReadPagesForRecovery(long untilAddress, RecoveryStatus recoveryStat // Ensure that page slots that will be read into, have been flushed from previous reads. Due to the use of a single read semaphore, // this must be done in batches of "all flushes' followed by "all reads" to ensure proper sequencing of reads when - // usableCapacity != capacity (and thus the page-read index is not equal to the page-flush index). + // we are not using the full BufferSize (and thus the page-read index is not equal to the page-flush index). WaitUntilAllPagesHaveBeenFlushed(page, readEndPage, recoveryStatus); - ReadPagesWithMemoryConstraint(untilAddress, capacity, recoveryStatus, page, readEndPage, numPagesToRead); + return (readEndPage, ReadPagesWithMemoryConstraint(untilAddress, recoveryStatus, page, readEndPage, numPagesToRead)); } - return readEndPage; + return (readEndPage, NoPageFreed); } - private async ValueTask ReadPagesForRecoveryAsync(long untilAddress, RecoveryStatus recoveryStatus, long endPage, int capacity, int numPagesToReadPerIteration, long page, CancellationToken cancellationToken) + private async ValueTask<(long end, long freedPage)> ReadPagesForRecoveryAsync(long untilAddress, RecoveryStatus recoveryStatus, long endPage, int numPagesToReadPerIteration, long page, CancellationToken cancellationToken) { var readEndPage = Math.Min(page + numPagesToReadPerIteration, endPage); if (page < readEndPage) @@ -749,58 +783,58 @@ private async ValueTask ReadPagesForRecoveryAsync(long untilAddress, Recov // this must be done in batches of "all flushes' followed by "all reads" to ensure proper sequencing of reads when // usableCapacity != capacity (and thus the page-read index is not equal to the page-flush index). await WaitUntilAllPagesHaveBeenFlushedAsync(page, readEndPage, recoveryStatus, cancellationToken).ConfigureAwait(false); - ReadPagesWithMemoryConstraint(untilAddress, capacity, recoveryStatus, page, readEndPage, numPagesToRead); - } - - return readEndPage; - } - - private async Task FreePagesToLimitHeapMemoryAsync(RecoveryStatus recoveryStatus, long page, CancellationToken cancellationToken) - { - long lastFreedPage = NoPageFreed; - if (hlogBase.IsSizeBeyondLimit == null) - return lastFreedPage; - - // free up additional pages, one at a time, to bring memory usage under control starting with the earliest possible page - for (var p = Math.Max(0, page - recoveryStatus.usableCapacity + 1); p < page && hlogBase.IsSizeBeyondLimit(); p++) - { - var pageIndex = hlogBase.GetPageIndexForPage(p); - if (hlog.IsAllocated(pageIndex)) - { - await recoveryStatus.WaitFlushAsync(pageIndex, cancellationToken); - hlogBase.EvictPage(p); - lastFreedPage = p; - } + return (readEndPage, ReadPagesWithMemoryConstraint(untilAddress, recoveryStatus, page, readEndPage, numPagesToRead)); } - return lastFreedPage; + return (readEndPage, NoPageFreed); } + /// + /// Synchronously recover the hybrid log from hybrid log files (not snapshot files). This also deserializes any objects or overflow and creates + /// entries for them in the . + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The next version of the database at the time of checkpoint flush + /// The type of checkpoint + /// The recovery options + /// The last freed page, if it was necessary to free any to limit heap memory private long RecoverHybridLog(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, CheckpointType checkpointType, RecoveryOptions options) { long lastFreedPage = NoPageFreed; if (untilAddress <= scanFromAddress) return lastFreedPage; - var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadPerIteration); - for (long page = startPage; page < endPage; page += numPagesToReadPerIteration) + var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int numPagesToReadPerIteration); + + Debug.Assert(hlogBase.logSizeTracker is null || numPagesToReadPerIteration == 1, "numPagesToReadPerIteration must be 1 when tracking sizes"); + for (var page = startPage; page < endPage; page += numPagesToReadPerIteration) { - var end = ReadPagesForRecovery(untilAddress, recoveryStatus, endPage, capacity, numPagesToReadPerIteration, page); + var (end, freedPage) = ReadPagesForRecovery(untilAddress, recoveryStatus, endPage, numPagesToReadPerIteration, page); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + var trimPageReadCount = numPagesToReadPerIteration; for (var p = page; p < end; p++) { // Ensure page has been read into memory int pageIndex = hlogBase.GetPageIndexForPage(p); recoveryStatus.WaitRead(pageIndex); - var freedPage = FreePagesToLimitHeapMemory(recoveryStatus, p); - if (freedPage != NoPageFreed) - lastFreedPage = freedPage; + if (hlogBase.logSizeTracker is not null) + { + // Trim the log memory again in case we read large objects on the current page. Add 1 to tailPage so that + // when the BufferSize subtraction wraps around the buffer it won't try to evict the page we just added. + // Decrease trimPageReadCount as we process each page so we don't over-prune. + freedPage = TrimLogMemorySize(recoveryStatus, tailPage: p + 1, trimPageReadCount--); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + } // We make an extra pass to clear locks when reading every page back into memory - ClearLocksOnPage(p, options); - - ProcessReadPageAndFlush(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex); + ClearBitsOnPage(p, untilAddress, options); + ProcessReadPageAndFlush(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex); } } @@ -808,32 +842,54 @@ private long RecoverHybridLog(long scanFromAddress, long recoverFromAddress, lon return lastFreedPage; } - private async ValueTask RecoverHybridLogAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, CheckpointType checkpointType, RecoveryOptions options, CancellationToken cancellationToken) + /// + /// Synchronously recover the hybrid log from hybrid log files (not snapshot files). This also deserializes any objects or overflow and creates + /// entries for them in the . + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The next version of the database at the time of checkpoint flush + /// The type of checkpoint + /// The recovery options + /// The cancellation token + /// The last freed page, if it was necessary to free any to limit heap memory + private async ValueTask RecoverHybridLogAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, + CheckpointType checkpointType, RecoveryOptions options, CancellationToken cancellationToken) { long lastFreedPage = NoPageFreed; if (untilAddress <= scanFromAddress) return lastFreedPage; - var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadPerIteration); + var recoveryStatus = GetPageRangesToRead(scanFromAddress, untilAddress, checkpointType, out long startPage, out long endPage, out int numPagesToReadPerIteration); + Debug.Assert(hlogBase.logSizeTracker is null || numPagesToReadPerIteration == 1, "numPagesToReadPerIteration must be 1 when tracking sizes"); for (long page = startPage; page < endPage; page += numPagesToReadPerIteration) { - var end = await ReadPagesForRecoveryAsync(untilAddress, recoveryStatus, endPage, capacity, numPagesToReadPerIteration, page, cancellationToken).ConfigureAwait(false); + var (end, freedPage) = await ReadPagesForRecoveryAsync(untilAddress, recoveryStatus, endPage, numPagesToReadPerIteration, page, cancellationToken).ConfigureAwait(false); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + var trimPageReadCount = numPagesToReadPerIteration; for (var p = page; p < end; p++) { // Ensure page has been read into memory - int pageIndex = hlogBase.GetPageIndexForPage(p); + var pageIndex = hlogBase.GetPageIndexForPage(p); await recoveryStatus.WaitReadAsync(pageIndex, cancellationToken).ConfigureAwait(false); - var freedPage = await FreePagesToLimitHeapMemoryAsync(recoveryStatus, p, cancellationToken).ConfigureAwait(false); - if (freedPage != NoPageFreed) - lastFreedPage = freedPage; + if (hlogBase.logSizeTracker is not null) + { + // Trim the log memory again in case we read large objects on the current page. Add 1 to tailPage so that + // when the BufferSize subtraction wraps around the buffer it won't try to evict the page we just added. + // Decrease trimPageReadCount as we process each page so we don't over-prune. + freedPage = await TrimLogMemorySizeAsync(recoveryStatus, tailPage: p + 1, trimPageReadCount--, cancellationToken).ConfigureAwait(false); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + } // We make an extra pass to clear locks when reading every page back into memory - ClearLocksOnPage(p, options); - - ProcessReadPageAndFlush(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex); + ClearBitsOnPage(p, untilAddress, options); + ProcessReadPageAndFlush(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex); } } @@ -841,30 +897,48 @@ private async ValueTask RecoverHybridLogAsync(long scanFromAddress, long r return lastFreedPage; } - private RecoveryStatus GetPageRangesToRead(long scanFromAddress, long untilAddress, CheckpointType checkpointType, out long startPage, out long endPage, out int capacity, out int numPagesToReadPerIteration) + /// + /// Get the range of pages to read from the hybrid log file(s) for recovery + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The + /// The first page to read + /// The last page to read + /// The number of pages to read per iteration + /// The allocated instance. + private RecoveryStatus GetPageRangesToRead(long scanFromAddress, long untilAddress, CheckpointType checkpointType, + out long startPage, out long endPage, out int numPagesToReadPerIteration) { startPage = hlogBase.GetPage(scanFromAddress); endPage = hlogBase.GetPage(untilAddress); - if (untilAddress > hlog.GetStartLogicalAddress(endPage) && untilAddress > scanFromAddress) - { + if (untilAddress > hlogBase.GetFirstValidLogicalAddressOnPage(endPage) && untilAddress > scanFromAddress) endPage++; - } - - capacity = hlogBase.GetCapacityNumPages(); - int totalPagesToRead = (int)(endPage - startPage); - // Leave out at least MinEmptyPageCount pages to maintain memory size during recovery // If heap memory is to be tracked, then read one page at a time to control memory usage - numPagesToReadPerIteration = hlogBase.IsSizeBeyondLimit == null ? Math.Min(capacity - hlogBase.MinEmptyPageCount, totalPagesToRead) : 1; - return new RecoveryStatus(capacity, hlogBase.MinEmptyPageCount, endPage, untilAddress, checkpointType); + var totalPagesToRead = (int)(endPage - startPage); + numPagesToReadPerIteration = hlogBase.logSizeTracker is null ? Math.Min(hlogBase.BufferSize, totalPagesToRead) : 1; + return new RecoveryStatus(hlogBase.BufferSize); } - private void ProcessReadPageAndFlush(long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, RecoveryStatus recoveryStatus, long page, int pageIndex) + /// + /// Process a page that has been read from the hybrid log file (not snapshot), and flush it if necessary + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The next version of the database at the time of checkpoint flush + /// The recovery options + /// The instance + /// The page number to process + /// The index of in the allocator's circular page buffer + private void ProcessReadPageAndFlush(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, + RecoveryStatus recoveryStatus, long page, int pageIndex) { if (ProcessReadPage(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, page, pageIndex)) { // Page was modified due to undoFutureVersion. Flush it to disk; the callback issues the after-capacity read request if necessary. - hlogBase.AsyncFlushPages(page, 1, AsyncFlushPageCallbackForRecovery, recoveryStatus); + hlogBase.AsyncFlushPagesForRecovery(scanFromAddress, page, 1, AsyncFlushPageCallbackForRecovery, recoveryStatus); return; } @@ -872,25 +946,39 @@ private void ProcessReadPageAndFlush(long recoverFromAddress, long untilAddress, recoveryStatus.flushStatus[pageIndex] = FlushStatus.Done; } - private bool ProcessReadPage(long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, RecoveryStatus recoveryStatus, long page, int pageIndex) + /// + /// Determine address ranges on a page that has been read from the hybrid log file (not snapshot), then recover from that page. + /// + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The next version of the database at the time of checkpoint flush + /// The recovery options + /// The instance + /// The page number to process + /// The index of in the allocator's circular page buffer + /// + private bool ProcessReadPage(long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, RecoveryStatus recoveryStatus, + long page, int pageIndex) { - var startLogicalAddress = hlog.GetStartLogicalAddress(page); - var endLogicalAddress = hlog.GetStartLogicalAddress(page + 1); - var physicalAddress = hlog.GetPhysicalAddress(startLogicalAddress); + var startLogicalAddressOfPage = hlogBase.GetLogicalAddressOfStartOfPage(page); // Do not offset for page header; that's done below and in RecoverFromPage + var endLogicalAddressOfPage = hlogBase.GetLogicalAddressOfStartOfPage(page + 1); + var startPhysicalAddressOfPage = hlogBase.GetPhysicalAddress(startLogicalAddressOfPage); - if (recoverFromAddress >= endLogicalAddress) + if (recoverFromAddress >= endLogicalAddressOfPage) return false; - var pageFromAddress = 0L; - var pageUntilAddress = hlogBase.GetPageSize(); - - if (recoverFromAddress > startLogicalAddress) - pageFromAddress = hlogBase.GetOffsetInPage(recoverFromAddress); + var pageFromAddressOffset = (long)hlogBase.pageHeaderSize; + var pageUntilAddressOffset = hlogBase.GetPageSize(); - if (untilAddress < endLogicalAddress) - pageUntilAddress = hlogBase.GetOffsetInPage(untilAddress); + if (recoverFromAddress > startLogicalAddressOfPage) + { + pageFromAddressOffset = hlogBase.GetOffsetOnPage(recoverFromAddress); + Debug.Assert(pageFromAddressOffset >= hlogBase.pageHeaderSize, $"pageFromAddressOffset {pageFromAddressOffset} must be >= hlogBase.pageHeaderSize {hlogBase.pageHeaderSize} (which may be 0)"); + } + if (untilAddress < endLogicalAddressOfPage) + pageUntilAddressOffset = hlogBase.GetOffsetOnPage(untilAddress); - if (RecoverFromPage(recoverFromAddress, pageFromAddress, pageUntilAddress, startLogicalAddress, physicalAddress, nextVersion, options)) + if (RecoverFromPage(recoverFromAddress, pageFromAddressOffset, pageUntilAddressOffset, startLogicalAddressOfPage, startPhysicalAddressOfPage, options)) { // The current page was modified due to undoFutureVersion; caller will flush it to storage and issue a read request if necessary. recoveryStatus.readStatus[pageIndex] = ReadStatus.Pending; @@ -913,15 +1001,36 @@ private async ValueTask WaitUntilAllPagesHaveBeenFlushedAsync(long startPage, lo await recoveryStatus.WaitFlushAsync(hlogBase.GetPageIndexForPage(page), cancellationToken).ConfigureAwait(false); } - private long RecoverHybridLogFromSnapshotFile(long scanFromAddress, long recoverFromAddress, long untilAddress, long snapshotStartAddress, long snapshotEndAddress, long nextVersion, Guid guid, RecoveryOptions options, DeltaLog deltaLog, long recoverTo) + /// + /// Synchronously recover the hybrid log from snapshot files + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The start of the mutable region; the FlushedUntilAddress at the start of the WAIT_FLUSH phase + /// The end of the snapshot; the tailAddress at the start of the WAIT_FLUSH phase + /// The next version of the database at the time of checkpoint flush + /// The checkpoint token guid + /// The recovery options + /// The last freed page, if it was necessary to free any to limit heap memory + private long RecoverHybridLogFromSnapshotFile(long scanFromAddress, long recoverFromAddress, long untilAddress, + long snapshotStartAddress, long snapshotEndAddress, long nextVersion, Guid guid, RecoveryOptions options) { long lastFreedPage = NoPageFreed; - GetSnapshotPageRangesToRead(scanFromAddress, untilAddress, snapshotStartAddress, snapshotEndAddress, guid, out long startPage, out long endPage, out long snapshotEndPage, out int capacity, out var recoveryStatus, out int numPagesToReadPerIteration); + GetSnapshotPageRangesToRead(scanFromAddress, untilAddress, snapshotStartAddress, snapshotEndAddress, guid, out long startPage, + out long endPage, out long snapshotEndPage, out var recoveryStatus, out int numPagesToReadPerIteration); + + // Notify application of checkpoint token before processing snapshot records + if (storeFunctions.CallOnDiskRead) + storeFunctions.OnRecovery(guid); for (long page = startPage; page < endPage; page += numPagesToReadPerIteration) { - ReadPagesForRecovery(snapshotEndAddress, recoveryStatus, snapshotEndPage, capacity, numPagesToReadPerIteration, page); + var (_, freedPage) = ReadPagesForRecovery(snapshotEndAddress, recoveryStatus, snapshotEndPage, numPagesToReadPerIteration, page); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; var end = Math.Min(page + numPagesToReadPerIteration, endPage); + for (long p = page; p < end; p++) { int pageIndex = hlogBase.GetPageIndexForPage(p); @@ -929,24 +1038,30 @@ private long RecoverHybridLogFromSnapshotFile(long scanFromAddress, long recover { // Ensure the page is read from file recoveryStatus.WaitRead(pageIndex); - var freedPage = FreePagesToLimitHeapMemory(recoveryStatus, p); - if (freedPage != NoPageFreed) - lastFreedPage = freedPage; + + if (hlogBase.logSizeTracker is not null) + { + // Trim the log memory again in case we read large objects on the current page. Use 0 for numPagesToRead so we don't over-prune. + freedPage = TrimLogMemorySize(recoveryStatus, tailPage: p + 1, 0); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + } // We make an extra pass to clear locks when reading pages back into memory - ClearLocksOnPage(p, options); + ClearBitsOnPage(p, untilAddress, options, snapshotFromAddress: scanFromAddress); } else { recoveryStatus.WaitFlush(pageIndex); - if (!hlog.IsAllocated(pageIndex)) + if (!hlogBase.IsAllocated(pageIndex)) hlog.AllocatePage(pageIndex); else - hlog.ClearPage(pageIndex); + hlogBase.ClearPage(pageIndex); } } - ApplyDelta(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, deltaLog, recoverTo, endPage, snapshotEndPage, capacity, numPagesToReadPerIteration, recoveryStatus, page, end); + RecoverSnapshotPages(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, + endPage, snapshotEndPage, numPagesToReadPerIteration, recoveryStatus, page, end); } WaitUntilAllPagesHaveBeenFlushed(startPage, endPage, recoveryStatus); @@ -954,16 +1069,37 @@ private long RecoverHybridLogFromSnapshotFile(long scanFromAddress, long recover return lastFreedPage; } - private async ValueTask RecoverHybridLogFromSnapshotFileAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, long snapshotStartAddress, long snapshotEndAddress, long nextVersion, Guid guid, RecoveryOptions options, DeltaLog deltaLog, long recoverTo, CancellationToken cancellationToken) + /// + /// Asynchronously recover the hybrid log from snapshot files + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The start of the mutable region; the FlushedUntilAddress at the start of the WAIT_FLUSH phase + /// The end of the snapshot; the tailAddress at the start of the WAIT_FLUSH phase + /// The next version of the database at the time of checkpoint flush + /// The checkpoint token guid + /// The recovery options + /// The last freed page, if it was necessary to free any to limit heap memory + private async ValueTask RecoverHybridLogFromSnapshotFileAsync(long scanFromAddress, long recoverFromAddress, long untilAddress, + long snapshotStartAddress, long snapshotEndAddress, long nextVersion, Guid guid, RecoveryOptions options, + CancellationToken cancellationToken) { long lastFreedPage = NoPageFreed; - GetSnapshotPageRangesToRead(scanFromAddress, untilAddress, snapshotStartAddress, snapshotEndAddress, guid, out long startPage, out long endPage, out long snapshotEndPage, out int capacity, out var recoveryStatus, out int numPagesToReadPerIteration); + GetSnapshotPageRangesToRead(scanFromAddress, untilAddress, snapshotStartAddress, snapshotEndAddress, guid, out long startPage, + out long endPage, out long snapshotEndPage, out var recoveryStatus, out int numPagesToReadPerIteration); + + // Notify application of checkpoint token before processing snapshot records + if (storeFunctions.CallOnDiskRead) + storeFunctions.OnRecovery(guid); for (long page = startPage; page < endPage; page += numPagesToReadPerIteration) { - await ReadPagesForRecoveryAsync(snapshotEndAddress, recoveryStatus, snapshotEndPage, capacity, numPagesToReadPerIteration, page, cancellationToken).ConfigureAwait(false); - + var (_, freedPage) = await ReadPagesForRecoveryAsync(snapshotEndAddress, recoveryStatus, snapshotEndPage, numPagesToReadPerIteration, page, cancellationToken).ConfigureAwait(false); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; var end = Math.Min(page + numPagesToReadPerIteration, endPage); + for (long p = page; p < end; p++) { int pageIndex = hlogBase.GetPageIndexForPage(p); @@ -971,24 +1107,30 @@ private async ValueTask RecoverHybridLogFromSnapshotFileAsync(long scanFro { // Ensure the page is read from file await recoveryStatus.WaitReadAsync(pageIndex, cancellationToken).ConfigureAwait(false); - var freedPage = await FreePagesToLimitHeapMemoryAsync(recoveryStatus, p, cancellationToken).ConfigureAwait(false); - if (freedPage != NoPageFreed) - lastFreedPage = freedPage; + + if (hlogBase.logSizeTracker is not null) + { + // Trim the log memory again in case we read large objects on the current page. Use 0 for numPagesToRead so we don't over-prune. + freedPage = await TrimLogMemorySizeAsync(recoveryStatus, tailPage: p + 1, numPagesToRead: 0, cancellationToken).ConfigureAwait(false); + if (freedPage != NoPageFreed) + lastFreedPage = freedPage; + } // We make an extra pass to clear locks when reading pages back into memory - ClearLocksOnPage(p, options); + ClearBitsOnPage(p, untilAddress, options, snapshotFromAddress: scanFromAddress); } else { await recoveryStatus.WaitFlushAsync(pageIndex, cancellationToken).ConfigureAwait(false); - if (!hlog.IsAllocated(pageIndex)) + if (!hlogBase.IsAllocated(pageIndex)) hlog.AllocatePage(pageIndex); else - hlog.ClearPage(pageIndex); + hlogBase.ClearPage(pageIndex); } } - ApplyDelta(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, deltaLog, recoverTo, endPage, snapshotEndPage, capacity, numPagesToReadPerIteration, recoveryStatus, page, end); + RecoverSnapshotPages(scanFromAddress, recoverFromAddress, untilAddress, nextVersion, options, + endPage, snapshotEndPage, numPagesToReadPerIteration, recoveryStatus, page, end); } await WaitUntilAllPagesHaveBeenFlushedAsync(startPage, endPage, recoveryStatus, cancellationToken).ConfigureAwait(false); @@ -996,15 +1138,17 @@ private async ValueTask RecoverHybridLogFromSnapshotFileAsync(long scanFro return lastFreedPage; } - private void ApplyDelta(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, DeltaLog deltaLog, long recoverTo, long endPage, long snapshotEndPage, int capacity, int numPagesToRead, RecoveryStatus recoveryStatus, long page, long end) + /// + /// For each page in the snapshot from [page, end), process the page for recovery. + /// + private void RecoverSnapshotPages(long scanFromAddress, long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, + long endPage, long snapshotEndPage, int numPagesToRead, RecoveryStatus recoveryStatus, long page, long end) { - hlogBase.ApplyDelta(deltaLog, page, end, recoverTo); - for (long p = page; p < end; p++) { int pageIndex = hlogBase.GetPageIndexForPage(p); - var endLogicalAddress = hlog.GetStartLogicalAddress(p + 1); + var endLogicalAddress = hlogBase.GetLogicalAddressOfStartOfPage(p + 1); if (recoverFromAddress < endLogicalAddress && recoverFromAddress < untilAddress) ProcessReadSnapshotPage(recoverFromAddress, untilAddress, nextVersion, options, recoveryStatus, p, pageIndex); @@ -1013,53 +1157,64 @@ private void ApplyDelta(long scanFromAddress, long recoverFromAddress, long unti { // Flush snapshot page to main log recoveryStatus.flushStatus[pageIndex] = FlushStatus.Pending; - hlogBase.AsyncFlushPages(p, 1, AsyncFlushPageCallbackForRecovery, recoveryStatus); + hlogBase.AsyncFlushPagesForRecovery(scanFromAddress, p, 1, AsyncFlushPageCallbackForRecovery, recoveryStatus); } } } - private void GetSnapshotPageRangesToRead(long fromAddress, long untilAddress, long snapshotStartAddress, long snapshotEndAddress, Guid guid, out long startPage, out long endPage, out long snapshotEndPage, out int capacity, - out RecoveryStatus recoveryStatus, out int numPagesToReadPerIteration) + /// + /// Get the range of pages to read from the snapshot file(s) for recovery + /// + /// The address to start scanning from; the lowest address at which we will bring pages into the circular buffer (may be in the middle of a page) + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, + /// The start of the mutable region; the FlushedUntilAddress at the start of the WAIT_FLUSH phase + /// The end of the snapshot; the tailAddress at the start of the WAIT_FLUSH phase + /// The checkpoint token guid + /// The first page to read; the page of + /// The last page to read; the page of + /// The page of + /// The allocated instance + /// The number of pages to read per iteration + private void GetSnapshotPageRangesToRead(long scanFromAddress, long untilAddress, long snapshotStartAddress, long snapshotEndAddress, Guid guid, + out long startPage, out long endPage, out long snapshotEndPage, out RecoveryStatus recoveryStatus, out int numPagesToReadPerIteration) { // Compute startPage and endPage - startPage = hlogBase.GetPage(fromAddress); + startPage = hlogBase.GetPage(scanFromAddress); endPage = hlogBase.GetPage(untilAddress); - if (untilAddress > hlog.GetStartLogicalAddress(endPage) && untilAddress > fromAddress) + if (untilAddress > hlogBase.GetFirstValidLogicalAddressOnPage(endPage) && untilAddress > scanFromAddress) endPage++; - long snapshotStartPage = hlogBase.GetPage(snapshotStartAddress); + var snapshotStartPage = hlogBase.GetPage(snapshotStartAddress); snapshotEndPage = hlogBase.GetPage(snapshotEndAddress); - if (snapshotEndAddress > hlog.GetStartLogicalAddress(snapshotEndPage) && snapshotEndAddress > snapshotStartAddress) + if (snapshotEndAddress > hlogBase.GetFirstValidLogicalAddressOnPage(snapshotEndPage) && snapshotEndAddress > snapshotStartAddress) snapshotEndPage++; // By default first page has one extra record - capacity = hlogBase.GetCapacityNumPages(); var recoveryDevice = checkpointManager.GetSnapshotLogDevice(guid); var objectLogRecoveryDevice = checkpointManager.GetSnapshotObjectLogDevice(guid); - recoveryDevice.Initialize(hlogBase.GetSegmentSize()); - objectLogRecoveryDevice.Initialize(-1); - recoveryStatus = new RecoveryStatus(capacity, hlogBase.MinEmptyPageCount, endPage, untilAddress, CheckpointType.Snapshot) + recoveryDevice.Initialize(hlogBase.GetMainLogSegmentSize()); + objectLogRecoveryDevice.Initialize(hlogBase.GetObjectLogSegmentSize()); + recoveryStatus = new RecoveryStatus(hlogBase.BufferSize) { recoveryDevice = recoveryDevice, objectLogRecoveryDevice = objectLogRecoveryDevice, - recoveryDevicePageOffset = snapshotStartPage, - snapshotEndPage = snapshotEndPage + recoveryDevicePageOffset = snapshotStartPage }; // Initially issue read request for all pages that can be held in memory // If heap memory is to be tracked, then read one page at a time to control memory usage - int totalPagesToRead = (int)(snapshotEndPage - startPage); - numPagesToReadPerIteration = hlogBase.IsSizeBeyondLimit == null ? Math.Min(capacity - hlogBase.MinEmptyPageCount, totalPagesToRead) : 1; + var totalPagesToRead = (int)(snapshotEndPage - startPage); + numPagesToReadPerIteration = hlogBase.logSizeTracker is null ? Math.Min(hlogBase.BufferSize, totalPagesToRead) : 1; } - private void ProcessReadSnapshotPage(long fromAddress, long untilAddress, long nextVersion, RecoveryOptions options, RecoveryStatus recoveryStatus, long page, int pageIndex) + private void ProcessReadSnapshotPage(long recoverFromAddress, long untilAddress, long nextVersion, RecoveryOptions options, RecoveryStatus recoveryStatus, long page, int pageIndex) { // Page at hand - var startLogicalAddress = hlog.GetStartLogicalAddress(page); - var endLogicalAddress = hlog.GetStartLogicalAddress(page + 1); + var startLogicalAddressOfPage = hlogBase.GetLogicalAddressOfStartOfPage(page); // Do not offset for page header; that's done below and in RecoverFromPage + var endLogicalAddressOfPage = hlogBase.GetLogicalAddressOfStartOfPage(page + 1); // Perform recovery if page is part of the re-do portion of log - if (fromAddress < endLogicalAddress && fromAddress < untilAddress) + if (recoverFromAddress < endLogicalAddressOfPage && recoverFromAddress < untilAddress) { /* * Handling corner-cases: @@ -1069,108 +1224,127 @@ private void ProcessReadSnapshotPage(long fromAddress, long untilAddress, long n * offset. Otherwise, scan the entire page [0, PageSize) */ - var pageFromAddress = 0L; - var pageUntilAddress = hlogBase.GetPageSize(); - var physicalAddress = hlog.GetPhysicalAddress(startLogicalAddress); + long pageFromAddressOffset = hlogBase.pageHeaderSize; + var pageUntilAddressOffset = hlogBase.GetPageSize(); + var startPhysicalAddressOfPage = hlogBase.GetPhysicalAddress(startLogicalAddressOfPage); + if (recoverFromAddress > startLogicalAddressOfPage && recoverFromAddress < endLogicalAddressOfPage) + pageFromAddressOffset = hlogBase.GetOffsetOnPage(recoverFromAddress); + if (endLogicalAddressOfPage > untilAddress) + pageUntilAddressOffset = hlogBase.GetOffsetOnPage(untilAddress); - if (fromAddress > startLogicalAddress && fromAddress < endLogicalAddress) - pageFromAddress = hlogBase.GetOffsetInPage(fromAddress); - if (endLogicalAddress > untilAddress) - pageUntilAddress = hlogBase.GetOffsetInPage(untilAddress); - - _ = RecoverFromPage(fromAddress, pageFromAddress, pageUntilAddress, - startLogicalAddress, physicalAddress, nextVersion, options); + _ = RecoverFromPage(recoverFromAddress, pageFromAddressOffset, pageUntilAddressOffset, startLogicalAddressOfPage, startPhysicalAddressOfPage, options); } recoveryStatus.flushStatus[pageIndex] = FlushStatus.Done; } - private unsafe void ClearLocksOnPage(long page, RecoveryOptions options) + /// The page number to process + /// The last address to process on this page + /// Recovery options (headAddress determines if page is in-memory) + /// If > 0, records at or above this address will get OnRecoverySnapshotRead. + /// Records below this address are main-log records that happened to share the boundary page with the snapshot. + private void ClearBitsOnPage(long page, long untilAddress, RecoveryOptions options, long snapshotFromAddress = 0) { - var startLogicalAddress = hlog.GetStartLogicalAddress(page); - var endLogicalAddress = hlog.GetStartLogicalAddress(page + 1); - var physicalAddress = hlog.GetPhysicalAddress(startLogicalAddress); + var startLogicalAddress = hlogBase.GetLogicalAddressOfStartOfPage(page); + var endLogicalAddress = hlogBase.GetLogicalAddressOfStartOfPage(page + 1); + var physicalAddress = hlogBase.GetPhysicalAddress(startLogicalAddress); // no need to clear locks for records that will not end up in main memory - if (options.headAddress >= endLogicalAddress) return; + if (options.headAddress >= endLogicalAddress) + return; - long untilLogicalAddressInPage = hlogBase.GetPageSize(); - long pointer = 0; + var pageSize = hlogBase.GetPageSize(); + var endOffset = (untilAddress < endLogicalAddress) ? hlogBase.GetOffsetOnPage(untilAddress) : pageSize; - while (pointer < untilLogicalAddressInPage) + long recordOffset = hlogBase.pageHeaderSize; + while (recordOffset < endOffset) { - long recordStart = physicalAddress + pointer; - ref RecordInfo info = ref hlog.GetInfo(recordStart); - info.ClearBitsForDiskImages(); - - if (info.IsNull()) - pointer += RecordInfo.GetLength(); - else + var logRecord = new LogRecord(physicalAddress + recordOffset); + logRecord.InfoRef.ClearBitsForDiskImages(); + if (storeFunctions.CallOnDiskRead) { - int size = hlog.GetRecordSize(recordStart).Item2; - Debug.Assert(size <= hlogBase.GetPageSize()); - pointer += size; + var recordLogicalAddress = startLogicalAddress + recordOffset; + + // On the snapshot path, skip records below snapshotFromAddress — + // they are main-log records on the boundary page that were already + // processed (with OnDiskRead) in the main-log recovery pass. + if (snapshotFromAddress == 0 || recordLogicalAddress >= snapshotFromAddress) + { + storeFunctions.OnDiskRead(ref logRecord); + + // OnRecoverySnapshotRead fires only for snapshot-file records. + if (snapshotFromAddress > 0) + storeFunctions.OnRecoverySnapshotRead(ref logRecord); + } } + + long recordSize = logRecord.AllocatedSize; + Debug.Assert(recordSize > 0 && recordSize <= endOffset - recordOffset, + $"recordSize {recordSize} must be > 0 and <= remaining page space (possibly limited by untilAddress) {pageSize - endOffset};" + + $" recordOffset {recordOffset}, endOffset {endOffset}, pageSize {pageSize}"); + recordOffset += recordSize; } } - // Re-do the necessary log entries. We ensure that the InNewVersion test (to skip v+1 records) - // runs ONLY for the fuzzy region (which has v and v+1 records) because the earlier parts may - // have an incorrect InNewVersion status. - private unsafe bool RecoverFromPage(long startRecoveryAddress, - long fromLogicalAddressInPage, - long untilLogicalAddressInPage, - long pageLogicalAddress, - long pagePhysicalAddress, - long nextVersion, RecoveryOptions options) + /// + /// Re-do the necessary log records: + /// + /// If the record is in v+1 *and* is in the fuzzy region (which has v and v+1 records) and we are undoing nextVersion records, invalidate it. + /// We do this only in the fuzzy region because the earlier records may have a stale InNewVersion status. + /// Otherwise, update the tag chain for the record's hash and tag by inserting the record at the tail of the + /// + /// + /// The address from which to perform recovery (undo v+1 records and append to tag-chain tail) + /// The start address offset on the page to recover from (must be >= if any + /// The end address offset on the page to recover from (must be < PageSize) + /// The logical address of the start of the page + /// The physical address of the start of the page + /// Recovery options + /// True if we touched the page (and thus it needs to be flushed), else false + private unsafe bool RecoverFromPage(long recoverFromAddress, long pageFromAddressOffset, long pageUntilAddressOffset, + long pageStartLogicalAddress, long pageStartPhysicalAddress, RecoveryOptions options) { + Debug.Assert(pageFromAddressOffset >= hlogBase.pageHeaderSize, $"fromLogicalAddressInPage {pageFromAddressOffset} must be >= hlogBase.pageHeaderSize {hlogBase.pageHeaderSize} (which may be 0)"); + Debug.Assert(pageUntilAddressOffset <= hlogBase.GetPageSize(), $"pageSize {pageUntilAddressOffset} must be <= PageSize {hlogBase.GetPageSize()}"); var touched = false; - var pointer = default(long); - var recordStart = default(long); - - pointer = fromLogicalAddressInPage; - while (pointer < untilLogicalAddressInPage) + var recordOffset = pageFromAddressOffset; + while (recordOffset < pageUntilAddressOffset) { - recordStart = pagePhysicalAddress + pointer; - ref RecordInfo info = ref hlog.GetInfo(recordStart); + var logRecord = new LogRecord(pageStartPhysicalAddress + recordOffset); + ref var info = ref logRecord.InfoRef; - if (info.IsNull()) + if (info.IsNull) { - pointer += RecordInfo.GetLength(); + recordOffset += RecordInfo.Size; continue; } if (!info.Invalid) { - HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(ref hlog.GetKey(recordStart))); + HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(logRecord)); FindOrCreateTag(ref hei, hlogBase.BeginAddress); - bool ignoreRecord = ((pageLogicalAddress + pointer) >= options.fuzzyRegionStartAddress) && info.IsInNewVersion; - if (!options.undoNextVersion) ignoreRecord = false; - - if (!ignoreRecord) + if ((pageStartLogicalAddress + recordOffset) < options.fuzzyRegionStartAddress || !info.IsInNewVersion || !options.undoNextVersion) { - hei.entry.Address = pageLogicalAddress + pointer; - hei.entry.Tag = hei.tag; - hei.entry.Tentative = false; + // Update the hash table with this record + hei.entry.Set(pageStartLogicalAddress + recordOffset, hei.tag); hei.bucket->bucket_entries[hei.slot] = hei.entry.word; } else { + // Ignore this record touched = true; info.SetInvalid(); - if (info.PreviousAddress < startRecoveryAddress) + if (info.PreviousAddress < recoverFromAddress) { - hei.entry.Address = info.PreviousAddress; - hei.entry.Tag = hei.tag; - hei.entry.Tentative = false; + hei.entry.Set(info.PreviousAddress, hei.tag); hei.bucket->bucket_entries[hei.slot] = hei.entry.word; } } } - pointer += hlog.GetRecordSize(recordStart).Item2; + recordOffset += logRecord.AllocatedSize; } return touched; @@ -1179,44 +1353,39 @@ private unsafe bool RecoverFromPage(long startRecoveryAddress, private void AsyncFlushPageCallbackForRecovery(uint errorCode, uint numBytes, object context) { if (errorCode != 0) - { logger?.LogError($"{nameof(AsyncFlushPageCallbackForRecovery)} error: {{errorCode}}", errorCode); - } - // Set the page status to flushed + // Set the page status to "flush done" var result = (PageAsyncFlushResult)context; - if (Interlocked.Decrement(ref result.count) == 0) + if (result.Release() == 0) { - int pageIndex = hlogBase.GetPageIndexForPage(result.page); - + var pageIndex = hlogBase.GetPageIndexForPage(result.page); if (errorCode != 0) result.context.SignalFlushedError(pageIndex); else result.context.SignalFlushed(pageIndex); - - result.Free(); } } } - public abstract partial class AllocatorBase : IDisposable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public abstract partial class AllocatorBase : IDisposable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// - /// Restore log + /// Restore log; called from TsavoriteLog /// /// /// /// - /// + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, /// Number of pages to preload into memory after recovery public void RestoreHybridLog(long beginAddress, long headAddress, long fromAddress, long untilAddress, int numPagesToPreload = -1) { - if (RestoreHybridLogInitializePages(beginAddress, headAddress, fromAddress, untilAddress, numPagesToPreload, out var recoveryStatus, out long headPage, out long tailPage)) + if (RestoreHybridLogInitializePages(beginAddress, headAddress, fromAddress, untilAddress, numPagesToPreload, out var recoveryStatus, out long headPage, out long fromPage)) { - for (long page = headPage; page <= tailPage; page++) + for (long page = headPage; page <= fromPage; page++) recoveryStatus.WaitRead(GetPageIndexForPage(page)); } @@ -1224,19 +1393,19 @@ public void RestoreHybridLog(long beginAddress, long headAddress, long fromAddre } /// - /// Restore log + /// Restore log; called from TsavoriteLog /// /// /// /// - /// + /// The last address to scan; this is initially the tailAddress at the time of checkpoint flush, /// Number of pages to preload into memory after recovery /// public async ValueTask RestoreHybridLogAsync(long beginAddress, long headAddress, long fromAddress, long untilAddress, int numPagesToPreload = -1, CancellationToken cancellationToken = default) { - if (RestoreHybridLogInitializePages(beginAddress, headAddress, fromAddress, untilAddress, numPagesToPreload, out var recoveryStatus, out long headPage, out long tailPage)) + if (RestoreHybridLogInitializePages(beginAddress, headAddress, fromAddress, untilAddress, numPagesToPreload, out var recoveryStatus, out long headPage, out long fromPage)) { - for (long page = headPage; page <= tailPage; page++) + for (long page = headPage; page <= fromPage; page++) await recoveryStatus.WaitReadAsync(GetPageIndexForPage(page), cancellationToken).ConfigureAwait(false); } @@ -1244,11 +1413,11 @@ public async ValueTask RestoreHybridLogAsync(long beginAddress, long headAddress } private bool RestoreHybridLogInitializePages(long beginAddress, long headAddress, long fromAddress, long untilAddress, int numPagesToPreload, - out RecoveryStatus recoveryStatus, out long headPage, out long tailPage) + out RecoveryStatus recoveryStatus, out long headPage, out long fromPage) { if (numPagesToPreload != -1) { - var head = (GetPage(untilAddress) - numPagesToPreload) << LogPageSizeBits; + var head = GetFirstValidLogicalAddressOnPage(GetPage(untilAddress) - numPagesToPreload); if (head > headAddress) headAddress = head; } @@ -1256,65 +1425,60 @@ private bool RestoreHybridLogInitializePages(long beginAddress, long headAddress Debug.Assert(headAddress <= untilAddress); // Special cases: we do not load any records into memory - if ( - (beginAddress == untilAddress) || // Empty log - ((headAddress == untilAddress) && (GetOffsetInPage(headAddress) == 0)) // Empty in-memory page - ) + if ((beginAddress == untilAddress) || // Empty log + ((headAddress == untilAddress) && (GetOffsetOnPage(headAddress) == 0))) // Empty in-memory page { - if (!_wrapper.IsAllocated(GetPageIndexForAddress(headAddress))) - _wrapper.AllocatePage(GetPageIndexForAddress(headAddress)); + var pageIndex = GetPageIndexForAddress(headAddress); + if (!IsAllocated(pageIndex)) + _wrapper.AllocatePage(pageIndex); } - else + else if (headAddress < fromAddress) { - if (headAddress < fromAddress) - { - tailPage = GetPage(fromAddress); - headPage = GetPage(headAddress); + fromPage = GetPage(fromAddress); + headPage = GetPage(headAddress); - recoveryStatus = new RecoveryStatus(GetCapacityNumPages(), MinEmptyPageCount, tailPage, untilAddress, 0); - for (int i = 0; i < recoveryStatus.capacity; i++) - { - recoveryStatus.readStatus[i] = ReadStatus.Done; - } + var capacity = logSizeTracker is null ? BufferSize : RoundUp(logSizeTracker.TargetSize, PageSize) / PageSize; - var numPages = 0; - for (var page = headPage; page <= tailPage; page++) - { - var pageIndex = GetPageIndexForPage(page); - recoveryStatus.readStatus[pageIndex] = ReadStatus.Pending; - numPages++; - } + // Set all ReadStatus to done for the page range we will initially read. + recoveryStatus = new RecoveryStatus(BufferSize); + for (int i = 0; i < capacity; i++) + recoveryStatus.readStatus[i] = ReadStatus.Done; - AsyncReadPagesFromDevice(headPage, numPages, untilAddress, AsyncReadPagesCallbackForRecovery, recoveryStatus); - return true; + // Set all PendingStatus to Pending for all pages we will read. + var numPages = 0; + for (var page = headPage; page <= fromPage; page++) + { + var pageIndex = GetPageIndexForPage(page); + recoveryStatus.readStatus[pageIndex] = ReadStatus.Pending; + numPages++; } + + // Passing no objectLogDevice means we'll use the one in the allocator + AsyncReadPagesForRecovery(headPage, numPages, untilAddress, recoveryStatus); + return true; } + // fromAddress <= headAddress, so no pages to read recoveryStatus = default; - headPage = tailPage = 0; + headPage = fromPage = 0; return false; } - internal unsafe void AsyncReadPagesCallbackForRecovery(uint errorCode, uint numBytes, object context) + internal void AsyncReadPagesForRecoveryCallback(uint errorCode, uint numBytes, object context) { if (errorCode != 0) - { - logger?.LogError($"{nameof(AsyncReadPagesCallbackForRecovery)} error: {{errorCode}}", errorCode); - } + logger?.LogError($"{nameof(AsyncReadPagesForRecoveryCallback)} error: {{errorCode}}", errorCode); // Set the page status to "read done" var result = (PageAsyncReadResult)context; - if (result.freeBuffer1 != null) - { - _wrapper.PopulatePage(result.freeBuffer1.GetValidPointer(), result.freeBuffer1.required_bytes, result.page); - result.freeBuffer1.Return(); - } - int pageIndex = GetPageIndexForPage(result.page); + var pageIndex = GetPageIndexForPage(result.page); if (errorCode != 0) result.context.SignalReadError(pageIndex); else result.context.SignalRead(pageIndex); + + result.DisposeHandle(); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/CheckpointTrigger.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/CheckpointTrigger.cs new file mode 100644 index 00000000000..1f9099ef86b --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/CheckpointTrigger.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + /// + /// Identifies the checkpoint lifecycle point at which is called. + /// + public enum CheckpointTrigger + { + /// + /// PREPARE → IN_PROGRESS transition (version shift from v to v+1). + /// The application should set a barrier to prevent v+1 operations + /// from modifying external resources. + /// + VersionShift, + + /// + /// WAIT_FLUSH phase, after all v threads have completed and before the + /// snapshot flush begins. The application should snapshot external resources + /// and clear the barrier set during . + /// + FlushBegin, + + /// + /// REST phase, after the checkpoint is fully persisted. The application + /// should clean up outdated external checkpoint artifacts. + /// + CheckpointCompleted + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/DisposeReason.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/DisposeReason.cs index f412382110f..060aa490b6b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/DisposeReason.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/DisposeReason.cs @@ -4,7 +4,7 @@ namespace Tsavorite.core { /// - /// The reason for a call to + /// The reason for a call to . /// public enum DisposeReason { @@ -14,9 +14,14 @@ public enum DisposeReason None, /// - /// Failure of SingleWriter insertion of a record at the tail of the cache. + /// CopyUpdate cleared the object immediately for more efficient size tracking /// - SingleWriterCASFailed, + CopyUpdated, + + /// + /// Failure of InitialWriter insertion of a record at the tail of the cache. + /// + InitialWriterCASFailed, /// /// Failure of CopyUpdater insertion of a record at the tail of the cache. @@ -29,12 +34,17 @@ public enum DisposeReason InitialUpdaterCASFailed, /// - /// Failure of SingleDeleter insertion of a record at the tail of the cache. + /// Failure of InitialDeleter insertion of a record at the tail of the cache. + /// + InitialDeleterCASFailed, + + /// + /// Some CAS failed and retry could not use the record due to size or address restrictions /// - SingleDeleterCASFailed, + CASAndRetryFailed, /// - /// A record was deserialized from the disk for a pending Read or RMW operation. + /// A record was deserialized from the disk (or network buffer) for a pending Read or RMW operation. /// DeserializedFromDisk, @@ -44,8 +54,23 @@ public enum DisposeReason RevivificationFreeList, /// - /// A page was evicted from the in-memory portion of the main log, or from the readcache. + /// A new record was created for Upsert or RMW but the InitialWriter or InitialUpdater operation returned false + /// + InsertAbandoned, + + /// + /// Deleted but remains in hash chain so Key is unchanged + /// + Deleted, + + /// + /// Record expiration + /// + Expired, + + /// + /// Elided from hash chain but not put into Revivification free list /// - PageEviction + Elided } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordDisposer.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordDisposer.cs deleted file mode 100644 index 145c1a10805..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordDisposer.cs +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace Tsavorite.core -{ - /// - /// Interface to implement the Disposer component of - /// - public interface IRecordDisposer - { - /// - /// If true, with - /// is called on page evictions from both readcache and main log. Otherwise, the user can register an Observer and do any needed disposal there. - /// - public bool DisposeOnPageEviction { get; } - - /// - /// Dispose the Key and Value of a record, if necessary. See comments in for details. - /// - void DisposeRecord(ref TKey key, ref TValue value, DisposeReason reason, int newKeySize); - } - - /// - /// Default no-op implementation if - /// - /// It is appropriate to call methods on this instance as a no-op. - public struct DefaultRecordDisposer : IRecordDisposer - { - /// - /// Default instance - /// - public static readonly DefaultRecordDisposer Instance = new(); - - /// - /// Assumes the key and value have no need of Dispose(), and does nothing. - /// - public readonly bool DisposeOnPageEviction => false; - - /// - /// Assumes the key and value have no need of Dispose(), and does nothing. - /// - public readonly void DisposeRecord(ref TKey key, ref TValue value, DisposeReason reason, int newKeySize) - { - Debug.Assert(typeof(TKey) != typeof(SpanByte) && typeof(TValue) != typeof(SpanByte), "Must use SpanByteRecordDisposer"); - } - } - - /// - /// Default no-op implementation if for SpanByte - /// - public struct SpanByteRecordDisposer : IRecordDisposer - { - /// - /// Default instance - /// - public static readonly SpanByteRecordDisposer Instance = new(); - - /// - /// Assumes the key and value have no need of Dispose(), and does nothing. - /// - public readonly bool DisposeOnPageEviction => false; - - /// - /// If is and is >= 0, - /// this adjusts the key (and if necessary value) space as needed to preserve log zero-init correctness. - /// Otherwise the key and value have no need of disposal, and this does nothing. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public unsafe void DisposeRecord(ref SpanByte key, ref SpanByte value, DisposeReason reason, int newKeySize) - { - // We don't have to do anything with the Value unless the new key size requires adjusting the key length. - // newKeySize == -1 means we are preserving the existing key (e.g. for in-chain revivification). - if (reason != DisposeReason.RevivificationFreeList || newKeySize < 0) - return; - - var oldKeySize = Utility.RoundUp(key.TotalSize, Constants.kRecordAlignment); - - // We are changing the key size (e.g. revivification from the freelist with a new key). - // Our math here uses record alignment of keys as in the allocator, and assumes this will always be at least int alignment. - newKeySize = Utility.RoundUp(newKeySize, Constants.kRecordAlignment); - int keySizeChange = newKeySize - oldKeySize; - if (keySizeChange == 0) - return; - - // We are growing or shrinking. We don't care (here or in SingleWriter, InitialUpdater, CopyUpdater) what is inside the Key and Value, - // as long as we don't leave nonzero bytes after the used value space. So we just need to make sure the Value space starts immediately - // after the new key size. SingleWriter et al. will do the ShrinkSerializedLength on Value as needed. - if (keySizeChange < 0) - { - // We are shrinking the key; the Value of the new record will start after key + newKeySize, so set the new value length there. - *(int*)((byte*)Unsafe.AsPointer(ref key) + newKeySize) = value.Length - keySizeChange; // minus negative => plus positive - } - else - { - // We are growing the key; the Value of the new record will start somewhere in the middle of where the old Value was, so set the new value length there. - *(int*)((byte*)Unsafe.AsPointer(ref value) + keySizeChange) = value.Length - keySizeChange; - } - - // NewKeySize is (newKey).TotalSize. - key.Length = newKeySize - sizeof(int); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordTriggers.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordTriggers.cs new file mode 100644 index 00000000000..cb4e439c116 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordTriggers.cs @@ -0,0 +1,233 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ + /// + /// Identifies which log a record eviction originated from. Main log and read cache are + /// separate instances but share + /// a single ; this enum lets an + /// implementation tell the two apart when reacting to . + /// + public enum EvictionSource + { + /// The record is being evicted from the main hybrid log. + MainLog, + + /// The record is being evicted from the read cache. + ReadCache, + } + + /// + /// Per-record lifecycle callbacks invoked by the store at key events: + /// flush to disk, eviction, disposal, and disk read. + /// + public interface IRecordTriggers + { + /// + /// If true, is called per valid record on the + /// original in-memory page before it is flushed to disk. + /// + bool CallOnFlush => false; + + /// + /// If true, is called per non-tombstoned + /// record when a page is evicted past HeadAddress. Returning false lets the allocator skip the + /// per-record OnEvict callback when the application has no work to do. + /// Note: Tsavorite's internal heap-size accounting runs regardless of this flag. + /// + bool CallOnEvict => false; + + /// + /// If true, is called per record loaded from + /// disk into memory (recovery, delta log apply, pending reads, push scans). + /// + bool CallOnDiskRead => false; + + /// + /// If true, + /// is called after a successful CAS into the hash chain by TryCopyToTail (compaction lookup/scan, + /// CopyReadsToTail, ConditionalCopyToTail). Allows the application to perform per-record post-copy work + /// while the destination record is still sealed (concurrent readers see + /// and retry). + /// + bool CallPostCopyToTail => false; + + /// + /// If true, is called from + /// + /// after the underlying device has been truncated to the new BeginAddress. + /// + bool CallOnTruncate => false; + + /// + /// Called when a record is disposed due to delete, expiration, CAS failure, elision, + /// revivification, or other store-internal reasons. Use to + /// distinguish the event type (e.g. for tombstoning). + /// Heap-size accounting is handled internally by the store — implementations only need + /// this callback for app-level resource cleanup (e.g. releasing native handles). + /// NOT called for page eviction (use instead). + /// NOT called for transient records materialized from disk (use ). + /// Default implementation is a no-op. + /// + void OnDispose(ref LogRecord logRecord, DisposeReason reason) { } + + /// + /// Called when a transient is about to be disposed — e.g. a record + /// deserialized from disk for a pending Read/RMW, delivered via scan iteration, or streamed + /// during cluster migration/replication. If the value object implements + /// and holds resources that this DiskLogRecord owns, the application should invoke + /// from this callback. + /// + /// Caveat: scan iterators may briefly wrap an in-memory log record as a DiskLogRecord that + /// shares its value-object reference with the live on-log record. Implementations that + /// hold external resources should either gate disposal on or avoid + /// disposing the value object from this callback; uncritical disposal there would corrupt the + /// still-alive on-log record. + /// + /// Default implementation is a no-op. + /// + void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) { } + + /// + /// Called per valid record on the original in-memory page before flush to disk. + /// Allows the application to snapshot external resources (using + /// to disambiguate per-flush snapshot identity) and set flags on the live record. + /// Only called when is true. Default implementation is a no-op. + /// + /// The record being flushed (in-memory copy). + /// Logical address of the record being flushed; useful for naming + /// per-flush snapshot files immutably. + void OnFlush(ref LogRecord logRecord, long logicalAddress) { } + + /// + /// Called per non-tombstoned record when a page is evicted past HeadAddress. + /// Allows the application to free external resources (e.g. native memory). + /// Only called when is true. + /// + /// The record being evicted. + /// Which log (main or read cache) the record is being evicted from. + void OnEvict(ref LogRecord logRecord, EvictionSource source) { } + + /// + /// Called per record loaded from disk into memory. Allows the application to invalidate + /// stale external resource handles (e.g. native pointers from a previous process). + /// Only called when is true. Default implementation is a no-op. + /// + void OnDiskRead(ref LogRecord logRecord) { } + + /// + /// Called once before recovering records from a checkpoint snapshot file. + /// Default implementation is a no-op. + /// + void OnRecovery(System.Guid checkpointToken) { } + + /// + /// Called per record recovered from a checkpoint snapshot file (above FlushedUntilAddress). + /// Only called when is true. Default implementation is a no-op. + /// + void OnRecoverySnapshotRead(ref LogRecord logRecord) { } + + /// + /// Called at checkpoint lifecycle points identified by . + /// Default implementation is a no-op. + /// + void OnCheckpoint(CheckpointTrigger trigger, System.Guid checkpointToken) { } + + /// + /// Called after TryCopyToTail has CAS'd a destination record into the hash chain + /// but before unsealing the destination. While this callback runs, the destination record + /// is sealed (concurrent readers see and retry), so + /// implementations may safely mutate 's value bytes (e.g. + /// adjust embedded native handles). The source record is still in the chain with its old + /// state; implementations may also clear handles on the source if appropriate (analogous + /// to RIPROMOTE's PostCopyUpdater for live transfers). + /// + /// Only called when is true. Default implementation is a no-op. + /// + /// Source record type (in-memory log record or DiskLogRecord). + /// The source record that was copied from. + /// Logical address of the source record (or + /// if not available, e.g. read-cache source). + /// The destination record at the tail (sealed). + /// Logical address of the destination record at the tail. + void PostCopyToTail(in TSourceLogRecord srcLogRecord, long srcLogicalAddress, + ref LogRecord dstLogRecord, long dstLogicalAddress) + where TSourceLogRecord : ISourceLogRecord +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { } + + /// + /// Called from + /// AFTER the device has been truncated to . Allows the application + /// to clean up external state (e.g. per-flush snapshot files) tied to log addresses below the new + /// BeginAddress. + /// Only called when is true. Default implementation is a no-op. + /// + /// The new BeginAddress. Application state tied to addresses + /// strictly less than this value may be reclaimed. + void OnTruncate(long newBeginAddress) { } + } + + /// + /// Default no-op implementation of . + /// + /// It is appropriate to call methods on this instance as a no-op. + public readonly struct DefaultRecordTriggers : IRecordTriggers + { + /// Default instance. + public static readonly DefaultRecordTriggers Instance = new(); + + /// + public bool CallOnFlush => false; + + /// + public bool CallOnEvict => false; + + /// + public bool CallOnDiskRead => false; + + /// + public bool CallPostCopyToTail => false; + + /// + public bool CallOnTruncate => false; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) { } + } + + /// + /// No-op implementation of for SpanByte. + /// + public readonly struct SpanByteRecordTriggers : IRecordTriggers // TODO remove for dual + { + /// Default instance. + public static readonly SpanByteRecordTriggers Instance = new(); + + /// + public bool CallOnFlush => false; + + /// + public bool CallOnEvict => false; + + /// + public bool CallOnDiskRead => false; + + /// + public bool CallPostCopyToTail => false; + + /// + public bool CallOnTruncate => false; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) { } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IStoreFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IStoreFunctions.cs index 92ae0d94b66..08c7d5c4480 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IStoreFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IStoreFunctions.cs @@ -9,57 +9,98 @@ namespace Tsavorite.core /// /// The interface to define functions on the TsavoriteKV store itself (rather than a session). /// - public interface IStoreFunctions + public interface IStoreFunctions { #region Key Comparer /// Get a 64-bit hash code for a key - long GetKeyHashCode64(ref TKey key); + long GetKeyHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; /// Compare two keys for equality - bool KeysEqual(ref TKey k1, ref TKey k2); + bool KeysEqual(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; #endregion Key Comparer - #region Key Serializer - /// Indicates whether the Key Serializer is to be used - bool HasKeySerializer { get; } - - /// Instatiate a KeySerializer and begin Key serialization to the given stream. - /// This must instantiate a new serializer as multiple threads may be serializing or deserializing. - IObjectSerializer BeginSerializeKey(Stream stream); - - /// Instatiate a KeySerializer and begin Key deserialization from the given stream. - /// This must instantiate a new serializer as multiple threads may be serializing or deserializing. - IObjectSerializer BeginDeserializeKey(Stream stream); - #endregion Key Serializer - #region Value Serializer + /// Creates an instance of the Value Serializer + IObjectSerializer CreateValueObjectSerializer(); + /// Indicates whether the Value Serializer is to be used bool HasValueSerializer { get; } /// Instatiate a ValueSerializer and begin Value serialization to the given stream. /// This must instantiate a new serializer as multiple threads may be serializing or deserializing. - IObjectSerializer BeginSerializeValue(Stream stream); + IObjectSerializer BeginSerializeValue(Stream stream); /// Instatiate a ValueSerializer and begin Value deserialization from the given stream. /// This must instantiate a new serializer as multiple threads may be serializing or deserializing. - IObjectSerializer BeginDeserializeValue(Stream stream); + IObjectSerializer BeginDeserializeValue(Stream stream); #endregion Value Serializer - #region Record Disposer - /// - /// If true, with - /// is called on page evictions from both readcache and main log. Otherwise, the user can register an Observer and - /// do any needed disposal there. - /// - bool DisposeOnPageEviction { get; } - - /// Dispose the Key and Value of a record, if necessary. - /// The key for the record - /// The value for the record - /// For only, this is a record from the freelist and we may be disposing the key as well as value - /// (it is -1 when revivifying a record in the hash chain or when doing a RETRY; for these the key does not change) - void DisposeRecord(ref TKey key, ref TValue value, DisposeReason reason, int newKeySize = -1); - #endregion Record Disposer + #region Record Triggers + /// + bool CallOnFlush { get; } + + /// + bool CallOnEvict { get; } + + /// + bool CallOnDiskRead { get; } + + /// + bool CallPostCopyToTail { get; } + + /// + bool CallOnTruncate { get; } + + /// + void OnDispose(ref LogRecord logRecord, DisposeReason reason); + + /// + void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason); + + /// + void OnFlush(ref LogRecord logRecord, long logicalAddress); + + /// + void OnEvict(ref LogRecord logRecord, EvictionSource source); + + /// + void OnDiskRead(ref LogRecord logRecord); + + /// + void OnRecovery(System.Guid checkpointToken); + + /// + void OnRecoverySnapshotRead(ref LogRecord logRecord); + + /// + void OnCheckpoint(CheckpointTrigger trigger, System.Guid checkpointToken); + + /// + void PostCopyToTail(in TSourceLogRecord srcLogRecord, long srcLogicalAddress, + ref LogRecord dstLogRecord, long dstLogicalAddress) + where TSourceLogRecord : ISourceLogRecord +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; + + /// + void OnTruncate(long newBeginAddress); + #endregion Record Triggers #region Checkpoint Completion /// Set the parameterless checkpoint completion callback. diff --git a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/StoreFunctions.cs b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/StoreFunctions.cs index a175988f27e..33aa6ebe5a7 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/StoreFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/StoreFunctions.cs @@ -8,7 +8,7 @@ namespace Tsavorite.core { /// - /// Store functions for and . + /// Store functions for an instance of TsavoriteKV. /// /// /// The implementation takes instances of the supported interfaces (e.g. ) to allow custom @@ -16,24 +16,19 @@ namespace Tsavorite.core /// because there is no need to wrap calls to them with additional functionality. This can be changed to redirect if such wrapper /// functionality is needed. /// - public struct StoreFunctions - (TKeyComparer keyComparer, Func> keySerializerCreator, Func> valueSerializerCreator, TRecordDisposer recordDisposer) - : IStoreFunctions - where TKeyComparer : IKeyComparer - where TRecordDisposer : IRecordDisposer + public struct StoreFunctions(TKeyComparer keyComparer, Func> valueSerializerCreator, TRecordTriggers recordTriggers) : IStoreFunctions + where TKeyComparer : IKeyComparer + where TRecordTriggers : IRecordTriggers { #region Fields /// Compare two keys for equality, and get a key's hash code. readonly TKeyComparer keyComparer = keyComparer; - /// Serialize a Key to persistent storage - readonly Func> keySerializerCreator = keySerializerCreator; - /// Serialize a Value to persistent storage - readonly Func> valueSerializerCreator = valueSerializerCreator; + readonly Func> valueSerializerCreator = valueSerializerCreator; /// Dispose a record - readonly TRecordDisposer recordDisposer = recordDisposer; + readonly TRecordTriggers recordTriggers = recordTriggers; /// Optional checkpoint completion callback, set separately from ctor. Action checkpointCompletionCallback = () => { }; @@ -42,63 +37,112 @@ public struct StoreFunctions #region Key Comparer /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly long GetKeyHashCode64(ref TKey key) => keyComparer.GetHashCode64(ref key); + public readonly long GetKeyHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => keyComparer.GetHashCode64(key); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly bool KeysEqual(ref TKey k1, ref TKey k2) => keyComparer.Equals(ref k1, ref k2); + public readonly bool KeysEqual(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => keyComparer.Equals(k1, k2); #endregion Key Comparer - #region Key Serializer - /// - public readonly bool HasKeySerializer => keySerializerCreator is not null; - - /// - public readonly IObjectSerializer BeginSerializeKey(Stream stream) - { - var keySerializer = keySerializerCreator(); - keySerializer.BeginSerialize(stream); - return keySerializer; - } - + #region Value Serializer /// - public readonly IObjectSerializer BeginDeserializeKey(Stream stream) - { - var keySerializer = keySerializerCreator(); - keySerializer.BeginDeserialize(stream); - return keySerializer; - } - #endregion Key Serializer + public readonly IObjectSerializer CreateValueObjectSerializer() => valueSerializerCreator is null ? default : valueSerializerCreator(); - #region Value Serializer /// public readonly bool HasValueSerializer => valueSerializerCreator is not null; /// - public readonly IObjectSerializer BeginSerializeValue(Stream stream) + public readonly IObjectSerializer BeginSerializeValue(Stream stream) { - var valueSerializer = valueSerializerCreator(); + var valueSerializer = CreateValueObjectSerializer(); valueSerializer.BeginSerialize(stream); return valueSerializer; } /// - public readonly IObjectSerializer BeginDeserializeValue(Stream stream) + public readonly IObjectSerializer BeginDeserializeValue(Stream stream) { - var valueSerializer = valueSerializerCreator(); + var valueSerializer = CreateValueObjectSerializer(); valueSerializer.BeginDeserialize(stream); return valueSerializer; } #endregion Value Serializer - #region Record Disposer + #region Record Triggers /// - public readonly bool DisposeOnPageEviction => recordDisposer.DisposeOnPageEviction; + public readonly bool CallOnFlush => recordTriggers.CallOnFlush; + + /// + public readonly bool CallOnEvict => recordTriggers.CallOnEvict; + + /// + public readonly bool CallOnDiskRead => recordTriggers.CallOnDiskRead; + + /// + public readonly bool CallPostCopyToTail => recordTriggers.CallPostCopyToTail; + + /// + public readonly bool CallOnTruncate => recordTriggers.CallOnTruncate; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnDispose(ref LogRecord logRecord, DisposeReason reason) => recordTriggers.OnDispose(ref logRecord, reason); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) => recordTriggers.OnDisposeDiskRecord(ref logRecord, reason); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnFlush(ref LogRecord logRecord, long logicalAddress) => recordTriggers.OnFlush(ref logRecord, logicalAddress); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnEvict(ref LogRecord logRecord, EvictionSource source) => recordTriggers.OnEvict(ref logRecord, source); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnDiskRead(ref LogRecord logRecord) => recordTriggers.OnDiskRead(ref logRecord); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnRecovery(System.Guid checkpointToken) => recordTriggers.OnRecovery(checkpointToken); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void OnRecoverySnapshotRead(ref LogRecord logRecord) => recordTriggers.OnRecoverySnapshotRead(ref logRecord); + + /// + public readonly void OnCheckpoint(CheckpointTrigger trigger, System.Guid checkpointToken) => recordTriggers.OnCheckpoint(trigger, checkpointToken); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void PostCopyToTail(in TSourceLogRecord srcLogRecord, long srcLogicalAddress, + ref LogRecord dstLogRecord, long dstLogicalAddress) + where TSourceLogRecord : ISourceLogRecord +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => recordTriggers.PostCopyToTail(in srcLogRecord, srcLogicalAddress, ref dstLogRecord, dstLogicalAddress); /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly void DisposeRecord(ref TKey key, ref TValue value, DisposeReason reason, int newKeySize) => recordDisposer.DisposeRecord(ref key, ref value, reason, newKeySize); - #endregion Record Disposer + public readonly void OnTruncate(long newBeginAddress) => recordTriggers.OnTruncate(newBeginAddress); + #endregion Record Triggers #region Checkpoint Completion /// @@ -110,48 +154,45 @@ public readonly IObjectSerializer BeginDeserializeValue(Stream stream) } /// - /// A non-parameterized version of StoreFunctions that provides type-reduced Create() methods. + /// A minimally-parameterized version of StoreFunctions that provides type-reduced Create() methods. /// - public struct StoreFunctions + public struct StoreFunctions { /// /// Construct a StoreFunctions instance with all types specified and contained instances passed, e.g. for custom objects. /// - public static StoreFunctions Create - (TKeyComparer keyComparer, Func> keySerializerCreator, Func> valueSerializerCreator, TRecordDisposer recordDisposer) - where TKeyComparer : IKeyComparer - where TRecordDisposer : IRecordDisposer - => new(keyComparer, keySerializerCreator, valueSerializerCreator, recordDisposer); + public static StoreFunctions Create + (TKeyComparer keyComparer, Func> valueSerializerCreator, TRecordTriggers recordTriggers) + where TKeyComparer : IKeyComparer + where TRecordTriggers : IRecordTriggers + => new(keyComparer, valueSerializerCreator, recordTriggers); /// /// Construct a StoreFunctions instance with all types specified and contained instances passed, e.g. for custom objects. /// - public static StoreFunctions> Create - (TKeyComparer keyComparer, Func> keySerializerCreator, Func> valueSerializerCreator) - where TKeyComparer : IKeyComparer - => new(keyComparer, keySerializerCreator, valueSerializerCreator, new DefaultRecordDisposer()); + public static StoreFunctions Create(TKeyComparer keyComparer, Func> valueSerializerCreator) + where TKeyComparer : IKeyComparer + => new(keyComparer, valueSerializerCreator, DefaultRecordTriggers.Instance); /// /// Construct a StoreFunctions instance with all types specified and contained instances passed, e.g. for custom objects. /// - public static StoreFunctions Create - (TKeyComparer keyComparer, TRecordDisposer recordDisposer) - where TKeyComparer : IKeyComparer - where TRecordDisposer : IRecordDisposer - => new(keyComparer, keySerializerCreator: null, valueSerializerCreator: null, recordDisposer); + public static StoreFunctions Create(TKeyComparer keyComparer, TRecordTriggers recordTriggers) + where TKeyComparer : IKeyComparer + where TRecordTriggers : IRecordTriggers + => new(keyComparer, valueSerializerCreator: null, recordTriggers); /// - /// Store functions for and that take only the + /// Store functions that take only the /// - public static StoreFunctions> Create - (TKeyComparer keyComparer) - where TKeyComparer : IKeyComparer - => new(keyComparer, keySerializerCreator: null, valueSerializerCreator: null, DefaultRecordDisposer.Instance); + public static StoreFunctions Create(TKeyComparer keyComparer) + where TKeyComparer : IKeyComparer + => new(keyComparer, valueSerializerCreator: null, DefaultRecordTriggers.Instance); /// - /// Store functions for Key and Value + /// Store functions for Key and Value /// - public static StoreFunctions Create() - => new(SpanByteComparer.Instance, keySerializerCreator: null, valueSerializerCreator: null, SpanByteRecordDisposer.Instance); + public static StoreFunctions Create() + => new(SpanByteComparer.Instance, valueSerializerCreator: null, DefaultRecordTriggers.Instance); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Constants.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Constants.cs index 61c5a6538d4..f6518800ec3 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Constants.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Constants.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +#pragma warning disable IDE1006 // Naming Styles: Must begin with uppercase letter + namespace Tsavorite.core { internal static class Constants @@ -10,38 +12,13 @@ internal static class Constants // RecordInfo has a long field, so it should be aligned to 8-bytes public const int kRecordAlignment = 8; - - public const bool kFineGrainedHandoverRecord = false; - public const bool kFineGrainedHandoverBucket = true; + public const int kRecordAlignmentMask = kRecordAlignment - 1; /// Number of entries per bucket (assuming 8-byte entries to fill a cacheline) /// Number of bits per bucket (assuming 8-byte entries to fill a cacheline) public const int kBitsPerBucket = 3; - public const int kEntriesPerBucket = 1 << kBitsPerBucket; - // Position of fields in hash-table entry - public const int kTentativeBitShift = 63; - - public const long kTentativeBitMask = 1L << kTentativeBitShift; - - public const int kPendingBitShift = 62; - - public const long kPendingBitMask = 1L << kPendingBitShift; - - public const int kReadCacheBitShift = 47; - public const long kReadCacheBitMask = 1L << kReadCacheBitShift; - - public const int kTagSize = 14; - public const int kTagShift = 62 - kTagSize; - public const long kTagMask = (1L << kTagSize) - 1; - public const long kTagPositionMask = kTagMask << kTagShift; - public const int kAddressBits = 48; - public const long kAddressMask = (1L << kAddressBits) - 1; - - // Position of tag in hash value (offset is always in the least significant bits) - public const int kHashTagShift = 64 - kTagSize; - // Default number of entries in the lock table. public const int kDefaultLockTableSize = 16 * 1024; @@ -58,9 +35,6 @@ internal static class Constants /// Invalid value in the hash table public const long kInvalidEntry = 0; - /// Number of times to retry a compare-and-swap before failure - public const long kRetryThreshold = 1000000; // TODO unused - /// Number of times to spin before awaiting or Waiting for a Flush Task. public const long kFlushSpinCount = 10; // TODO verify this number @@ -71,10 +45,5 @@ internal static class Constants // Size of chunks for garbage collection public const int kSizeofChunkBits = 14; public const int kSizeofChunk = 1 << 14; - - public const long kInvalidAddress = 0; - public const long kTempInvalidAddress = 1; - public const long kUnknownAddress = 2; - public const int kFirstValidAddress = 64; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Extensions.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Extensions.cs deleted file mode 100644 index f3e6079939e..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Extensions.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -#pragma warning disable 0162 - -using System; - -namespace Tsavorite.core -{ - /// - /// Log subscription extensions - /// - public static class Extensions - { - /// - /// Create observable of log records - /// - /// - /// - /// - /// - public static IObservable> ToRecordObservable(this IObservable> source) - { - return new RecordObservable(source); - } - - internal sealed class RecordObservable : IObservable> - { - readonly IObservable> o; - - public RecordObservable(IObservable> o) - { - this.o = o; - } - - public IDisposable Subscribe(IObserver> observer) - { - return o.Subscribe(new RecordObserver(observer)); - } - } - - internal sealed class RecordObserver : IObserver> - { - private readonly IObserver> observer; - - public RecordObserver(IObserver> observer) - { - this.observer = observer; - } - - public void OnCompleted() - { - observer.OnCompleted(); - } - - public void OnError(Exception error) - { - observer.OnError(error); - } - - public void OnNext(ITsavoriteScanIterator v) - { - while (v.GetNext(out RecordInfo info, out TKey key, out TValue value)) - { - observer.OnNext(new AllocatorRecord { info = info, key = key, value = value }); - } - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucket.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucket.cs index 62134ad6afd..0584ee7c178 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucket.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucket.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; +using static Tsavorite.core.LogAddress; namespace Tsavorite.core { @@ -12,11 +13,11 @@ namespace Tsavorite.core internal unsafe struct HashBucket { // We use the first overflow bucket for latching, reusing all bits after the address. - const int kSharedLatchBits = 63 - Constants.kAddressBits; + const int kSharedLatchBits = 63 - kAddressBits; const int kExclusiveLatchBits = 1; // Shift positions of latches in word - const int kSharedLatchBitOffset = Constants.kAddressBits; + const int kSharedLatchBitOffset = kAddressBits; const int kExclusiveLatchBitOffset = kSharedLatchBitOffset + kSharedLatchBits; // Shared latch constants @@ -40,7 +41,8 @@ public static bool TryAcquireSharedLatch(HashBucket* bucket) { ref long entry_word = ref bucket->bucket_entries[Constants.kOverflowBucketIndex]; - for (int spinCount = Constants.kMaxLockSpins; ; Thread.Yield()) + var spinCount = Constants.kMaxLockSpins; + while (true) { // Note: If reader starvation is encountered, consider rotating priority between reader and writer locks. long expected_word = entry_word; @@ -54,6 +56,7 @@ public static bool TryAcquireSharedLatch(HashBucket* bucket) } if (--spinCount <= 0) return false; + _ = Thread.Yield(); } } @@ -79,7 +82,8 @@ public static bool TryAcquireExclusiveLatch(HashBucket* bucket) ref long entry_word = ref bucket->bucket_entries[Constants.kOverflowBucketIndex]; // Acquire exclusive lock (readers may still be present; we'll drain them later) - for (int spinCount = Constants.kMaxLockSpins; ; Thread.Yield()) + var spinCount = Constants.kMaxLockSpins; + while (true) { long expected_word = entry_word; if ((expected_word & kExclusiveLatchBitMask) == 0) @@ -89,10 +93,11 @@ public static bool TryAcquireExclusiveLatch(HashBucket* bucket) } if (--spinCount <= 0) return false; + _ = Thread.Yield(); } // Wait for readers to drain. Another session may hold an SLock on this bucket and need an epoch refresh to unlock, so limit this to avoid deadlock. - for (var ii = 0; ii < Constants.kMaxReaderLockDrainSpins; ++ii) + for (var ii = 0; ii < Constants.kMaxReaderLockDrainSpins; ii++) { if ((entry_word & kSharedLatchBitMask) == 0) return true; @@ -100,11 +105,12 @@ public static bool TryAcquireExclusiveLatch(HashBucket* bucket) } // Release the exclusive bit and return false so the caller will retry the operation. Since we still have readers, we must CAS. - for (; ; Thread.Yield()) + while (true) { long expected_word = entry_word; if (Interlocked.CompareExchange(ref entry_word, expected_word & ~kExclusiveLatchBitMask, expected_word) == expected_word) break; + _ = Thread.Yield(); } return false; } @@ -115,7 +121,8 @@ public static bool TryPromoteLatch(HashBucket* bucket) ref long entry_word = ref bucket->bucket_entries[Constants.kOverflowBucketIndex]; // Acquire shared lock - for (int spinCount = Constants.kMaxLockSpins; ; Thread.Yield()) + var spinCount = Constants.kMaxLockSpins; + while (true) { long expected_word = entry_word; Debug.Assert((expected_word & kSharedLatchBitMask) != 0, "Trying to promote a bucket that is not S latched to X latch"); @@ -127,10 +134,11 @@ public static bool TryPromoteLatch(HashBucket* bucket) } if (--spinCount <= 0) return false; + _ = Thread.Yield(); } // Wait for readers to drain. Another session may hold an SLock on this bucket and need an epoch refresh to unlock, so limit this to avoid deadlock. - for (var ii = 0; ii < Constants.kMaxReaderLockDrainSpins; ++ii) + for (var ii = 0; ii < Constants.kMaxReaderLockDrainSpins; ii++) { if ((entry_word & kSharedLatchBitMask) == 0) return true; @@ -138,12 +146,13 @@ public static bool TryPromoteLatch(HashBucket* bucket) } // Reverse the shared-to-exclusive bit transition and return false so the caller will retry the operation. Since we still have readers, we must CAS. - for (; ; Thread.Yield()) + while (true) { long expected_word = entry_word; long new_word = (expected_word & ~kExclusiveLatchBitMask) + kSharedLatchIncrement; if (expected_word == Interlocked.CompareExchange(ref entry_word, new_word, expected_word)) break; + _ = Thread.Yield(); } return false; } @@ -160,11 +169,12 @@ public static void ReleaseExclusiveLatch(HashBucket* bucket) Debug.Assert((entry_word & kExclusiveLatchBitMask) != 0, "Trying to X unlatch an unlatched bucket"); // CAS is necessary to preserve the reader count, and also the address in the overflow bucket may change from unassigned to assigned. - for (; ; Thread.Yield()) + while (true) { - long expected_word = entry_word; + var expected_word = entry_word; if (expected_word == Interlocked.CompareExchange(ref entry_word, expected_word & ~kExclusiveLatchBitMask, expected_word)) break; + _ = Thread.Yield(); } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucketEntry.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucketEntry.cs index a73cf7b887a..3ce675dc325 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucketEntry.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/HashBucketEntry.cs @@ -3,67 +3,79 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using static Tsavorite.core.LogAddress; + +#pragma warning disable IDE1006 // Naming Styles: Must begin with uppercase letter namespace Tsavorite.core { - // Long value layout: [1-bit tentative][15-bit TAG][48-bit address] - // Physical little endian memory layout: [48-bit address][15-bit TAG][1-bit tentative] + // Long value layout: [1-bit tentative][13-bit TAG][50-bit address] + // Physical little endian memory layout: [50-bit address][13-bit TAG][1-bit tentative] [StructLayout(LayoutKind.Explicit, Size = 8)] internal struct HashBucketEntry { + // Position of fields in hash-table entry + public const int kTentativeBitShift = 63; + public const long kTentativeBitMask = 1L << kTentativeBitShift; + + public const int kTagSize = 63 - kAddressBits; + public const int kTagShift = 63 - kTagSize; + public const long kTagMask = (1L << kTagSize) - 1; + public const long kTagPositionMask = kTagMask << kTagShift; + + // Position of tag in hash value (offset is always in the least significant bits) + public const int kHashTagShift = 64 - kTagSize; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ushort GetTag(long hashCode) => (ushort)(((ulong)hashCode >> kHashTagShift) & kTagMask); + [FieldOffset(0)] public long word; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(long address, ushort tag) + { + word = (address & kAddressBitMask) + | ((tag & kTagMask) << kTagShift); + } + public long Address { [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get => word & Constants.kAddressMask; + readonly get => word & kAddressBitMask; - set - { - word &= ~Constants.kAddressMask; - word |= value & Constants.kAddressMask; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set => word = (word & ~kAddressBitMask) | (value & kAddressBitMask); } - public readonly long AbsoluteAddress => Utility.AbsoluteAddress(Address); - public ushort Tag { [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get => (ushort)((word & Constants.kTagPositionMask) >> Constants.kTagShift); + readonly get => (ushort)((word & kTagPositionMask) >> kTagShift); - set - { - word &= ~Constants.kTagPositionMask; - word |= (long)value << Constants.kTagShift; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set => word = (word & ~kTagPositionMask) | ((value & kTagMask) << kTagShift); } public bool Tentative { [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get => (word & Constants.kTentativeBitMask) != 0; - - set - { - if (value) - word |= Constants.kTentativeBitMask; - else - word &= ~Constants.kTentativeBitMask; - } + readonly get => (word & kTentativeBitMask) != 0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set => word = value ? (word | kTentativeBitMask) : (word & ~kTentativeBitMask); } - public readonly bool ReadCache + public readonly bool IsReadCache { [MethodImpl(MethodImplOptions.AggressiveInlining)] - get => (word & Constants.kReadCacheBitMask) != 0; + get => LogAddress.IsReadCache(word); } public override readonly string ToString() { - var addrRC = ReadCache ? "(rc)" : string.Empty; static string bstr(bool value) => value ? "T" : "F"; - return $"addr {AbsoluteAddress}{addrRC}, tag {Tag}, tent {bstr(Tentative)}"; + return $"addr {AddressString(Address)}, tag {Tag}, tent {bstr(Tentative)}"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/BlockAllocate.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/BlockAllocate.cs index 9b0173c6ab5..1dfc857fe2f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/BlockAllocate.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/BlockAllocate.cs @@ -7,30 +7,30 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool TryBlockAllocate( - AllocatorBase allocator, + AllocatorBase allocator, int recordSize, out long logicalAddress, ref PendingContext pendingContext, out OperationStatus internalStatus) { - pendingContext.flushEvent = allocator.FlushEvent; - logicalAddress = allocator.TryAllocateRetryNow(recordSize); - if (logicalAddress > 0) + pendingContext.flushEvent = allocator.flushEvent; + if (allocator.TryAllocateRetryNow(recordSize, out logicalAddress)) { pendingContext.flushEvent = default; internalStatus = OperationStatus.SUCCESS; return true; } - // logicalAddress less than 0 (RETRY_NOW) should already have been handled - Debug.Assert(logicalAddress == 0); - // We expect flushEvent to be signaled. + // logicalAddress less than 0 (RETRY_NOW) should already have been handled. We expect flushEvent to be signaled. + Debug.Assert(logicalAddress == 0, "Expected RETRY_LATER"); internalStatus = OperationStatus.ALLOCATE_FAILED; return false; } @@ -39,70 +39,88 @@ private static bool TryBlockAllocate( internal struct AllocateOptions { /// If true, use the non-revivification recycling of records that failed to CAS and are carried in PendingContext through RETRY. - internal bool Recycle; + internal bool recycle; /// If true, the source record is elidable so we can try to elide from the tag chain (and transfer it to the FreeList if we're doing Revivification). - internal bool ElideSourceRecord; + internal bool elideSourceRecord; }; [MethodImpl(MethodImplOptions.AggressiveInlining)] bool TryAllocateRecord(TSessionFunctionsWrapper sessionFunctions, ref PendingContext pendingContext, - ref OperationStackContext stackCtx, int actualSize, ref int allocatedSize, int newKeySize, AllocateOptions options, + ref OperationStackContext stackCtx, ref RecordSizeInfo sizeInfo, AllocateOptions options, out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { status = OperationStatus.SUCCESS; // MinRevivAddress is also needed for pendingContext-based record reuse. - var minMutableAddress = GetMinRevivifiableAddress(); - var minRevivAddress = minMutableAddress; + var minRevivAddress = GetMinRevivifiableAddress(); + + // If are eliding the first record in the tag chain, then there can be no others; we are removing the only record in the chain. Otherwise, the new record must be above hei.Address. + if ((minRevivAddress <= stackCtx.hei.Address) && (!options.elideSourceRecord || stackCtx.hei.Address != stackCtx.recSrc.LogicalAddress)) + minRevivAddress = stackCtx.hei.Address; - if (options.Recycle && pendingContext.retryNewLogicalAddress != Constants.kInvalidAddress && GetAllocationForRetry(sessionFunctions, ref pendingContext, minRevivAddress, ref allocatedSize, newKeySize, out newLogicalAddress, out newPhysicalAddress)) + if (options.recycle && pendingContext.retryNewLogicalAddress != kInvalidAddress + && GetAllocationForRetry(sessionFunctions, ref pendingContext, minRevivAddress, in sizeInfo, out newLogicalAddress, out newPhysicalAddress)) + { + new LogRecord(newPhysicalAddress).PrepareForRevivification(ref sizeInfo); return true; + } if (RevivificationManager.UseFreeRecordPool) { - if (!options.ElideSourceRecord && stackCtx.hei.Address >= minMutableAddress) - minRevivAddress = stackCtx.hei.Address; if (sessionFunctions.Ctx.IsInV1) { var fuzzyStartAddress = _hybridLogCheckpoint.info.startLogicalAddress; if (fuzzyStartAddress > minRevivAddress) minRevivAddress = fuzzyStartAddress; } - if (TryTakeFreeRecord(sessionFunctions, actualSize, ref allocatedSize, newKeySize, minRevivAddress, out newLogicalAddress, out newPhysicalAddress)) + if (TryTakeFreeRecord(sessionFunctions, in sizeInfo, minRevivAddress, out newLogicalAddress, out newPhysicalAddress)) + { + new LogRecord(newPhysicalAddress).PrepareForRevivification(ref sizeInfo); return true; + } } // Spin to make sure newLogicalAddress is > recSrc.LatestLogicalAddress (the .PreviousAddress and CAS comparison value). - for (; ; Thread.Yield()) + while (true) { - if (!TryBlockAllocate(hlogBase, allocatedSize, out newLogicalAddress, ref pendingContext, out status)) + if (!TryBlockAllocate(hlogBase, sizeInfo.AllocatedInlineRecordSize, out newLogicalAddress, ref pendingContext, out status)) break; - newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); + newPhysicalAddress = hlogBase.GetPhysicalAddress(newLogicalAddress); + + // If allocation had to flush and did it inline, then the epoch was refreshed and we need to check for address safety. if (VerifyInMemoryAddresses(ref stackCtx)) { if (newLogicalAddress > stackCtx.recSrc.LatestLogicalAddress) return true; // This allocation is below the necessary address so put it on the free list or abandon it, then repeat the loop. - if (!RevivificationManager.UseFreeRecordPool || !RevivificationManager.TryAdd(newLogicalAddress, newPhysicalAddress, allocatedSize, ref sessionFunctions.Ctx.RevivificationStats)) - hlog.GetInfo(newPhysicalAddress).SetInvalid(); // Skip on log scan + if (RevivificationManager.UseFreeRecordPool) + { + // Set up a simple LogRecord with specified key size and value size taking the entire non-key space (we don't have optionals now) + // so revivification can read the record size. + var logRecord = hlog.CreateLogRecord(newLogicalAddress, newPhysicalAddress); + logRecord.InitializeForReuse(in sizeInfo); + + // Call RevivificationManager.TryAdd() directly, as here we've done InitializeForReuse of a new record so don't want OnDispose. + if (RevivificationManager.TryAdd(newLogicalAddress, ref logRecord, ref sessionFunctions.Ctx.RevivificationStats)) + continue; + } + LogRecord.GetInfo(newPhysicalAddress).SetInvalid(); // Skip on log scan + _ = Thread.Yield(); continue; } - // In-memory source dropped below HeadAddress during BlockAllocate. Save the record for retry if we can. - ref var newRecordInfo = ref hlog.GetInfo(newPhysicalAddress); - if (options.Recycle) + // The in-memory source dropped below HeadAddress during BlockAllocate. Save the record for retry if we can and return RETRY_LATER. + if (options.recycle) { - ref var newValue = ref hlog.GetValue(newPhysicalAddress); - _ = hlog.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize); - var valueOffset = (int)((long)Unsafe.AsPointer(ref newValue) - newPhysicalAddress); - SetExtraValueLength(ref hlog.GetValue(newPhysicalAddress), ref newRecordInfo, actualSize - valueOffset, allocatedSize - valueOffset); - SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress, allocatedSize); + var logRecord = new LogRecord(newPhysicalAddress); + logRecord.InitializeForReuse(in sizeInfo); + SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress); } else - newRecordInfo.SetInvalid(); // Skip on log scan + LogRecord.GetInfoRef(newPhysicalAddress).SetInvalid(); // Skip on log scan status = OperationStatus.RETRY_LATER; break; } @@ -112,28 +130,29 @@ bool TryAllocateRecord(TSes } [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool TryAllocateRecordReadCache(ref PendingContext pendingContext, ref OperationStackContext stackCtx, - int allocatedSize, out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status) + bool TryAllocateRecordReadCache(ref PendingContext pendingContext, ref OperationStackContext stackCtx, + in RecordSizeInfo recordSizeInfo, out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status) { // Spin to make sure the start of the tag chain is not readcache, or that newLogicalAddress is > the first address in the tag chain. - for (; ; Thread.Yield()) + while (true) { - if (!TryBlockAllocate(readCacheBase, allocatedSize, out newLogicalAddress, ref pendingContext, out status)) + if (!TryBlockAllocate(readcacheBase, recordSizeInfo.AllocatedInlineRecordSize, out newLogicalAddress, ref pendingContext, out status)) break; + newPhysicalAddress = readcacheBase.GetPhysicalAddress(newLogicalAddress); - newPhysicalAddress = readcache.GetPhysicalAddress(newLogicalAddress); if (VerifyInMemoryAddresses(ref stackCtx)) { - if (!stackCtx.hei.IsReadCache || newLogicalAddress > stackCtx.hei.AbsoluteAddress) + if (!stackCtx.hei.IsReadCache || newLogicalAddress > AbsoluteAddress(stackCtx.hei.Address)) return true; // This allocation is below the necessary address so abandon it and repeat the loop. - ReadCacheAbandonRecord(newPhysicalAddress); + TsavoriteKV.ReadCacheAbandonRecord(newPhysicalAddress); + _ = Thread.Yield(); continue; } - // In-memory source dropped below HeadAddress during BlockAllocate. - ReadCacheAbandonRecord(newPhysicalAddress); + // The in-memory source dropped below HeadAddress during BlockAllocate. Abandon the record (TODO: reuse readcache records) and return RETRY_LATER. + TsavoriteKV.ReadCacheAbandonRecord(newPhysicalAddress); status = OperationStatus.RETRY_LATER; break; } @@ -142,56 +161,53 @@ bool TryAllocateRecordReadCache(ref PendingContext(ref PendingContext pendingContext, long logicalAddress, long physicalAddress, int allocatedSize) + [MethodImpl(MethodImplOptions.NoInlining)] // Do not inline, to keep CreateNewRecord* lean + void SaveAllocationForRetry(ref PendingContext pendingContext, long logicalAddress, long physicalAddress) { - ref var recordInfo = ref hlog.GetInfo(physicalAddress); + ref var recordInfo = ref LogRecord.GetInfoRef(physicalAddress); // TryAllocateRecord may stash this before WriteRecordInfo is called, leaving .PreviousAddress set to kInvalidAddress. // This is zero, and setting Invalid will result in recordInfo.IsNull being true, which will cause log-scan problems. // We don't need whatever .PreviousAddress was there, so set it to kTempInvalidAddress (which is nonzero). - recordInfo.PreviousAddress = Constants.kTempInvalidAddress; + recordInfo.PreviousAddress = kTempInvalidAddress; recordInfo.SetInvalid(); // Skip on log scan - // ExtraValueLength has been set by caller. - pendingContext.retryNewLogicalAddress = logicalAddress < hlogBase.HeadAddress ? Constants.kInvalidAddress : logicalAddress; + pendingContext.retryNewLogicalAddress = logicalAddress < hlogBase.HeadAddress ? kInvalidAddress : logicalAddress; } - // Do not inline, to keep TryAllocateRecord lean + [MethodImpl(MethodImplOptions.NoInlining)] // Do not inline, to keep TryAllocateRecord lean bool GetAllocationForRetry(TSessionFunctionsWrapper sessionFunctions, ref PendingContext pendingContext, long minAddress, - ref int allocatedSize, int newKeySize, out long newLogicalAddress, out long newPhysicalAddress) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + in RecordSizeInfo sizeInfo, out long newLogicalAddress, out long newPhysicalAddress) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // Use an earlier allocation from a failed operation, if possible. newLogicalAddress = pendingContext.retryNewLogicalAddress; pendingContext.retryNewLogicalAddress = 0; - if (newLogicalAddress < hlogBase.HeadAddress) + if (newLogicalAddress <= minAddress || newLogicalAddress < hlogBase.HeadAddress) { - // The record dropped below headAddress. If it needs DisposeRecord, it will be done on eviction. - newPhysicalAddress = 0; - return false; + // The record is too small or dropped below headAddress. + goto Fail; } - newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); - ref var recordInfo = ref hlog.GetInfo(newPhysicalAddress); - Debug.Assert(!recordInfo.IsNull(), "RecordInfo should not be IsNull"); - ref var recordValue = ref hlog.GetValue(newPhysicalAddress); - (int usedValueLength, int fullValueLength, int fullRecordLength) = GetRecordLengths(newPhysicalAddress, ref recordValue, ref recordInfo); + newPhysicalAddress = hlogBase.GetPhysicalAddress(newLogicalAddress); + var newLogRecord = new LogRecord(newPhysicalAddress); - // Dispose the record for either reuse or abandonment. - ClearExtraValueSpace(ref recordInfo, ref recordValue, usedValueLength, fullValueLength); - storeFunctions.DisposeRecord(ref hlog.GetKey(newPhysicalAddress), ref recordValue, DisposeReason.RevivificationFreeList, newKeySize); + if (newLogRecord.AllocatedSize < sizeInfo.AllocatedInlineRecordSize) + goto Fail; + return true; - if (newLogicalAddress <= minAddress || fullRecordLength < allocatedSize) + Fail: + // Only dispose if the record is still in memory. If the page was evicted (address < HeadAddress), + // the record's heap fields were already cleared by OnDispose before SaveAllocationForRetry, and + // the page memory may have been recycled — accessing it would crash or corrupt live records. + if (newLogicalAddress >= hlogBase.HeadAddress) { - // Can't reuse, so abandon it. - newPhysicalAddress = 0; - return false; + var logRecord = hlog.CreateLogRecord(newLogicalAddress); + OnDispose(ref logRecord, DisposeReason.CASAndRetryFailed); } - - allocatedSize = fullRecordLength; - return true; + newPhysicalAddress = 0; + return false; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ConditionalCopyToTail.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ConditionalCopyToTail.cs index 1c66b5bbb08..b709e119fec 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ConditionalCopyToTail.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ConditionalCopyToTail.cs @@ -6,33 +6,31 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Copy a record to the tail of the log after caller has verified it does not exist within a specified range. /// /// Callback functions. /// pending context created when the operation goes pending. - /// key of the record. - /// input passed through. - /// the value to insert - /// Location to store output computed from input and value. - /// user context corresponding to operation used during completion callback. + /// key of the record. /// Contains information about the call context, record metadata, and so on - /// The reason the CopyToTail is being done /// Whether to do IO if the search must go below HeadAddress. ReadFromImmutable, for example, /// is just an optimization to avoid future IOs, so if we need an IO here we just defer them to the next Read(). + /// Maximum address for determining liveness, records after this address are not considered when checking validity. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private OperationStatus ConditionalCopyToTail(TSessionFunctionsWrapper sessionFunctions, - ref PendingContext pendingContext, - ref TKey key, ref TInput input, ref TValue value, ref TOutput output, TContext userContext, - ref OperationStackContext stackCtx, WriteReason writeReason, bool wantIO = true, long maxAddress = long.MaxValue) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private OperationStatus ConditionalCopyToTail(TSessionFunctionsWrapper sessionFunctions, + ref PendingContext pendingContext, in TSourceLogRecord srcLogRecord, + ref OperationStackContext stackCtx, bool wantIO = true, long maxAddress = long.MaxValue) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - bool callerHasTransientLock = stackCtx.recSrc.HasTransientSLock; + var callerHasEphemeralLock = stackCtx.recSrc.HasEphemeralSLock; // We are called by one of ReadFromImmutable, CompactionConditionalCopyToTail, or ContinuePendingConditionalCopyToTail; // these have already searched to see if the record is present above minAddress, and stackCtx is set up for the first try. @@ -43,18 +41,17 @@ private OperationStatus ConditionalCopyToTail(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (callerHasEphemeralLock || TryEphemeralSLock(sessionFunctions, ref stackCtx, out var status)) { try { - RecordInfo dummyRecordInfo = default; // TryCopyToTail only needs this for readcache record invalidation. - status = TryCopyToTail(ref pendingContext, ref key, ref input, ref value, ref output, ref stackCtx, ref dummyRecordInfo, sessionFunctions, writeReason); + status = TryCopyToTail(in srcLogRecord, sessionFunctions, ref pendingContext, ref stackCtx); } finally { stackCtx.HandleNewRecordOnException(this); - if (!callerHasTransientLock) - TransientSUnlock(sessionFunctions, ref key, ref stackCtx); + if (!callerHasEphemeralLock) + EphemeralSUnlock(sessionFunctions, ref stackCtx); } } @@ -66,11 +63,12 @@ private OperationStatus ConditionalCopyToTail stackCtx2 = new(stackCtx.hei.hash); + OperationStackContext stackCtx2 = new(stackCtx.hei.hash); bool needIO; do { - if (TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx2, stackCtx.recSrc.LogicalAddress, minAddress, maxAddress, out status, out needIO)) + if (TryFindRecordInMainLogForConditionalOperation( + sessionFunctions, srcLogRecord, ref stackCtx2, stackCtx.recSrc.LogicalAddress, minAddress, maxAddress, out status, out needIO)) return OperationStatus.SUCCESS; } while (HandleImmediateNonPendingRetryStatus(status, sessionFunctions)); @@ -83,65 +81,58 @@ private OperationStatus ConditionalCopyToTail(TSessionFunctionsWrapper sessionFunctions, ref TKey key, ref TInput input, ref TValue value, - ref TOutput output, long currentAddress, long minAddress, long maxAddress = long.MaxValue) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status CompactionConditionalCopyToTail( + TSessionFunctionsWrapper sessionFunctions, in TSourceLogRecord srcLogRecord, long currentAddress, long minAddress, long maxAddress = long.MaxValue) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { Debug.Assert(epoch.ThisInstanceProtected(), "This is called only from Compaction so the epoch should be protected"); - PendingContext pendingContext = new(); + PendingContext pendingContext = new(storeFunctions.GetKeyHashCode64(srcLogRecord)); + // Plumb the source logical address so PostCopyToTail can name per-flush snapshot files + // (used by RangeIndex's BfTree compaction path). + pendingContext.originalAddress = currentAddress; - OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(ref key)); + OperationStackContext stackCtx = new(pendingContext.keyHash); OperationStatus status; bool needIO; do { - if (TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx, currentAddress, minAddress, maxAddress, out status, out needIO)) + if (TryFindRecordInMainLogForConditionalOperation(sessionFunctions, srcLogRecord, ref stackCtx, currentAddress, minAddress, maxAddress, out status, out needIO)) return Status.CreateFound(); } while (sessionFunctions.Store.HandleImmediateNonPendingRetryStatus(status, sessionFunctions)); if (needIO) - status = PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, ref key, ref input, ref value, ref output, default, - ref stackCtx, minAddress, maxAddress, WriteReason.Compaction); + status = PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, in srcLogRecord, ref stackCtx, minAddress, maxAddress); else - status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, ref key, ref input, ref value, ref output, default, ref stackCtx, WriteReason.Compaction, true, maxAddress); + status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, in srcLogRecord, ref stackCtx, maxAddress: maxAddress); return HandleOperationStatus(sessionFunctions.Ctx, ref pendingContext, status, out _); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus PrepareIOForConditionalOperation(TSessionFunctionsWrapper sessionFunctions, - ref PendingContext pendingContext, - ref TKey key, ref TInput input, ref TValue value, ref TOutput output, TContext userContext, - ref OperationStackContext stackCtx, long minAddress, long maxAddress, WriteReason writeReason, - OperationType opType = OperationType.CONDITIONAL_INSERT) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal OperationStatus PrepareIOForConditionalOperation( + TSessionFunctionsWrapper sessionFunctions, + ref PendingContext pendingContext, in TSourceLogRecord srcLogRecord, + ref OperationStackContext stackCtx, long minAddress, long maxAddress, OperationType opType = OperationType.CONDITIONAL_INSERT) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { + pendingContext.CopyKey(srcLogRecord, hlogBase.bufferPool, sessionFunctions); pendingContext.type = opType; pendingContext.minAddress = minAddress; pendingContext.maxAddress = maxAddress; - pendingContext.writeReason = writeReason; - pendingContext.InitialEntryAddress = Constants.kInvalidAddress; - pendingContext.InitialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; - - if (!pendingContext.NoKey && pendingContext.key == default) // If this is true, we don't have a valid key - pendingContext.key = hlog.GetKeyContainer(ref key); - if (pendingContext.input == default) - pendingContext.input = sessionFunctions.GetHeapContainer(ref input); - if (pendingContext.value == default) - pendingContext.value = hlog.GetValueContainer(ref value); - - pendingContext.output = output; - sessionFunctions.ConvertOutputToHeap(ref input, ref pendingContext.output); - - pendingContext.userContext = userContext; + pendingContext.initialEntryAddress = kInvalidAddress; + pendingContext.initialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + // Transfer the log record to the pending context. We do not want to dispose memory log records; those objects are still alive in the log. + if (!pendingContext.diskLogRecord.IsSet) + pendingContext.CopyFrom(in srcLogRecord, hlogBase.bufferPool, hlogBase.transientObjectIdMap); return OperationStatus.RECORD_ON_DISK; } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContainsKeyInMemory.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContainsKeyInMemory.cs index d8c0c9c6876..798d41039de 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContainsKeyInMemory.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ContainsKeyInMemory.cs @@ -5,16 +5,20 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status InternalContainsKeyInMemory( - ref TKey key, TSessionFunctionsWrapper sessionFunctions, out long logicalAddress, long fromAddress = -1) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status InternalContainsKeyInMemory( + TKey key, TSessionFunctionsWrapper sessionFunctions, out long logicalAddress, long fromAddress = -1) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(ref key)); + OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(key)); if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); @@ -29,7 +33,7 @@ internal Status InternalContainsKeyInMemory : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Continue a pending read operation. Computes 'output' from 'input' and value corresponding to 'key' @@ -28,137 +28,148 @@ public unsafe partial class TsavoriteKV /// /// - internal OperationStatus ContinuePendingRead(AsyncIOContext request, + internal OperationStatus ContinuePendingRead(AsyncIOContext request, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - ref RecordInfo srcRecordInfo = ref hlog.GetInfoFromBytePointer(request.record.GetValidPointer()); - srcRecordInfo.ClearBitsForDiskImages(); + if (request.logicalAddress < hlogBase.BeginAddress || request.logicalAddress < pendingContext.minAddress) + goto NotFound; - if (request.logicalAddress >= hlogBase.BeginAddress && request.logicalAddress >= pendingContext.minAddress) - { - SpinWaitUntilClosed(request.logicalAddress); + if (pendingContext.IsReadAtAddress && !pendingContext.IsNoKey && !storeFunctions.KeysEqual(pendingContext.DiskLogRecord, pendingContext.requestKey)) + goto NotFound; - // If NoKey, we do not have the key in the initial call and must use the key from the satisfied request. - ref TKey key = ref pendingContext.NoKey ? ref hlog.GetContextRecordKey(ref request) : ref pendingContext.key.Get(); - OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(ref key)); + SpinWaitUntilClosed(request.logicalAddress); + OperationStackContext stackCtx = new(storeFunctions.GetKeyHashCode64(pendingContext.DiskLogRecord)); - while (true) + while (true) + { + if (!FindTagAndTryEphemeralSLock(sessionFunctions, ref stackCtx, out var status)) { - if (!FindTagAndTryTransientSLock(sessionFunctions, ref key, ref stackCtx, out var status)) - { - Debug.Assert(status != OperationStatus.NOTFOUND, "Expected to FindTag in InternalContinuePendingRead"); - if (HandleImmediateRetryStatus(status, sessionFunctions, ref pendingContext)) - continue; - return status; - } + Debug.Assert(status != OperationStatus.NOTFOUND, "Expected to FindTag in InternalContinuePendingRead"); + if (HandleImmediateRetryStatus(status, sessionFunctions, ref pendingContext)) + continue; + return status; + } - stackCtx.SetRecordSourceToHashEntry(hlogBase); + stackCtx.SetRecordSourceToHashEntry(hlogBase); - try + try + { + // During the pending operation, a record for the key may have been added to the log or readcache. Don't look for this if we are reading at address (rather than key). + LogRecord memoryRecord = default; + if (!pendingContext.IsReadAtAddress) { - ref var value = ref hlog.GetContextRecordValue(ref request); - - // During the pending operation, a record for the key may have been added to the log or readcache. Don't look for this if we are reading at address (rather than key). - if (!pendingContext.IsReadAtAddress) + if (TryFindRecordInMemory(pendingContext.DiskLogRecord, ref stackCtx, ref pendingContext)) { - if (TryFindRecordInMemory(ref key, ref stackCtx, ref pendingContext)) - { - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + memoryRecord = stackCtx.recSrc.CreateLogRecord(); + if (memoryRecord.Info.Tombstone) + goto NotFound; - // V threads cannot access V+1 records. Use the latest logical address rather than the traced address (logicalAddress) per comments in AcquireCPRLatchRMW. - if (sessionFunctions.Ctx.phase == Phase.PREPARE && IsEntryVersionNew(ref stackCtx.hei.entry)) - return OperationStatus.CPR_SHIFT_DETECTED; // Pivot thread; retry - value = ref stackCtx.recSrc.GetValue(); - } - else + // V threads cannot access V+1 records. Use the latest logical address rather than the traced address (logicalAddress) per comments in AcquireCPRLatchRMW. + if (sessionFunctions.Ctx.phase == Phase.PREPARE && IsEntryVersionNew(ref stackCtx.hei.entry)) + return OperationStatus.CPR_SHIFT_DETECTED; // Pivot thread; retry + } + else + { + // We didn't find a record for the key in memory, but if recSrc.LogicalAddress (which is the .PreviousAddress of the lowest record + // above InitialLatestLogicalAddress we could reach) is > InitialLatestLogicalAddress, then it means InitialLatestLogicalAddress is + // now below HeadAddress and there is at least one record below HeadAddress but above InitialLatestLogicalAddress. Reissue the Read(), + // using the LogicalAddress we just found as minAddress. We will either find an in-memory version of the key that was added after the + // TryFindRecordInMemory we just did, or do IO and find the record we just found or one above it. Read() updates InitialLatestLogicalAddress, + // so if we do IO, the next time we come to CompletePendingRead we will only search for a newer version of the key in any records added + // after our just-completed TryFindRecordInMemory. + if (stackCtx.recSrc.LogicalAddress > pendingContext.initialLatestLogicalAddress + && (!pendingContext.HasMinAddress || stackCtx.recSrc.LogicalAddress >= pendingContext.minAddress)) { - // We didn't find a record for the key in memory, but if recSrc.LogicalAddress (which is the .PreviousAddress of the lowest record - // above InitialLatestLogicalAddress we could reach) is > InitialLatestLogicalAddress, then it means InitialLatestLogicalAddress is - // now below HeadAddress and there is at least one record below HeadAddress but above InitialLatestLogicalAddress. Reissue the Read(), - // using the LogicalAddress we just found as minAddress. We will either find an in-memory version of the key that was added after the - // TryFindRecordInMemory we just did, or do IO and find the record we just found or one above it. Read() updates InitialLatestLogicalAddress, - // so if we do IO, the next time we come to CompletePendingRead we will only search for a newer version of the key in any records added - // after our just-completed TryFindRecordInMemory. - if (stackCtx.recSrc.LogicalAddress > pendingContext.InitialLatestLogicalAddress - && (!pendingContext.HasMinAddress || stackCtx.recSrc.LogicalAddress >= pendingContext.minAddress)) + OperationStatus internalStatus; + do { - OperationStatus internalStatus; - do - { - internalStatus = InternalRead(ref key, pendingContext.keyHash, ref pendingContext.input.Get(), ref pendingContext.output, - pendingContext.userContext, ref pendingContext, sessionFunctions); - } - while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pendingContext)); - return internalStatus; + internalStatus = InternalRead(pendingContext.DiskLogRecord, pendingContext.keyHash, ref pendingContext.input.Get(), ref pendingContext.output, + pendingContext.userContext, ref pendingContext, sessionFunctions); } + while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pendingContext)); + return internalStatus; } } + } - if (srcRecordInfo.Tombstone) - goto NotFound; - - ReadInfo readInfo = new() - { - Version = sessionFunctions.Ctx.version, - Address = request.logicalAddress, - IsFromPending = pendingContext.type != OperationType.NONE, - }; - readInfo.SetRecordInfo(ref srcRecordInfo); + ReadInfo readInfo = new() + { + Version = sessionFunctions.Ctx.version, + Address = request.logicalAddress, + IsFromPending = pendingContext.type != OperationType.NONE, + }; + pendingContext.logicalAddress = request.logicalAddress; - bool success = false; - if (stackCtx.recSrc.HasMainLogSrc && stackCtx.recSrc.LogicalAddress >= hlogBase.ReadOnlyAddress) - { - // If this succeeds, we don't need to copy to tail or readcache, so return success. - if (sessionFunctions.ConcurrentReader(ref key, ref pendingContext.input.Get(), ref value, ref pendingContext.output, ref readInfo, ref srcRecordInfo)) - return OperationStatus.SUCCESS; - } - else + if (!memoryRecord.IsSet && pendingContext.diskLogRecord.Info.Tombstone) + { + if (pendingContext.IsReadAtAddress) { - // This may be in the immutable region, which means it may be an updated version of the record. - success = sessionFunctions.SingleReader(ref key, ref pendingContext.input.Get(), ref value, ref pendingContext.output, ref readInfo); + // Be consistent with InternalReadAtAddress and return the tombstoned record we retrieved from disk. + _ = sessionFunctions.Reader(in pendingContext.diskLogRecord, ref pendingContext.input.Get(), ref pendingContext.output, ref readInfo); } + goto NotFound; + } - if (!success) + var success = false; + if (stackCtx.recSrc.HasMainLogSrc && stackCtx.recSrc.LogicalAddress >= hlogBase.ReadOnlyAddress) + { + // If this succeeds, we don't need to copy to tail or readcache, so return success. + if (sessionFunctions.Reader(in memoryRecord, ref pendingContext.input.Get(), ref pendingContext.output, ref readInfo)) { - pendingContext.recordInfo = srcRecordInfo; - if (readInfo.Action == ReadAction.CancelOperation) - return OperationStatus.CANCELED; - if (readInfo.Action == ReadAction.Expire) - return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.Expired); - goto NotFound; + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + return OperationStatus.SUCCESS; } + } + else if (memoryRecord.IsSet) + { + // This may be in the immutable region, which means it may be an updated version of the record. + success = sessionFunctions.Reader(in memoryRecord, ref pendingContext.input.Get(), ref pendingContext.output, ref readInfo); + } + else // Not found in memory so return the disk copy. + success = sessionFunctions.Reader(in pendingContext.diskLogRecord, ref pendingContext.input.Get(), ref pendingContext.output, ref readInfo); - // See if we are copying to read cache or tail of log. If we are copying to readcache but already found the record in the readcache, we're done. - if (pendingContext.readCopyOptions.CopyFrom != ReadCopyFrom.None) - { - if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.MainLog) - status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, ref key, ref pendingContext.input.Get(), ref value, ref pendingContext.output, - pendingContext.userContext, ref stackCtx, WriteReason.CopyToTail); - else if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.ReadCache && !stackCtx.recSrc.HasReadCacheSrc - && TryCopyToReadCache(sessionFunctions, ref pendingContext, ref key, ref pendingContext.input.Get(), ref value, ref stackCtx)) - status |= OperationStatus.COPIED_RECORD_TO_READ_CACHE; - } - else + if (!success) + { + if (readInfo.Action == ReadAction.CancelOperation) + return OperationStatus.CANCELED; + if (readInfo.Action == ReadAction.Expire) + return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.Expired); + if (readInfo.Action == ReadAction.WrongType) + return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.WrongType); + goto NotFound; + } + + // See if we are copying to read cache or tail of log. If we already found the record in memory, we're done. + if (pendingContext.readCopyOptions.CopyFrom != ReadCopyFrom.None && !memoryRecord.IsSet) + { + if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.MainLog) { - pendingContext.recordInfo = srcRecordInfo; - return OperationStatus.SUCCESS; + // Plumb source logical address so PostCopyToTail can name per-flush snapshot files. + pendingContext.originalAddress = request.logicalAddress; + status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, in pendingContext.diskLogRecord, ref stackCtx); } + else if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.ReadCache && !stackCtx.recSrc.HasReadCacheSrc + && TryCopyToReadCache(in pendingContext.diskLogRecord, sessionFunctions, ref pendingContext, ref stackCtx)) + status |= OperationStatus.COPIED_RECORD_TO_READ_CACHE; } - finally + else { - stackCtx.HandleNewRecordOnException(this); - TransientSUnlock(sessionFunctions, ref key, ref stackCtx); + return OperationStatus.SUCCESS; } + } + finally + { + stackCtx.HandleNewRecordOnException(this); + EphemeralSUnlock(sessionFunctions, ref stackCtx); + } - // Must do this *after* Unlocking. Status was set by InternalTryCopyToTail. - if (!HandleImmediateRetryStatus(status, sessionFunctions, ref pendingContext)) - return status; - } // end while (true) - } + // Must do this *after* Unlocking. Status was set by InternalTryCopyToTail. + if (!HandleImmediateRetryStatus(status, sessionFunctions, ref pendingContext)) + return status; + } // end while (true) NotFound: - pendingContext.recordInfo = srcRecordInfo; return OperationStatus.NOTFOUND; } @@ -184,18 +195,13 @@ internal OperationStatus ContinuePendingRead /// /// - internal OperationStatus ContinuePendingRMW(AsyncIOContext request, + internal OperationStatus ContinuePendingRMW(AsyncIOContext request, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - ref TKey key = ref pendingContext.key.Get(); - - SpinWaitUntilClosed(request.logicalAddress); - - byte* recordPointer = request.record.GetValidPointer(); - var requestRecordInfo = hlog.GetInfoFromBytePointer(recordPointer); // Not ref, as we don't want to write into request.record - ref var srcRecordInfo = ref requestRecordInfo; - srcRecordInfo.ClearBitsForDiskImages(); + var keyFound = request.logicalAddress >= hlogBase.BeginAddress && request.logicalAddress >= pendingContext.minAddress; + if (keyFound) + SpinWaitUntilClosed(request.logicalAddress); RMWInfo rmwInfo = new() { @@ -207,15 +213,15 @@ internal OperationStatus ContinuePendingRMW stackCtx = new(pendingContext.keyHash); - if (!FindOrCreateTagAndTryTransientXLock(sessionFunctions, ref key, ref stackCtx, out status)) + OperationStackContext stackCtx = new(pendingContext.keyHash); + if (!FindOrCreateTagAndTryEphemeralXLock(sessionFunctions, ref stackCtx, out status)) goto CheckRetry; try { // During the pending operation a record for the key may have been added to the log. If so, break and go through the full InternalRMW sequence; // the record in 'request' is stale. We only lock for tag-chain stability during search. - if (TryFindRecordForPendingOperation(ref key, ref stackCtx, hlogBase.HeadAddress, out status, ref pendingContext)) + if (TryFindRecordForPendingOperation(pendingContext.requestKey, ref stackCtx, out status, ref pendingContext)) { if (status != OperationStatus.SUCCESS) goto CheckRetry; @@ -225,30 +231,27 @@ internal OperationStatus ContinuePendingRMW InitialLatestLogicalAddress, then it means InitialLatestLogicalAddress is // now below HeadAddress and there is at least one record below HeadAddress but above InitialLatestLogicalAddress. We must do InternalRMW. - if (stackCtx.recSrc.LogicalAddress > pendingContext.InitialLatestLogicalAddress) + if (stackCtx.recSrc.LogicalAddress > pendingContext.initialLatestLogicalAddress) { - Debug.Assert(pendingContext.InitialLatestLogicalAddress < hlogBase.HeadAddress, "Failed to search all in-memory records"); + Debug.Assert(pendingContext.initialLatestLogicalAddress < hlogBase.HeadAddress, "Failed to search all in-memory records"); break; } // Here, the input data for 'doingCU' is the from the request, so populate the RecordSource copy from that, preserving LowestReadCache*. stackCtx.recSrc.LogicalAddress = request.logicalAddress; - stackCtx.recSrc.PhysicalAddress = (long)recordPointer; - - status = CreateNewRecordRMW(ref key, ref pendingContext.input.Get(), ref hlog.GetContextRecordValue(ref request), ref pendingContext.output, - ref pendingContext, sessionFunctions, ref stackCtx, ref srcRecordInfo, - doingCU: request.logicalAddress >= hlogBase.BeginAddress && !srcRecordInfo.Tombstone, ref rmwInfo); + status = CreateNewRecordRMW(pendingContext.requestKey, in pendingContext.diskLogRecord, ref pendingContext.input.Get(), ref pendingContext.output, + ref pendingContext, sessionFunctions, ref stackCtx, doingCU: keyFound && !pendingContext.diskLogRecord.Info.Tombstone, ref rmwInfo); } finally { stackCtx.HandleNewRecordOnException(this); try { - sessionFunctions.PostRMWOperation(ref key, ref pendingContext.input.Get(), ref rmwInfo, epoch); + sessionFunctions.PostRMWOperation(pendingContext.requestKey, ref pendingContext.input.Get(), ref rmwInfo, epoch); } finally { - TransientXUnlock(sessionFunctions, ref key, ref stackCtx); + EphemeralXUnlock(sessionFunctions, ref stackCtx); } } @@ -261,7 +264,7 @@ internal OperationStatus ContinuePendingRMW /// /// - internal OperationStatus ContinuePendingConditionalCopyToTail(AsyncIOContext request, + internal OperationStatus ContinuePendingConditionalCopyToTail(AsyncIOContext request, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // If the key was found at or above minAddress, do nothing. // If we're here we know the key matches because AllocatorBase.AsyncGetFromDiskCallback skips colliding keys by following the .PreviousAddress chain. @@ -299,24 +302,26 @@ internal OperationStatus ContinuePendingConditionalCopyToTail stackCtx = new(storeFunctions.GetKeyHashCode64(ref key)); + OperationStackContext stackCtx = new(pendingContext.keyHash); // See if the record was added above the highest address we checked before issuing the IO. - var minAddress = pendingContext.InitialLatestLogicalAddress + 1; + var minAddress = pendingContext.initialLatestLogicalAddress + 1; OperationStatus internalStatus; do { - if (TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx, currentAddress: request.logicalAddress, minAddress, pendingContext.maxAddress, out internalStatus, out bool needIO)) + if (TryFindRecordInMainLogForConditionalOperation(sessionFunctions, pendingContext.requestKey, ref stackCtx, + currentAddress: request.logicalAddress, minAddress, pendingContext.maxAddress, out internalStatus, out var needIO)) return OperationStatus.SUCCESS; if (!OperationStatusUtils.IsRetry(internalStatus)) { + // Plumb source logical address so PostCopyToTail can name per-flush snapshot files. + // request.logicalAddress is the actual disk-resolved source (AsyncGetFromDiskCallback + // walks the chain to skip colliding keys). + pendingContext.originalAddress = request.logicalAddress; // HeadAddress may have risen above minAddress; if so, we need IO. internalStatus = needIO - ? PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, ref key, ref pendingContext.input.Get(), ref pendingContext.value.Get(), - ref pendingContext.output, pendingContext.userContext, ref stackCtx, minAddress, pendingContext.maxAddress, WriteReason.Compaction) - : ConditionalCopyToTail(sessionFunctions, ref pendingContext, ref key, ref pendingContext.input.Get(), ref pendingContext.value.Get(), - ref pendingContext.output, pendingContext.userContext, ref stackCtx, pendingContext.writeReason); + ? PrepareIOForConditionalOperation(sessionFunctions, ref pendingContext, in pendingContext.diskLogRecord, ref stackCtx, minAddress, pendingContext.maxAddress) + : ConditionalCopyToTail(sessionFunctions, ref pendingContext, in pendingContext.diskLogRecord, ref stackCtx); } } while (sessionFunctions.Store.HandleImmediateNonPendingRetryStatus(internalStatus, sessionFunctions)); @@ -346,9 +351,9 @@ internal OperationStatus ContinuePendingConditionalCopyToTail /// /// - internal OperationStatus ContinuePendingConditionalScanPush(AsyncIOContext request, + internal OperationStatus ContinuePendingConditionalScanPush(AsyncIOContext request, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // If the key was found at or above minAddress, do nothing; we'll push it when we get to it. If we flagged the iteration to stop, do nothing. // If we're here we know the key matches because AllocatorBase.AsyncGetFromDiskCallback skips colliding keys by following the .PreviousAddress chain. @@ -358,8 +363,9 @@ internal OperationStatus ContinuePendingConditionalScanPush(sessionFunctions, pendingContext.scanCursorState, pendingContext.recordInfo, ref pendingContext.key.Get(), ref pendingContext.value.Get(), - currentAddress: request.logicalAddress, minAddress: pendingContext.InitialLatestLogicalAddress + 1, maxAddress: pendingContext.maxAddress); + _ = hlogBase.ConditionalScanPush(sessionFunctions, pendingContext.scanCursorState, + in pendingContext.diskLogRecord, originalAddress: pendingContext.originalAddress, currentAddress: request.logicalAddress, + minAddress: pendingContext.initialLatestLogicalAddress + 1, maxAddress: pendingContext.maxAddress); // ConditionalScanPush has already called HandleOperationStatus, so return SUCCESS here. return OperationStatus.SUCCESS; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/EpochOperations.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/EpochOperations.cs index aa79edbded9..b5db8780133 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/EpochOperations.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/EpochOperations.cs @@ -7,16 +7,16 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SynchronizeEpoch( TsavoriteExecutionContext sessionCtx, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var version = sessionCtx.version; Debug.Assert(sessionCtx.version == version, $"sessionCtx.version ({sessionCtx.version}) should == version ({version})"); @@ -38,7 +38,7 @@ void SpinWaitUntilClosed(long address) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - void SpinWaitUntilRecordIsClosed(long logicalAddress, AllocatorBase log) + void SpinWaitUntilRecordIsClosed(long logicalAddress, AllocatorBase log) { Debug.Assert(logicalAddress < log.HeadAddress, "SpinWaitUntilRecordIsClosed should not be called for addresses above HeadAddress"); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/FindRecord.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/FindRecord.cs index 37cfc7b7710..7e4e89e3103 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/FindRecord.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/FindRecord.cs @@ -3,68 +3,43 @@ using System.Diagnostics; using System.Runtime.CompilerServices; -using static Tsavorite.core.Utility; namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryFindRecordInMemory(ref TKey key, ref OperationStackContext stackCtx, long minAddress, bool stopAtHeadAddress = true) - { - if (UseReadCache && FindInReadCache(ref key, ref stackCtx, minAddress: Constants.kInvalidAddress)) - return true; - if (minAddress < hlogBase.HeadAddress && stopAtHeadAddress) - minAddress = hlogBase.HeadAddress; - return TryFindRecordInMainLog(ref key, ref stackCtx, minAddress: minAddress); - } + using static LogAddress; + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryFindRecordInMemory(ref TKey key, ref OperationStackContext stackCtx, + private bool TryFindRecordInMemory(TKey key, ref OperationStackContext stackCtx, ref PendingContext pendingContext) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // Add 1 to the pendingContext minAddresses because we don't want an inclusive search; we're looking to see if it was added *after*. if (UseReadCache) { - var minRC = IsReadCache(pendingContext.InitialEntryAddress) ? pendingContext.InitialEntryAddress + 1 : Constants.kInvalidAddress; - if (FindInReadCache(ref key, ref stackCtx, minAddress: minRC)) + var minRC = IsReadCache(pendingContext.initialEntryAddress) ? pendingContext.initialEntryAddress + 1 : kInvalidAddress; + if (FindInReadCache(key, ref stackCtx, minAddress: minRC)) return true; } - var minLog = pendingContext.InitialLatestLogicalAddress < hlogBase.HeadAddress ? hlogBase.HeadAddress : pendingContext.InitialLatestLogicalAddress + 1; - return TryFindRecordInMainLog(ref key, ref stackCtx, minAddress: minLog); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryFindRecordInMainLog(ref TKey key, ref OperationStackContext stackCtx, long minAddress) - { - Debug.Assert(!stackCtx.recSrc.HasInMemorySrc, "Should not have found record before this call"); - if (stackCtx.recSrc.LogicalAddress >= minAddress) - { - stackCtx.recSrc.SetPhysicalAddress(); - TraceBackForKeyMatch(ref key, ref stackCtx.recSrc, minAddress); - } - return stackCtx.recSrc.HasInMemorySrc; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryFindRecordInMainLog(ref TKey key, ref OperationStackContext stackCtx, long minAddress, long maxAddress) - { - Debug.Assert(!stackCtx.recSrc.HasInMemorySrc, "Should not have found record before this call"); - if (stackCtx.recSrc.LogicalAddress >= minAddress) - { - stackCtx.recSrc.SetPhysicalAddress(); - TraceBackForKeyMatch(ref key, ref stackCtx.recSrc, minAddress, maxAddress); - } - return stackCtx.recSrc.HasInMemorySrc; + var minLog = pendingContext.initialLatestLogicalAddress < hlogBase.HeadAddress ? hlogBase.HeadAddress : pendingContext.initialLatestLogicalAddress + 1; + return TraceBackForKeyMatch(key, ref stackCtx.recSrc, minAddress: minLog); } [MethodImpl(MethodImplOptions.AggressiveInlining)] // Return true if the record is found in the log, else false and an indication of whether we need to do IO to continue the search - internal bool TryFindRecordInMainLogForConditionalOperation(TSessionFunctionsWrapper sessionFunctions, - ref TKey key, ref OperationStackContext stackCtx, long currentAddress, long minAddress, long maxAddress, out OperationStatus internalStatus, out bool needIO) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool TryFindRecordInMainLogForConditionalOperation(TSessionFunctionsWrapper sessionFunctions, + TKey key, ref OperationStackContext stackCtx, long currentAddress, long minAddress, long maxAddress, out OperationStatus internalStatus, out bool needIO) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { if (!FindTag(ref stackCtx.hei)) { @@ -92,12 +67,12 @@ internal bool TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx, out internalStatus)) + // The EphemeralSLock here is necessary only for the tag chain to avoid record elision/revivification during traceback. + if (!TryEphemeralSLock(sessionFunctions, ref stackCtx, out internalStatus)) return needIO = false; } - - stackCtx.SetRecordSourceToHashEntry(hlogBase); + else + stackCtx.SetRecordSourceToHashEntry(hlogBase); try { @@ -106,7 +81,7 @@ internal bool TryFindRecordInMainLogForConditionalOperation= minAddress && stackCtx.recSrc.LogicalAddress < hlogBase.HeadAddress && stackCtx.recSrc.LogicalAddress >= hlogBase.BeginAddress; @@ -114,7 +89,7 @@ internal bool TryFindRecordInMainLogForConditionalOperation(sessionFunctions, ref key, ref stackCtx); + EphemeralSUnlock(sessionFunctions, ref stackCtx); } } @@ -123,112 +98,121 @@ internal bool TryFindRecordInMainLogForConditionalOperation !recordInfo.Invalid || recordInfo.IsSealed; [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TraceBackForKeyMatch(ref TKey key, ref RecordSource recSrc, long minAddress) + private bool TraceBackForKeyMatch(TKey key, ref RecordSource recSrc, long minAddress) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - // PhysicalAddress must already be populated by callers. - ref var recordInfo = ref recSrc.GetInfo(); - if (IsValidTracebackRecord(recordInfo) && storeFunctions.KeysEqual(ref key, ref recSrc.GetKey())) + Debug.Assert(!recSrc.HasInMemorySrc, "Should not have found record before this call"); + if (recSrc.LogicalAddress >= minAddress) { - recSrc.SetHasMainLogSrc(); - return true; - } + _ = recSrc.SetPhysicalAddress(); + var logRecord = recSrc.CreateLogRecord(); + if (IsValidTracebackRecord(logRecord.Info) && storeFunctions.KeysEqual(key, logRecord)) + { + recSrc.SetHasMainLogSrc(); + return true; + } - recSrc.LogicalAddress = recordInfo.PreviousAddress; - if (TraceBackForKeyMatch(ref key, recSrc.LogicalAddress, minAddress, out recSrc.LogicalAddress, out recSrc.PhysicalAddress)) - { - recSrc.SetHasMainLogSrc(); - return true; + if (TraceBackForKeyMatch(key, logRecord.Info.PreviousAddress, minAddress, out recSrc.LogicalAddress, out recSrc.PhysicalAddress)) + { + recSrc.SetHasMainLogSrc(); + return true; + } } return false; } - // Overload with maxAddress to avoid the extra condition - TODO: check that this duplication saves on IL/perf + // Overload with maxAddress to avoid the extra if condition - TODO: check that this duplication saves on IL/perf [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TraceBackForKeyMatch(ref TKey key, ref RecordSource recSrc, long minAddress, long maxAddress) + private bool TraceBackForKeyMatch(TKey key, ref RecordSource recSrc, long minAddress, long maxAddress) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - // PhysicalAddress must already be populated by callers. - ref var recordInfo = ref recSrc.GetInfo(); - if (IsValidTracebackRecord(recordInfo) && recSrc.LogicalAddress < maxAddress && storeFunctions.KeysEqual(ref key, ref recSrc.GetKey())) + Debug.Assert(!recSrc.HasInMemorySrc, "Should not have found record before this call"); + if (recSrc.LogicalAddress >= minAddress) { - recSrc.SetHasMainLogSrc(); - return true; - } + _ = recSrc.SetPhysicalAddress(); + var logRecord = recSrc.CreateLogRecord(); + if (IsValidTracebackRecord(logRecord.Info) && recSrc.LogicalAddress < maxAddress && storeFunctions.KeysEqual(key, logRecord)) + { + recSrc.SetHasMainLogSrc(); + return true; + } - recSrc.LogicalAddress = recordInfo.PreviousAddress; - if (TraceBackForKeyMatch(ref key, recSrc.LogicalAddress, minAddress, maxAddress, out recSrc.LogicalAddress, out recSrc.PhysicalAddress)) - { - recSrc.SetHasMainLogSrc(); - return true; + if (TraceBackForKeyMatch(key, logRecord.Info.PreviousAddress, minAddress, maxAddress, out recSrc.LogicalAddress, out recSrc.PhysicalAddress)) + { + recSrc.SetHasMainLogSrc(); + return true; + } } return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TraceBackForKeyMatch(ref TKey key, long fromLogicalAddress, long minAddress, out long foundLogicalAddress, out long foundPhysicalAddress) + private bool TraceBackForKeyMatch(TKey key, long fromLogicalAddress, long minAddress, out long foundLogicalAddress, out long foundPhysicalAddress) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // This overload is called when the record at the "current" logical address does not match 'key'; fromLogicalAddress is its .PreviousAddress. foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minAddress) { - foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); + var logRecord = hlog.CreateLogRecord(foundLogicalAddress); + foundPhysicalAddress = logRecord.physicalAddress; - ref var recordInfo = ref hlog.GetInfo(foundPhysicalAddress); - if (IsValidTracebackRecord(recordInfo) && storeFunctions.KeysEqual(ref key, ref hlog.GetKey(foundPhysicalAddress))) + if (IsValidTracebackRecord(logRecord.Info) && storeFunctions.KeysEqual(key, logRecord)) return true; - foundLogicalAddress = recordInfo.PreviousAddress; + foundLogicalAddress = logRecord.Info.PreviousAddress; } - foundPhysicalAddress = Constants.kInvalidAddress; + foundPhysicalAddress = kInvalidAddress; return false; } - // Overload with maxAddress to avoid the extra condition - TODO: check that this duplication saves on IL/perf + // Overload with maxAddress to avoid the extra if condition - TODO: check that this duplication saves on IL/perf [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TraceBackForKeyMatch(ref TKey key, long fromLogicalAddress, long minAddress, long maxAddress, out long foundLogicalAddress, out long foundPhysicalAddress) + private bool TraceBackForKeyMatch(TKey key, long fromLogicalAddress, long minAddress, long maxAddress, out long foundLogicalAddress, out long foundPhysicalAddress) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // This overload is called when the record at the "current" logical address does not match 'key'; fromLogicalAddress is its .PreviousAddress. foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minAddress) { - foundPhysicalAddress = hlog.GetPhysicalAddress(foundLogicalAddress); + var logRecord = hlog.CreateLogRecord(foundLogicalAddress); + foundPhysicalAddress = logRecord.physicalAddress; - ref var recordInfo = ref hlog.GetInfo(foundPhysicalAddress); - if (IsValidTracebackRecord(recordInfo) && foundLogicalAddress < maxAddress && storeFunctions.KeysEqual(ref key, ref hlog.GetKey(foundPhysicalAddress))) + if (IsValidTracebackRecord(logRecord.Info) && foundLogicalAddress < maxAddress && storeFunctions.KeysEqual(key, logRecord)) return true; - foundLogicalAddress = recordInfo.PreviousAddress; + foundLogicalAddress = logRecord.Info.PreviousAddress; } - foundPhysicalAddress = Constants.kInvalidAddress; + foundPhysicalAddress = kInvalidAddress; return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryFindRecordForUpdate(ref TKey key, ref OperationStackContext stackCtx, long minAddress, out OperationStatus internalStatus) + private bool TryFindRecordForUpdate(TKey key, ref OperationStackContext stackCtx, long minAddress, out OperationStatus internalStatus) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // This routine returns true if we should proceed with the InternalXxx operation (whether the record was found or not), // else false (including false if we need a RETRY). If it returns true with recSrc.HasInMemorySrc, caller must set srcRecordInfo. // We are not here from Read() so have not processed readcache; search that as well as the in-memory log. - if (TryFindRecordInMemory(ref key, ref stackCtx, minAddress)) - { - if (stackCtx.recSrc.GetInfo().IsClosed) - { - internalStatus = OperationStatus.RETRY_LATER; - return false; - } - } - internalStatus = OperationStatus.SUCCESS; - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryFindRecordForRead(ref TKey key, ref OperationStackContext stackCtx, long minAddress, out OperationStatus internalStatus) - { - // This routine returns true if we should proceed with the InternalXxx operation (whether the record was found or not), - // else false (including false if we need a RETRY). If it returns true with recSrc.HasInMemorySrc, caller must set srcRecordInfo. - - // We are here for Read() so we have already processed readcache and are just here for the traceback in the main log. - if (TryFindRecordInMainLog(ref key, ref stackCtx, minAddress)) + // minAddress is either HeadAddress or ReadOnlyAddress for the main log. + if ((UseReadCache && FindInReadCache(key, ref stackCtx, minAddress: kInvalidAddress)) + || TraceBackForKeyMatch(key, ref stackCtx.recSrc, minAddress: minAddress)) { if (stackCtx.recSrc.GetInfo().IsClosed) { @@ -241,13 +225,17 @@ private bool TryFindRecordForRead(ref TKey key, ref OperationStackContext(ref TKey key, ref OperationStackContext stackCtx, long minAddress, out OperationStatus internalStatus, + private bool TryFindRecordForPendingOperation(TKey key, ref OperationStackContext stackCtx, out OperationStatus internalStatus, ref PendingContext pendingContext) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // This routine returns true if we find the key, else false. internalStatus = OperationStatus.SUCCESS; - if (!TryFindRecordInMemory(ref key, ref stackCtx, ref pendingContext)) + if (!TryFindRecordInMemory(key, ref stackCtx, ref pendingContext)) return false; if (stackCtx.recSrc.GetInfo().IsClosed) internalStatus = OperationStatus.RETRY_LATER; @@ -257,14 +245,18 @@ private bool TryFindRecordForPendingOperation(ref TKe } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryFindRecordInMainLogForPendingOperation(ref TKey key, ref OperationStackContext stackCtx, long minAddress, long maxAddress, out OperationStatus internalStatus) + private bool TryFindRecordInMainLogForPendingOperation(TKey key, ref OperationStackContext stackCtx, long minAddress, long maxAddress, out OperationStatus internalStatus) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { // This overload is called when we do not have a PendingContext to get minAddress from, and we've skipped the readcache if present. // This routine returns true if we find the key, else false. internalStatus = OperationStatus.SUCCESS; - if (!TryFindRecordInMainLog(ref key, ref stackCtx, minAddress, maxAddress)) + if (!TraceBackForKeyMatch(key, ref stackCtx.recSrc, minAddress, maxAddress)) return false; if (stackCtx.recSrc.GetInfo().IsClosed) internalStatus = OperationStatus.RETRY_LATER; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HandleOperationStatus.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HandleOperationStatus.cs index fafb4231afb..2e61d927b90 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HandleOperationStatus.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HandleOperationStatus.cs @@ -3,21 +3,21 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; -using System.Threading.Tasks; namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool HandleImmediateRetryStatus( OperationStatus internalStatus, TSessionFunctionsWrapper sessionFunctions, ref PendingContext pendingContext) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper => (internalStatus & OperationStatus.BASIC_MASK) > OperationStatus.MAX_MAP_TO_COMPLETED_STATUSCODE && HandleRetryStatus(internalStatus, sessionFunctions, ref pendingContext); @@ -26,17 +26,17 @@ private bool HandleImmediateRetryStatus [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool HandleImmediateNonPendingRetryStatus(OperationStatus internalStatus, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { Debug.Assert(epoch.ThisInstanceProtected()); switch (internalStatus) { case OperationStatus.RETRY_NOW: - Thread.Yield(); + _ = Thread.Yield(); return true; case OperationStatus.RETRY_LATER: InternalRefresh(sessionFunctions); - Thread.Yield(); + _ = Thread.Yield(); return true; default: return false; @@ -48,17 +48,17 @@ private bool HandleRetryStatus pendingContext) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { Debug.Assert(epoch.ThisInstanceProtected()); switch (internalStatus) { case OperationStatus.RETRY_NOW: - Thread.Yield(); + _ = Thread.Yield(); return true; case OperationStatus.RETRY_LATER: InternalRefresh(sessionFunctions); - Thread.Yield(); + _ = Thread.Yield(); return true; case OperationStatus.CPR_SHIFT_DETECTED: // Retry as (v+1) Operation @@ -67,8 +67,6 @@ private bool HandleRetryStatusInternal status of the trial. /// Operation status [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status HandleOperationStatus( - TsavoriteExecutionContext sessionCtx, - ref PendingContext pendingContext, - OperationStatus operationStatus) - { - if (OperationStatusUtils.TryConvertToCompletedStatusCode(operationStatus, out Status status)) - return status; - return HandleOperationStatus(sessionCtx, ref pendingContext, operationStatus, out _); - } + internal Status HandleOperationStatus(TsavoriteExecutionContext sessionCtx, + ref PendingContext pendingContext, OperationStatus operationStatus) + => OperationStatusUtils.TryConvertToCompletedStatusCode(operationStatus, out var status) + ? status + : HandleOperationStatus(sessionCtx, ref pendingContext, operationStatus, out _); /// /// Performs appropriate handling based on the internal failure status of the trial. @@ -111,12 +105,12 @@ internal Status HandleOperationStatus( /// Internal status of the trial. /// IO request, if operation went pending /// Operation status - [MethodImpl(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] internal Status HandleOperationStatus( TsavoriteExecutionContext sessionCtx, ref PendingContext pendingContext, OperationStatus operationStatus, - out AsyncIOContext request) + out AsyncIOContext request) { Debug.Assert(operationStatus != OperationStatus.RETRY_NOW, "OperationStatus.RETRY_NOW should have been handled before HandleOperationStatus"); Debug.Assert(operationStatus != OperationStatus.RETRY_LATER, "OperationStatus.RETRY_LATER should have been handled before HandleOperationStatus"); @@ -124,13 +118,13 @@ internal Status HandleOperationStatus( request = default; - if (OperationStatusUtils.TryConvertToCompletedStatusCode(operationStatus, out Status status)) + if (OperationStatusUtils.TryConvertToCompletedStatusCode(operationStatus, out var status)) return status; if (operationStatus == OperationStatus.ALLOCATE_FAILED) { - Debug.Assert(pendingContext.IsAsync, "Sync ops should have handled ALLOCATE_FAILED before HandleOperationStatus"); Debug.Assert(!pendingContext.flushEvent.IsDefault(), "Expected flushEvent for ALLOCATE_FAILED"); + Debug.Fail("Should have handled ALLOCATE_FAILED before HandleOperationStatus"); return new(StatusCode.Pending); } else if (operationStatus == OperationStatus.RECORD_ON_DISK) @@ -140,23 +134,31 @@ internal Status HandleOperationStatus( pendingContext.id = sessionCtx.totalPending++; sessionCtx.ioPendingRequests.Add(pendingContext.id, pendingContext); + if (!pendingContext.IsConditionalOp) + { + // We may have come from an already-pending operation, in which case we don't want to copy the diskLogRecord into the queue. + // But we do want to keep the diskLogRecord in the incoming "ref pendingContext" for disposal, so clear it in the dictionary. + // (We know this will not be a nullref because we just added it). Don't do this for CONDITIONAL_*; the diskLogRecord is what + // we'll insert or push if an overriding record is not found. + CollectionsMarshal.GetValueRefOrNullRef(sessionCtx.ioPendingRequests, pendingContext.id).diskLogRecord = default; + } + // Issue asynchronous I/O request request.id = pendingContext.id; - request.request_key = pendingContext.key; + + // Copying the key is stable; the pendingContext.requestKey will remain valid until it is freed (after the callback is invoked). + request.requestKey = pendingContext.requestKey; request.logicalAddress = pendingContext.logicalAddress; request.minAddress = pendingContext.minAddress; request.record = default; - if (pendingContext.IsAsync) - request.asyncOperation = new TaskCompletionSource>(TaskCreationOptions.RunContinuationsAsynchronously); - else - request.callbackQueue = sessionCtx.readyResponses; + request.callbackQueue = sessionCtx.readyResponses; - hlogBase.AsyncGetFromDisk(pendingContext.logicalAddress, hlog.GetAverageRecordSize(), request); + hlogBase.AsyncGetFromDisk(pendingContext.logicalAddress, IStreamBuffer.InitialIOSize, request); return new(StatusCode.Pending); } else { - Debug.Assert(pendingContext.IsAsync, "Sync ops should never return status.IsFaulted"); + Debug.Fail($"Unexpected OperationStatus {operationStatus}"); return new(StatusCode.Error); } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HashEntryInfo.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HashEntryInfo.cs index c8d07ce23b2..bbe7489b70d 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HashEntryInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/HashEntryInfo.cs @@ -3,10 +3,12 @@ using System.Runtime.CompilerServices; using System.Threading; -using static Tsavorite.core.Utility; namespace Tsavorite.core { + using static LogAddress; + using static Utility; + /// Hash table entry information for a key public unsafe struct HashEntryInfo { @@ -38,14 +40,13 @@ internal HashEntryInfo(long hash) slot = default; entry = default; this.hash = hash; - tag = (ushort)((ulong)this.hash >> Constants.kHashTagShift); + tag = HashBucketEntry.GetTag(hash); } /// /// The original address of this hash entry (at the time of FindTag, etc.) /// internal readonly long Address => entry.Address; - internal readonly long AbsoluteAddress => Utility.AbsoluteAddress(Address); /// /// The current address of this hash entry (which may have been updated (via CAS) in the bucket after FindTag, etc.) @@ -56,28 +57,20 @@ internal readonly long CurrentAddress get { return new HashBucketEntry() { word = bucket->bucket_entries[slot] }.Address; } } - internal readonly long AbsoluteCurrentAddress => Utility.AbsoluteAddress(CurrentAddress); - - /// - /// Return whether the has been updated - /// - internal readonly bool IsCurrent => CurrentAddress == Address; - /// /// Whether the original address for this hash entry (at the time of FindTag, etc.) is a readcache address. /// - internal readonly bool IsReadCache => entry.ReadCache; - - /// - /// Whether the current address for this hash entry (possibly modified after FindTag, etc.) is a readcache address. - /// - internal readonly bool IsCurrentReadCache => IsReadCache(CurrentAddress); + internal readonly bool IsReadCache + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => entry.IsReadCache; + } /// /// Set members to the current entry (which may have been updated (via CAS) in the bucket after FindTag, etc.) /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetToCurrent() => entry = new() { word = bucket->bucket_entries[slot] }; + internal void SetToCurrent() => entry.word = bucket->bucket_entries[slot]; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryCAS(long newLogicalAddress) @@ -86,9 +79,8 @@ internal bool TryCAS(long newLogicalAddress) HashBucketEntry updatedEntry = new() { Tag = tag, - Address = newLogicalAddress & Constants.kAddressMask, + Address = newLogicalAddress & kAddressBitMask, Tentative = false - // .ReadCache is included in newLogicalAddress }; if (entry.word == Interlocked.CompareExchange(ref bucket->bucket_entries[slot], updatedEntry.word, entry.word)) @@ -102,27 +94,26 @@ internal bool TryCAS(long newLogicalAddress) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool TryElide() { - if (entry.word != Interlocked.CompareExchange(ref bucket->bucket_entries[slot], 0L, entry.word)) + if (entry.word != Interlocked.CompareExchange(ref bucket->bucket_entries[slot], kInvalidAddress, entry.word)) return false; - entry.word = 0L; + + // We have successfully updated the actual bucket; set the local copy of the entry to match. + entry.word = kInvalidAddress; return true; } - public override string ToString() + /// + public override readonly string ToString() { var hashStr = GetHashString(hash); if (bucket == null) return $"hash {hashStr} "; - var isRC = "(rc)"; - var addrRC = IsReadCache ? isRC : string.Empty; - var currAddrRC = IsCurrentReadCache ? isRC : string.Empty; var isNotCurr = Address == CurrentAddress ? string.Empty : "*"; - var result = $"addr {AbsoluteAddress}{addrRC}, currAddr {AbsoluteCurrentAddress}{currAddrRC}{isNotCurr}, hash {hashStr}, tag {tag}, slot {slot},"; - result += $" Bkt1 [index {bucketIndex}, {HashBucket.ToString(firstBucket)}]"; - return result; + return $"addr {AddressString(Address)}, currAddr {AddressString(CurrentAddress)}{isNotCurr}, hash {hashStr}, tag {tag}, slot {slot}," + + $" Bkt1 [index {bucketIndex}, {HashBucket.ToString(firstBucket)}]"; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Helpers.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Helpers.cs index 601cc8b0325..845f278848b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Helpers.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Helpers.cs @@ -3,13 +3,13 @@ using System.Diagnostics; using System.Runtime.CompilerServices; -using static Tsavorite.core.Utility; +using static Tsavorite.core.LogAddress; namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { private enum LatchDestination { @@ -19,22 +19,24 @@ private enum LatchDestination } [MethodImpl(MethodImplOptions.AggressiveInlining)] - static ref RecordInfo WriteNewRecordInfo(ref TKey key, AllocatorBase log, long newPhysicalAddress, bool inNewVersion, long previousAddress) + static LogRecord WriteNewRecordInfo(TKey key, AllocatorBase log, long logicalAddress, long physicalAddress, + in RecordSizeInfo sizeInfo, bool inNewVersion, long previousAddress) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - ref RecordInfo recordInfo = ref log._wrapper.GetInfo(newPhysicalAddress); - recordInfo.WriteInfo(inNewVersion, previousAddress); - log._wrapper.SerializeKey(ref key, newPhysicalAddress); - return ref recordInfo; + var logRecord = log._wrapper.CreateLogRecord(logicalAddress, physicalAddress); + logRecord.InfoRef.WriteInfo(inNewVersion, previousAddress); + log._wrapper.InitializeRecord(key, logicalAddress, in sizeInfo, ref logRecord); + return logRecord; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void MarkPage(long logicalAddress, TsavoriteExecutionContext sessionCtx) - { - if (sessionCtx.phase == Phase.REST) - hlog.MarkPage(logicalAddress, sessionCtx.version); - else - hlog.MarkPageAtomic(logicalAddress, sessionCtx.version); - } + void OnDispose(ref LogRecord logRecord, DisposeReason disposeReason) => hlog.OnDispose(ref logRecord, disposeReason); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason disposeReason) => hlog.OnDisposeDiskRecord(ref logRecord, disposeReason); /// /// This is a wrapper for checking the record's version instead of just peeking at the latest record at the tail of the bucket. @@ -56,7 +58,7 @@ private bool IsRecordVersionNew(long logicalAddress) /// /// the last entry of a bucket /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] + [MethodImpl(MethodImplOptions.NoInlining)] // Called only if in PREPARE, so don't inline for the usual case private bool IsEntryVersionNew(ref HashBucketEntry entry) { // A version shift can only happen in an address after the checkpoint starts, as v_new threads RCU entries to the tail. @@ -64,13 +66,13 @@ private bool IsEntryVersionNew(ref HashBucketEntry entry) return false; // Read cache entries are not in new version - if (UseReadCache && entry.ReadCache) + if (UseReadCache && entry.IsReadCache) return false; // If the record is in memory, check if it has the new version bit set if (entry.Address < hlogBase.HeadAddress) return false; - return hlog.GetInfo(hlog.GetPhysicalAddress(entry.Address)).IsInNewVersion; + return LogRecord.GetInfo(hlogBase.GetPhysicalAddress(entry.Address)).IsInNewVersion; } // Can only elide the record if it is the tail of the tag chain (i.e. is the record in the hash bucket entry) and its @@ -78,20 +80,20 @@ private bool IsEntryVersionNew(ref HashBucketEntry entry) // Also, it cannot be elided if it is frozen due to checkpointing. [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool CanElide(TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + ref OperationStackContext stackCtx, RecordInfo srcRecordInfo) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { Debug.Assert(!stackCtx.recSrc.HasReadCacheSrc, "Should not call CanElide() for readcache records"); return stackCtx.hei.Address == stackCtx.recSrc.LogicalAddress && srcRecordInfo.PreviousAddress < hlogBase.BeginAddress - && !IsFrozen(sessionFunctions, ref stackCtx, ref srcRecordInfo); + && !IsFrozen(sessionFunctions, ref stackCtx, srcRecordInfo); } // If the record is in a checkpoint range, it must not be modified. If it is in the fuzzy region, it can only be modified // if it is a new record. [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool IsFrozen(TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + ref OperationStackContext stackCtx, RecordInfo srcRecordInfo) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { Debug.Assert(!stackCtx.recSrc.HasReadCacheSrc, "Should not call IsFrozen() for readcache records"); return sessionFunctions.Ctx.IsInV1 @@ -100,98 +102,121 @@ private bool IsFrozen(TSess } [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal long GetMinRevivifiableAddress() + => RevivificationManager.GetMinRevivifiableAddress(hlogBase.GetTailAddress(), hlogBase.ReadOnlyAddress); + + [MethodImpl(MethodImplOptions.NoInlining)] private (bool elided, bool added) TryElideAndTransferToFreeList(TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, (int usedValueLength, int fullValueLength, int fullRecordLength) recordLengths) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + ref OperationStackContext stackCtx, ref LogRecord logRecord) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // Try to CAS out of the hashtable and if successful, add it to the free list. - Debug.Assert(srcRecordInfo.IsSealed, "Expected a Sealed record in TryElideAndTransferToFreeList"); + Debug.Assert(logRecord.Info.IsSealed, "Expected a Sealed record in TryElideAndTransferToFreeList"); if (!stackCtx.hei.TryElide()) return (false, false); - return (true, TryTransferToFreeList(sessionFunctions, ref stackCtx, ref srcRecordInfo, recordLengths)); + return (true, TryTransferToFreeList(sessionFunctions, stackCtx.recSrc.LogicalAddress, ref logRecord)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryTransferToFreeList(TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, - ref RecordInfo srcRecordInfo, (int usedValueLength, int fullValueLength, int fullRecordLength) recordLengths) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool TryTransferToFreeList(TSessionFunctionsWrapper sessionFunctions, long logicalAddress, ref LogRecord logRecord) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // The record has been CAS'd out of the hashtable or elided from the chain, so add it to the free list. - Debug.Assert(srcRecordInfo.IsSealed, "Expected a Sealed record in TryTransferToFreeList"); + Debug.Assert(logRecord.Info.IsClosed, "Expected a Closed record in TryTransferToFreeList"); - // Dispose any existing key and value. We do this as soon as we have elided so objects are released for GC as early as possible. - // We don't want the caller to know details of the Filler, so we cleared out any extraValueLength entry to ensure the space beyond - // usedValueLength is zero'd for log-scan correctness. - ref TValue recordValue = ref stackCtx.recSrc.GetValue(); - ClearExtraValueSpace(ref srcRecordInfo, ref recordValue, recordLengths.usedValueLength, recordLengths.fullValueLength); - storeFunctions.DisposeRecord(ref stackCtx.recSrc.GetKey(), ref recordValue, DisposeReason.RevivificationFreeList); - - // Now that we've Disposed the record, see if its address is revivifiable. If not, just leave it orphaned and invalid. - if (stackCtx.recSrc.LogicalAddress < GetMinRevivifiableAddress()) + // If its address is not revivifiable, just leave it orphaned and invalid. + if (logicalAddress < GetMinRevivifiableAddress()) return false; - SetFreeRecordSize(stackCtx.recSrc.PhysicalAddress, ref srcRecordInfo, recordLengths.fullRecordLength); - return RevivificationManager.TryAdd(stackCtx.recSrc.LogicalAddress, recordLengths.fullRecordLength, ref sessionFunctions.Ctx.RevivificationStats); + // Application-level dispose (storeFunctions.OnDispose) was already called at the delete site. + // Call hlog.OnDispose with RevivificationFreeList to clear the key for freelist reuse + // (Deleted reason passes clearKey=false, but freelist reuse needs the key space cleared). + OnDispose(ref logRecord, DisposeReason.RevivificationFreeList); + + return RevivificationManager.TryAdd(logicalAddress, ref logRecord, ref sessionFunctions.Ctx.RevivificationStats); } + [MethodImpl(MethodImplOptions.NoInlining)] // Do not try to inline this, to keep TryAllocateRecord lean + bool TryTakeFreeRecord(TSessionFunctionsWrapper sessionFunctions, in RecordSizeInfo sizeInfo, long minRevivAddress, + out long logicalAddress, out long physicalAddress) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + { + // Caller checks for UseFreeRecordPool + if (RevivificationManager.TryTake(sizeInfo.ActualInlineRecordSize, minRevivAddress, out logicalAddress, ref sessionFunctions.Ctx.RevivificationStats)) + { + var logRecord = hlog.CreateLogRecord(logicalAddress); + Debug.Assert(logRecord.Info.IsSealed, "TryTakeFreeRecord: recordInfo should still have the revivification Seal"); + + // Preserve the Sealed bit due to checkpoint/recovery; see RecordInfo.WriteInfo. + physicalAddress = logRecord.physicalAddress; + return true; + } + + // No free record available. + logicalAddress = physicalAddress = default; + return false; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void SetRecordInvalid(long logicalAddress) { // This is called on exception recovery for a newly-inserted record. - var localLog = IsReadCache(logicalAddress) ? readcache : hlog; - ref var recordInfo = ref localLog.GetInfo(localLog.GetPhysicalAddress(AbsoluteAddress(logicalAddress))); - recordInfo.SetInvalid(); + var localLog = IsReadCache(logicalAddress) ? readcacheBase : hlogBase; + LogRecord.GetInfoRef(localLog.GetPhysicalAddress(logicalAddress)).SetInvalid(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool CASRecordIntoChain(ref TKey key, ref OperationStackContext stackCtx, long newLogicalAddress, ref RecordInfo newRecordInfo) + private bool CASRecordIntoChain(long newLogicalAddress, ref LogRecord newLogRecord, ref OperationStackContext stackCtx) { - var result = stackCtx.recSrc.LowestReadCachePhysicalAddress == Constants.kInvalidAddress + var result = stackCtx.recSrc.LowestReadCachePhysicalAddress == kInvalidAddress ? stackCtx.hei.TryCAS(newLogicalAddress) - : SpliceIntoHashChainAtReadCacheBoundary(ref key, ref stackCtx, newLogicalAddress); + : SpliceIntoHashChainAtReadCacheBoundary(ref stackCtx, newLogicalAddress); if (result) - newRecordInfo.UnsealAndValidate(); + newLogRecord.InfoRef.UnsealAndValidate(); return result; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void PostCopyToTail(ref TKey key, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo) - => PostCopyToTail(ref key, ref stackCtx, ref srcRecordInfo, stackCtx.hei.Address); + private void PostCopyToTail(in TSourceLogRecord srcLogRecord, ref OperationStackContext stackCtx) + where TSourceLogRecord : ISourceLogRecord + => PostCopyToTail(in srcLogRecord, ref stackCtx, stackCtx.hei.Address); [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void PostCopyToTail(ref TKey key, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, long highestReadCacheAddressChecked) + private void PostCopyToTail(in TSourceLogRecord srcLogRecord, ref OperationStackContext stackCtx, long highestReadCacheAddressChecked) + where TSourceLogRecord : ISourceLogRecord { // Nothing required here if not using ReadCache if (!UseReadCache) return; + // We're using the read cache, so any insertion must check that a readcache insertion wasn't done if (stackCtx.recSrc.HasReadCacheSrc) { // If we already have a readcache source, there will not be another inserted, so we can just invalidate the source directly. - srcRecordInfo.SetInvalidAtomic(); + srcLogRecord.InfoRef.SetInvalidAtomic(); } - else + else if (stackCtx.recSrc.HasMainLogSrc) { // We did not have a readcache source, so while we spliced a new record into the readcache/mainlog gap a competing readcache record may have been inserted at the tail. // If so, invalidate it. highestReadCacheAddressChecked is hei.Address unless we are from ConditionalCopyToTail, which may have skipped the readcache before this. // See "Consistency Notes" in TryCopyToReadCache for a discussion of why there ie no "momentary inconsistency" possible here. - ReadCacheCheckTailAfterSplice(ref key, ref stackCtx.hei, highestReadCacheAddressChecked); + ReadCacheCheckTailAfterSplice(srcLogRecord, ref stackCtx.hei, highestReadCacheAddressChecked); } } // Called after BlockAllocate or anything else that could shift HeadAddress, to adjust addresses or return false for RETRY as needed. // This refreshes the HashEntryInfo, so the caller needs to recheck to confirm the BlockAllocated address is still > hei.Address. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool VerifyInMemoryAddresses(ref OperationStackContext stackCtx) + private bool VerifyInMemoryAddresses(ref OperationStackContext stackCtx) { // If we have an in-memory source that fell below HeadAddress, return false and the caller will RETRY_LATER. if (stackCtx.recSrc.HasInMemorySrc && stackCtx.recSrc.LogicalAddress < stackCtx.recSrc.AllocatorBase.HeadAddress) return false; // If we're not using readcache or we don't have a splice point or it is still above readcache.HeadAddress, we're good. - if (!UseReadCache || stackCtx.recSrc.LowestReadCacheLogicalAddress == Constants.kInvalidAddress || stackCtx.recSrc.LowestReadCacheLogicalAddress >= readCacheBase.HeadAddress) + if (!UseReadCache || stackCtx.recSrc.LowestReadCacheLogicalAddress == kInvalidAddress || stackCtx.recSrc.LowestReadCacheLogicalAddress >= readcacheBase.HeadAddress) return true; // If the splice point went below readcache.HeadAddress, we would have to wait for the chain to be fixed up by eviction, @@ -200,13 +225,14 @@ private bool VerifyInMemoryAddresses(ref OperationStackContext(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx, out OperationStatus internalStatus) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool FindOrCreateTagAndTryEphemeralXLock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx, out OperationStatus internalStatus) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - // Transient must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. Manual locking already automatically locks the bucket. + // Ephemeral must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. + // Manual locking already automatically locks the bucket. hei already has the key's hashcode. FindOrCreateTag(ref stackCtx.hei, hlogBase.BeginAddress); - if (!TryTransientXLock(sessionFunctions, ref key, ref stackCtx, out internalStatus)) + if (!TryEphemeralXLock(sessionFunctions, ref stackCtx, out internalStatus)) return false; // Between the time we found the tag and the time we locked the bucket the record in hei.entry may have been elided, so make sure we don't have a stale address in hei.entry. @@ -216,13 +242,14 @@ private bool FindOrCreateTagAndTryTransientXLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx, out OperationStatus internalStatus) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool FindTagAndTryEphemeralXLock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx, out OperationStatus internalStatus) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - // Transient must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. Manual locking already automatically locks the bucket. + // Ephemeral must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. + // Manual locking already automatically locks the bucket. hei already has the key's hashcode. internalStatus = OperationStatus.NOTFOUND; - if (!FindTag(ref stackCtx.hei) || !TryTransientXLock(sessionFunctions, ref key, ref stackCtx, out internalStatus)) + if (!FindTag(ref stackCtx.hei) || !TryEphemeralXLock(sessionFunctions, ref stackCtx, out internalStatus)) return false; // Between the time we found the tag and the time we locked the bucket the record in hei.entry may have been elided, so make sure we don't have a stale address in hei.entry. @@ -232,13 +259,14 @@ private bool FindTagAndTryTransientXLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx, out OperationStatus internalStatus) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool FindTagAndTryEphemeralSLock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx, out OperationStatus internalStatus) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - // Transient must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. Manual locking already automatically locks the bucket. + // Ephemeral must lock the bucket before traceback, to prevent revivification from yanking the record out from underneath us. + // Manual locking already automatically locks the bucket. hei already has the key's hashcode. internalStatus = OperationStatus.NOTFOUND; - if (!FindTag(ref stackCtx.hei) || !TryTransientSLock(sessionFunctions, ref key, ref stackCtx, out internalStatus)) + if (!FindTag(ref stackCtx.hei) || !TryEphemeralSLock(sessionFunctions, ref stackCtx, out internalStatus)) return false; // Between the time we found the tag and the time we locked the bucket the record in hei.entry may have been elided, so make sure we don't have a stale address in hei.entry. @@ -251,18 +279,24 @@ private bool FindTagAndTryTransientSLock( - TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, int usedValueLength, int fullValueLength, int fullRecordLength) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref LogRecord srcLogRecord) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { + // Record was already disposed at the delete site (OnDispose with DisposeReason.Deleted). + // Heap fields and optionals are already cleared. This method only handles chain management. + if (!RevivificationManager.IsEnabled) { // We are not doing revivification, so we just want to remove the record from the tag chain so we don't potentially do an IO later for key // traceback. If we succeed, we need to SealAndInvalidate. It's fine if we don't succeed here; this is just tidying up the HashBucket. if (stackCtx.hei.TryElide()) - srcRecordInfo.SealAndInvalidate(); + srcLogRecord.InfoRef.SealAndInvalidate(); + return; } - else if (RevivificationManager.UseFreeRecordPool) + + if (RevivificationManager.UseFreeRecordPool) { // For non-FreeRecordPool revivification, we leave the record in as a normal tombstone so we can revivify it in the chain for the same key. // For FreeRecord Pool we must first Seal here, even if we're using the LockTable, because the Sealed state must survive this Delete() call. @@ -270,20 +304,16 @@ private void HandleRecordElision(sessionFunctions, ref stackCtx, ref srcRecordInfo, - (usedValueLength, fullValueLength, fullRecordLength)); + Debug.Assert(stackCtx.recSrc.LogicalAddress < hlogBase.ReadOnlyAddress || srcLogRecord.Info.Tombstone, $"Unexpected loss of Tombstone; Record should have been XLocked or SealInvalidated. RecordInfo: {srcLogRecord.Info.ToString()}"); + var (isElided, isAdded) = TryElideAndTransferToFreeList(sessionFunctions, ref stackCtx, ref srcLogRecord); if (!isElided) { // Leave this in the chain as a normal Tombstone; we aren't going to add a new record so we can't leave this one sealed. - srcRecordInfo.UnsealAndValidate(); + srcLogRecord.InfoRef.UnsealAndValidate(); } else if (!isAdded && RevivificationManager.restoreDeletedRecordsIfBinIsFull) { @@ -291,11 +321,12 @@ private void HandleRecordElision : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Delete operation. Replaces the value corresponding to 'key' with tombstone. @@ -40,21 +42,25 @@ public unsafe partial class TsavoriteKV /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalDelete(ref TKey key, long keyHash, ref TContext userContext, + internal OperationStatus InternalDelete(TKey key, long keyHash, ref TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - OperationStackContext stackCtx = new(keyHash); + OperationStackContext stackCtx = new(keyHash); pendingContext.keyHash = keyHash; + pendingContext.logicalAddress = kInvalidAddress; if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); - if (!FindTagAndTryTransientXLock(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (!FindTagAndTryEphemeralXLock(sessionFunctions, ref stackCtx, out OperationStatus status)) return status; - RecordInfo dummyRecordInfo = RecordInfo.InitialValid; - ref RecordInfo srcRecordInfo = ref dummyRecordInfo; + LogRecord srcLogRecord = default; DeleteInfo deleteInfo = new() { @@ -63,11 +69,11 @@ internal OperationStatus InternalDelete= hlogBase.ReadOnlyAddress) { - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); + + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; // If we already have a deleted record, there's nothing to do. - if (srcRecordInfo.Tombstone) + if (srcLogRecord.Info.Tombstone) return OperationStatus.NOTFOUND; // Mutable Region: Update the record in-place - deleteInfo.SetRecordInfo(ref srcRecordInfo); - ref TValue recordValue = ref stackCtx.recSrc.GetValue(); - // DeleteInfo's lengths are filled in and GetRecordLengths and SetDeletedValueLength are called inside ConcurrentDeleter. - if (sessionFunctions.ConcurrentDeleter(stackCtx.recSrc.PhysicalAddress, ref stackCtx.recSrc.GetKey(), ref recordValue, ref deleteInfo, ref srcRecordInfo, out int fullRecordLength)) + // DeleteInfo's lengths are filled in and GetRecordLengths and SetDeletedValueLength are called inside InPlaceDeleter. + if (sessionFunctions.InPlaceDeleter(ref srcLogRecord, ref deleteInfo)) { - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); + // Dispose resources and decrement value heap BEFORE setting Tombstone, + // so that GetValueHeapMemorySize returns the correct pre-tombstone value. + OnDispose(ref srcLogRecord, DisposeReason.Deleted); + + srcLogRecord.InfoRef.SetTombstone(); + srcLogRecord.InfoRef.SetModified(); // Try to transfer the record from the tag chain to the free record pool iff previous address points to invalid address. // Otherwise an earlier record for this key could be reachable again. - if (CanElide(sessionFunctions, ref stackCtx, ref srcRecordInfo)) - { - HandleRecordElision( - sessionFunctions, ref stackCtx, ref srcRecordInfo, deleteInfo.UsedValueLength, deleteInfo.FullValueLength, fullRecordLength); - } + if (CanElide(sessionFunctions, ref stackCtx, srcLogRecord.Info)) + HandleRecordElision(sessionFunctions, ref stackCtx, ref srcLogRecord); status = OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.InPlaceUpdatedRecord); goto Done; } + if (deleteInfo.Action == DeleteAction.CancelOperation) { status = OperationStatus.CANCELED; @@ -140,36 +149,33 @@ internal OperationStatus InternalDelete(sessionFunctions, ref key, ref stackCtx); + EphemeralXUnlock(sessionFunctions, ref stackCtx); } } @@ -177,19 +183,7 @@ internal OperationStatus InternalDelete(ref TKey key, TContext userContext, - ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - { - pendingContext.type = OperationType.DELETE; - if (pendingContext.key == default) pendingContext.key = hlog.GetKeyContainer(ref key); - pendingContext.userContext = userContext; - pendingContext.InitialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; - pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; - } - - private LatchDestination CheckCPRConsistencyDelete(Phase phase, ref OperationStackContext stackCtx, ref OperationStatus status) + private LatchDestination CheckCPRConsistencyDelete(Phase phase, ref OperationStackContext stackCtx, ref OperationStatus status) { // This is the same logic as Upsert; neither goes pending. return CheckCPRConsistencyUpsert(phase, ref stackCtx, ref status); @@ -199,79 +193,86 @@ private LatchDestination CheckCPRConsistencyDelete(Phase phase, ref OperationSta /// Create a new tombstoned record for Delete /// /// The record Key + /// The source record, if . and + /// it is either too small or is in readonly region, or is in raadcache /// Information about the operation context /// The current session - /// Contains the and structures for this operation, + /// Contains the and structures for this operation, /// and allows passing back the newLogicalAddress for invalidation in the case of exceptions. - /// If ., - /// this is the for /// DeleteInfo - private OperationStatus CreateNewRecordDelete(ref TKey key, ref PendingContext pendingContext, - TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, ref DeleteInfo deleteInfo) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private OperationStatus CreateNewRecordDelete(TKey key, ref LogRecord srcLogRecord, ref PendingContext pendingContext, + TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref DeleteInfo deleteInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - var value = default(TValue); - var (actualSize, allocatedSize, keySize) = hlog.GetRecordSize(ref key, ref value); + var sizeInfo = hlog.GetDeleteRecordSize(key); + AllocateOptions allocOptions = new() + { + recycle = true, + elideSourceRecord = stackCtx.recSrc.HasMainLogSrc && CanElide(sessionFunctions, ref stackCtx, srcLogRecord.Info) + }; // We know the existing record cannot be elided; it must point to a valid record; otherwise InternalDelete would have returned NOTFOUND. - if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, actualSize, ref allocatedSize, keySize, new AllocateOptions() { Recycle = true }, - out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status)) + if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, ref sizeInfo, allocOptions, out var newLogicalAddress, out var newPhysicalAddress, out var status)) return status; - ref RecordInfo newRecordInfo = ref WriteNewRecordInfo(ref key, hlogBase, newPhysicalAddress, inNewVersion: sessionFunctions.Ctx.InNewVersion, stackCtx.recSrc.LatestLogicalAddress); - newRecordInfo.SetTombstone(); - newRecordInfo.VectorSet = srcRecordInfo.VectorSet; + var newLogRecord = WriteNewRecordInfo(key, hlogBase, newLogicalAddress, newPhysicalAddress, in sizeInfo, sessionFunctions.Ctx.InNewVersion, previousAddress: stackCtx.recSrc.LatestLogicalAddress); + newLogRecord.InfoRef.SetTombstone(); stackCtx.SetNewRecord(newLogicalAddress); deleteInfo.Address = newLogicalAddress; deleteInfo.KeyHash = stackCtx.hei.hash; - deleteInfo.SetRecordInfo(ref newRecordInfo); - ref TValue newRecordValue = ref hlog.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize); - (deleteInfo.UsedValueLength, deleteInfo.FullValueLength) = GetNewValueLengths(actualSize, allocatedSize, newPhysicalAddress, ref newRecordValue); - - if (!sessionFunctions.SingleDeleter(ref key, ref newRecordValue, ref deleteInfo, ref newRecordInfo)) + if (!sessionFunctions.InitialDeleter(ref newLogRecord, ref deleteInfo)) { // This record was allocated with a minimal Value size (unless it was a revivified larger record) so there's no room for a Filler, // but we may want it for a later Delete, or for insert with a smaller Key. - if (RevivificationManager.UseFreeRecordPool && RevivificationManager.TryAdd(newLogicalAddress, newPhysicalAddress, allocatedSize, ref sessionFunctions.Ctx.RevivificationStats)) - stackCtx.ClearNewRecord(); - else - stackCtx.SetNewRecordInvalid(ref newRecordInfo); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + if (!RevivificationManager.UseFreeRecordPool || !TryTransferToFreeList(sessionFunctions, newLogicalAddress, ref newLogRecord)) + OnDispose(ref newLogRecord, DisposeReason.InsertAbandoned); if (deleteInfo.Action == DeleteAction.CancelOperation) return OperationStatus.CANCELED; return OperationStatus.NOTFOUND; // But not CreatedRecord } - SetTombstoneAndExtraValueLength(ref newRecordValue, ref newRecordInfo, deleteInfo.UsedValueLength, deleteInfo.FullValueLength); + newLogRecord.InfoRef.SetTombstone(); // Insert the new record by CAS'ing either directly into the hash entry or splicing into the readcache/mainlog boundary. - bool success = CASRecordIntoChain(ref key, ref stackCtx, newLogicalAddress, ref newRecordInfo); + var success = CASRecordIntoChain(newLogicalAddress, ref newLogRecord, ref stackCtx); if (success) { - PostCopyToTail(ref key, ref stackCtx, ref srcRecordInfo); + // Track key overflow internally — session functions only track value heap. + if (newLogRecord.Info.KeyIsOverflow) + hlogBase.logSizeTracker?.IncrementSize(newLogRecord.KeyOverflow.HeapMemorySize); + + PostCopyToTail(in srcLogRecord, ref stackCtx); // Note that this is the new logicalAddress; we have not retrieved the old one if it was below HeadAddress, and thus // we do not know whether 'logicalAddress' belongs to 'key' or is a collision. - sessionFunctions.PostSingleDeleter(ref key, ref deleteInfo, ref newRecordInfo); + sessionFunctions.PostInitialDeleter(ref newLogRecord, ref deleteInfo); - // Success should always Seal the old record. This may be readcache, readonly, or the temporary recordInfo, which is OK and saves the cost of an "if". - srcRecordInfo.Seal(); // Not elided so Seal without invalidate + // Success should always Seal the old record. This may be readcache or readonly, which is OK. + if (stackCtx.recSrc.HasMainLogSrc) + { + // Immediately dispose all resources on the source record before sealing. + OnDispose(ref srcLogRecord, DisposeReason.Deleted); + srcLogRecord.InfoRef.Seal(); // Not elided so Seal without invalidate + } stackCtx.ClearNewRecord(); - pendingContext.recordInfo = newRecordInfo; pendingContext.logicalAddress = newLogicalAddress; return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.CreatedRecord); } // CAS failed - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - ref TValue insertedValue = ref hlog.GetValue(newPhysicalAddress); - ref TKey insertedKey = ref hlog.GetKey(newPhysicalAddress); - storeFunctions.DisposeRecord(ref insertedKey, ref insertedValue, DisposeReason.SingleDeleterCASFailed); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + OnDispose(ref newLogRecord, DisposeReason.InitialDeleterCASFailed); - SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress, allocatedSize); + SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress); return OperationStatus.RETRY_NOW; // CAS failure does not require epoch refresh } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalLock.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalLock.cs index c36df9ad880..fccbac1a8ec 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalLock.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalLock.cs @@ -6,9 +6,9 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool InternalTryLockShared(long keyHash) diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs index ae6879d90ad..fb98e8c0712 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRMW.cs @@ -6,9 +6,11 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Read-Modify-Write Operation. Updates value of 'key' using 'input' and current value. @@ -47,44 +49,49 @@ public unsafe partial class TsavoriteKV /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalRMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, ref TContext userContext, + internal OperationStatus InternalRMW(TKey key, long keyHash, ref TInput input, ref TOutput output, ref TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - OperationStackContext stackCtx = new(keyHash); + OperationStackContext stackCtx = new(keyHash); pendingContext.keyHash = keyHash; + pendingContext.logicalAddress = kInvalidAddress; if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); - if (!FindOrCreateTagAndTryTransientXLock(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (!FindOrCreateTagAndTryEphemeralXLock(sessionFunctions, ref stackCtx, out var status)) return status; - RecordInfo dummyRecordInfo = RecordInfo.InitialValid; - ref RecordInfo srcRecordInfo = ref dummyRecordInfo; + LogRecord srcLogRecord = default; RMWInfo rmwInfo = new() { Version = sessionFunctions.Ctx.version, - SessionID = sessionFunctions.Ctx.sessionID + SessionID = sessionFunctions.Ctx.sessionID, + KeyHash = stackCtx.hei.hash }; // We must use try/finally to ensure unlocking even in the presence of exceptions. The inner try/finally ensures - // the transient X-lock on the hash bucket is released even if a Post*Operation callback (e.g. AOF append) throws. + // the ephemeral X-lock on the hash bucket is released even if a Post*Operation callback (e.g. AOF append) throws. try { // Search the entire in-memory region. - if (!TryFindRecordForUpdate(ref key, ref stackCtx, hlogBase.HeadAddress, out status)) + if (!TryFindRecordForUpdate(key, ref stackCtx, hlogBase.HeadAddress, out status)) return status; // These track the latest main-log address in the tag chain; InternalContinuePendingRMW uses them to check for new inserts. - pendingContext.InitialEntryAddress = stackCtx.hei.Address; - pendingContext.InitialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; + pendingContext.initialEntryAddress = stackCtx.hei.Address; + pendingContext.initialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; // If there is a readcache record, use it as the CopyUpdater source. if (stackCtx.recSrc.HasReadCacheSrc) { - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); goto CreateNewRecord; } @@ -98,7 +105,7 @@ internal OperationStatus InternalRMW= hlogBase.ReadOnlyAddress) { - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); // Mutable Region: Update the record in-place. We perform mutable updates only if we are in normal processing phase of checkpointing rmwInfo.Address = stackCtx.recSrc.LogicalAddress; - rmwInfo.KeyHash = stackCtx.hei.hash; - rmwInfo.IsFromPending = pendingContext.type != OperationType.NONE; - rmwInfo.SetRecordInfo(ref srcRecordInfo); - ref TValue recordValue = ref stackCtx.recSrc.GetValue(); - if (srcRecordInfo.Tombstone) + if (srcLogRecord.Info.Tombstone) { // If we're doing revivification and this is in the revivifiable range, try to revivify--otherwise we'll create a new record. if (RevivificationManager.IsEnabled && stackCtx.recSrc.LogicalAddress >= GetMinRevivifiableAddress()) { - if (!sessionFunctions.NeedInitialUpdate(ref key, ref input, ref output, ref rmwInfo)) + if (!sessionFunctions.NeedInitialUpdate(key, ref input, ref output, ref rmwInfo)) { status = OperationStatus.NOTFOUND; goto Done; } - if (TryRevivifyInChain(ref key, ref input, ref output, ref pendingContext, sessionFunctions, ref stackCtx, ref srcRecordInfo, ref rmwInfo, out status, ref recordValue) + if (TryRevivifyInChain(ref srcLogRecord, ref input, ref output, ref pendingContext, sessionFunctions, ref stackCtx, ref rmwInfo, out status) || status != OperationStatus.SUCCESS) goto Done; } goto CreateNewRecord; } - // rmwInfo's lengths are filled in and GetValueLengths and SetLength are called inside InPlaceUpdater. - if (sessionFunctions.InPlaceUpdater(stackCtx.recSrc.PhysicalAddress, ref key, ref input, ref recordValue, ref output, ref rmwInfo, out status, ref srcRecordInfo)) + // Track value heap delta across in-place update. HeapMemorySize is zero for inline values but IPU may change the value from inline to object or vice-versa. + var sizeTracker = hlogBase.logSizeTracker; + var ipuPreHeap = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() : 0L; + var ipuResult = sessionFunctions.InPlaceUpdater(ref srcLogRecord, ref input, ref output, ref rmwInfo, out status); + var ipuDelta = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() - ipuPreHeap : 0L; + if (ipuResult) { - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); + if (ipuDelta != 0) + sizeTracker.IncrementSize(ipuDelta); + - pendingContext.recordInfo = srcRecordInfo; - pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; // status has been set by InPlaceUpdater + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; goto Done; } if (rmwInfo.Action == RMWAction.ExpireAndStop) { - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); - srcRecordInfo.SetDirtyAndModified(); + // ExpireAndStop: the object was mutated in-place (e.g. last element removed) + // before IPU returned false. Track the delta before OnDispose subtracts the + // remaining empty-collection overhead. + if (ipuDelta != 0) + sizeTracker.IncrementSize(ipuDelta); - // ExpireAndStop means to override default Delete handling (which is to go to InitialUpdater) by leaving the tombstoned record as current. - // Our SessionFunctionsWrapper.InPlaceUpdater implementation has already reinitialized-in-place or set Tombstone as appropriate and marked the record. + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + + // Dispose resources and decrement value heap BEFORE setting Tombstone so GetValueHeapMemorySize returns the correct pre-tombstone value. + OnDispose(ref srcLogRecord, DisposeReason.Deleted); + + srcLogRecord.InfoRef.SetTombstone(); + srcLogRecord.InfoRef.SetModified(); // Try to transfer the record from the tag chain to the free record pool iff previous address points to invalid address. // Otherwise an earlier record for this key could be reachable again. - if (CanElide(sessionFunctions, ref stackCtx, ref srcRecordInfo)) - { - HandleRecordElision( - sessionFunctions, ref stackCtx, ref srcRecordInfo, rmwInfo.UsedValueLength, rmwInfo.FullValueLength, rmwInfo.FullRecordLength); - } + if (CanElide(sessionFunctions, ref stackCtx, srcLogRecord.Info)) + HandleRecordElision(sessionFunctions, ref stackCtx, ref srcLogRecord); - pendingContext.recordInfo = srcRecordInfo; + goto Done; + } + else if (rmwInfo.Action == RMWAction.ExpireAndResume) + { + // ExpireAndResume: for IPU, ReinitializeExpiredRecord already called OnDispose(Deleted). + // If it failed and we fall through to CreateNewRecord, the record is already disposed. + } + else if (rmwInfo.Action == RMWAction.WrongType) + { + // status has been set by InPlaceUpdater, and no modification should have been made to the record. pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.WrongType); goto Done; } @@ -183,7 +206,7 @@ internal OperationStatus InternalRMW= hlogBase.BeginAddress) @@ -193,26 +216,20 @@ internal OperationStatus InternalRMW(sessionFunctions, ref key, ref stackCtx); + EphemeralXUnlock(sessionFunctions, ref stackCtx); } } @@ -233,78 +250,65 @@ internal OperationStatus InternalRMW(ref TKey key, ref TInput input, TOutput output, TContext userContext, - ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + [MethodImpl(MethodImplOptions.NoInlining)] + private void CreatePendingRMWContext(TKey key, ref TInput input, ref TOutput output, TContext userContext, + ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { pendingContext.type = OperationType.RMW; - if (pendingContext.key == default) - pendingContext.key = hlog.GetKeyContainer(ref key); - if (pendingContext.input == default) - pendingContext.input = sessionFunctions.GetHeapContainer(ref input); - - pendingContext.output = output; - sessionFunctions.ConvertOutputToHeap(ref input, ref pendingContext.output); - - pendingContext.userContext = userContext; + pendingContext.CopyInputsForReadOrRMW(key, ref input, ref output, userContext, sessionFunctions, hlogBase.bufferPool); pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; } - private bool TryRevivifyInChain(ref TKey key, ref TInput input, ref TOutput output, ref PendingContext pendingContext, - TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, ref RMWInfo rmwInfo, - out OperationStatus status, ref TValue recordValue) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool TryRevivifyInChain(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref PendingContext pendingContext, + TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref RMWInfo rmwInfo, out OperationStatus status) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - if (IsFrozen(sessionFunctions, ref stackCtx, ref srcRecordInfo)) + if (IsFrozen(sessionFunctions, ref stackCtx, logRecord.Info)) goto NeedNewRecord; // This record is safe to revivify even if its PreviousAddress points to a valid record, because it is revivified for the same key. - var ok = true; + var ok = false; try { - if (srcRecordInfo.Tombstone) + var sizeInfo = hlog.GetRMWInitialRecordSize(logRecord, ref input, sessionFunctions); + ref RevivificationStats stats = ref sessionFunctions.Ctx.RevivificationStats; + if (logRecord.TrySetContentLengths(in sizeInfo)) { - srcRecordInfo.ClearTombstone(); + logRecord.InfoRef.ClearTombstone(); + logRecord.ClearOptionals(); - if (RevivificationManager.IsFixedLength) - rmwInfo.UsedValueLength = rmwInfo.FullValueLength = RevivificationManager.FixedValueLength; - else + if (sessionFunctions.InitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo)) { - var recordLengths = GetRecordLengths(stackCtx.recSrc.PhysicalAddress, ref recordValue, ref srcRecordInfo); - rmwInfo.FullValueLength = recordLengths.fullValueLength; + sessionFunctions.PostInitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); - // RMW uses GetInitialRecordSize because it has only the initial Input, not a Value - var (requiredSize, _, _) = hlog.GetRMWInitialRecordSize(ref key, ref input, sessionFunctions); - (ok, rmwInfo.UsedValueLength) = TryReinitializeTombstonedValue(sessionFunctions, - ref srcRecordInfo, ref key, ref recordValue, requiredSize, recordLengths, stackCtx.recSrc.PhysicalAddress); - } + // Track revived record's value heap — was subtracted at OnDispose(Deleted). + var valueHeap = logRecord.GetValueHeapMemorySize(); + if (valueHeap != 0) + hlogBase.logSizeTracker?.IncrementSize(valueHeap); - ref RevivificationStats stats = ref sessionFunctions.Ctx.RevivificationStats; - if (ok && sessionFunctions.InitialUpdater(ref key, ref input, ref recordValue, ref output, ref rmwInfo, ref srcRecordInfo)) - { - sessionFunctions.PostInitialUpdater(ref key, ref input, ref recordValue, ref output, ref rmwInfo, ref srcRecordInfo); // Success - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); - pendingContext.recordInfo = srcRecordInfo; pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; - // We "IPU'd" because we reused a tombstone, but since the record we have reused did not logically exist, we must also bubble up that the original key was not found (logically). OperationStatus.NOTFOUND bubbles up success but also indicates that the record was not found in the database. + + // We "IPU'd" because we reused a tombstone, but since the record we have reused did not logically exist, we must also bubble up that the original key was not found (logically). + // OperationStatus.NOTFOUND bubbles up success but also indicates that the record was not found in the database. status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.InPlaceUpdatedRecord); stats.inChainSuccesses++; + ok = true; return true; } - - // Did not revivify; restore the tombstone and leave the deleted record there. - srcRecordInfo.SetTombstone(); - stats.inChainFailures++; } + // Did not revivify; restore the tombstone in 'finally' and leave the deleted record there. + stats.inChainFailures++; } finally { - if (ok) - SetExtraValueLength(ref recordValue, ref srcRecordInfo, rmwInfo.UsedValueLength, rmwInfo.FullValueLength); - else - SetTombstoneAndExtraValueLength(ref recordValue, ref srcRecordInfo, rmwInfo.UsedValueLength, rmwInfo.FullValueLength); // Restore tombstone and ensure default value on inability to update in place + if (!ok) + logRecord.InfoRef.SetTombstone(); } NeedNewRecord: @@ -313,7 +317,7 @@ private bool TryRevivifyInChain stackCtx, ref OperationStatus status) + private LatchDestination CheckCPRConsistencyRMW(Phase phase, ref OperationStackContext stackCtx, ref OperationStatus status) { // The idea of CPR is that if a thread in version V tries to perform an operation and notices a record in V+1, it needs to back off and run CPR_SHIFT_DETECTED. // Similarly, a V+1 thread cannot update a V record; it needs to do a read-copy-update (or upsert at tail) instead of an in-place update. @@ -352,74 +356,81 @@ private LatchDestination CheckCPRConsistencyRMW(Phase phase, ref OperationStackC /// /// Create a new record for RMW /// - /// - /// - /// - /// - /// The record Key - /// Input to the operation - /// Old value - /// The result of ISessionFunctions.SingleWriter + /// Key, if inserting a new record. + /// The source record. If . + /// it is in-memory (either too small or is in readonly region, or is in readcache); otherwise it is from disk IO + /// Input to the ISessionFunctions operation + /// The result of ISessionFunctions operation /// Information about the operation context /// The current session - /// Contains the and structures for this operation, + /// Contains the and structures for this operation, /// and allows passing back the newLogicalAddress for invalidation in the case of exceptions. If called from pending IO, /// this is populated from the data read from disk. - /// If ., - /// this is the for . Otherwise, if called from pending IO, - /// this is the read from disk. If neither of these, it is a default . - /// Whether we are doing a CopyUpdate, either from in-memory or pending IO + /// Whether we are doing a CopyUpdate, either from in-memory or pending IO. /// RMWInfo /// - private OperationStatus CreateNewRecordRMW(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, + private OperationStatus CreateNewRecordRMW(TKey key, in TSourceLogRecord srcLogRecord, ref TInput input, ref TOutput output, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, bool doingCU, ref RMWInfo rmwInfo) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + ref OperationStackContext stackCtx, bool doingCU, ref RMWInfo rmwInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - bool forExpiration = false; - bool addTombstone = false; + var forExpiration = false; + var addTombstone = false; RetryNow: - rmwInfo.Address = doingCU && !stackCtx.recSrc.HasReadCacheSrc ? stackCtx.recSrc.LogicalAddress : Constants.kInvalidAddress; + rmwInfo.Address = doingCU && !stackCtx.recSrc.HasReadCacheSrc ? stackCtx.recSrc.LogicalAddress : kInvalidAddress; + rmwInfo.SourceAddress = rmwInfo.Address; rmwInfo.KeyHash = stackCtx.hei.hash; - rmwInfo.IsFromPending = pendingContext.type != OperationType.NONE; AllocateOptions allocOptions = new() { - Recycle = true, - ElideSourceRecord = stackCtx.recSrc.HasMainLogSrc && CanElide(sessionFunctions, ref stackCtx, ref srcRecordInfo) + recycle = true, + elideSourceRecord = stackCtx.recSrc.HasMainLogSrc && CanElide(sessionFunctions, ref stackCtx, srcLogRecord.Info) }; // Perform Need* if (doingCU) { - rmwInfo.SetRecordInfo(ref srcRecordInfo); - if (!sessionFunctions.NeedCopyUpdate(ref key, ref input, ref value, ref output, ref rmwInfo)) + if (!sessionFunctions.NeedCopyUpdate(in srcLogRecord, ref input, ref output, ref rmwInfo)) { if (rmwInfo.Action == RMWAction.CancelOperation) return OperationStatus.CANCELED; else if (rmwInfo.Action == RMWAction.ExpireAndResume) { + // The old value is logically deleted (expired). Dispose resources immediately. + if (stackCtx.recSrc.HasMainLogSrc) + OnDispose(ref srcLogRecord.AsMemoryLogRecordRef(), DisposeReason.Deleted); doingCU = false; forExpiration = true; } else if (rmwInfo.Action == RMWAction.ExpireAndStop) { - if (allocOptions.ElideSourceRecord) + // Immediately dispose all resources on the expired source record. + if (stackCtx.recSrc.HasMainLogSrc) + OnDispose(ref srcLogRecord.AsMemoryLogRecordRef(), DisposeReason.Deleted); + + if (allocOptions.elideSourceRecord) { - srcRecordInfo.SetTombstone(); - srcRecordInfo.SetDirtyAndModified(); - var oldRecordLengths = GetRecordLengths(stackCtx.recSrc.PhysicalAddress, ref hlog.GetValue(stackCtx.recSrc.PhysicalAddress), ref srcRecordInfo); - // Elide from hei, and try to either do in-chain tombstoning or free list transfer. - HandleRecordElision( - sessionFunctions, ref stackCtx, ref srcRecordInfo, oldRecordLengths.usedValueLength, oldRecordLengths.fullValueLength, oldRecordLengths.fullRecordLength); + srcLogRecord.InfoRef.SetTombstone(); + srcLogRecord.InfoRef.SetModified(); + + // Elide from hei, and try to either do in-chain tombstoning or free list transfer. srcLogRecord is elidable so must be a memory LogRecord. + ref var inMemoryLogRecord = ref srcLogRecord.AsMemoryLogRecordRef(); + HandleRecordElision(sessionFunctions, ref stackCtx, ref inMemoryLogRecord); // no new record created and hash entry is empty now - return OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.Found | StatusCode.Expired); + return OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.Expired); } - // otherwise we shall continue down the tombstoning path + // Non-elidable: create tombstone record addTombstone = true; } + else if (rmwInfo.Action == RMWAction.WrongType) + return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.WrongType); else return OperationStatus.SUCCESS; } @@ -427,47 +438,39 @@ private OperationStatus CreateNewRecordRMW(ref key, ref input, ref newRecordValue, ref output, ref newRecordInfo, - ref rmwInfo, newLogicalAddress, sessionFunctions, isIpu: false, out status)) + if (!ReinitializeExpiredRecord(ref newLogRecord, ref input, ref output, ref rmwInfo, newLogicalAddress, sessionFunctions, isIpu: false, out status)) { // An IPU was not (or could not) be done. Cancel if requested, else invalidate the allocated record and retry. if (status == OperationStatus.CANCELED) return status; // Save allocation for revivification (not retry, because this may have been false because the record was too small), or abandon it if that fails. - if (RevivificationManager.UseFreeRecordPool && RevivificationManager.TryAdd(newLogicalAddress, newPhysicalAddress, allocatedSize, ref sessionFunctions.Ctx.RevivificationStats)) - stackCtx.ClearNewRecord(); - else - stackCtx.SetNewRecordInvalid(ref newRecordInfo); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + if (!RevivificationManager.UseFreeRecordPool || !TryTransferToFreeList(sessionFunctions, newLogicalAddress, ref newLogRecord)) + OnDispose(ref newLogRecord, DisposeReason.InsertAbandoned); goto RetryNow; } - addTombstone = newRecordInfo.Tombstone; + addTombstone = newLogRecord.Info.Tombstone; goto DoCAS; } + else if (rmwInfo.Action == RMWAction.WrongType) + { + status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.CreatedRecord | StatusCode.Expired); + return OperationStatus.NOTFOUND; + } else return OperationStatus.SUCCESS | (forExpiration ? OperationStatus.EXPIRED : OperationStatus.SUCCESS); } else { - Debug.Assert(!addTombstone, "This block should only be handling tombstoning requests by NCU where the previous record was not elidable."); - newRecordInfo.SetDirtyAndModified(); - newRecordInfo.SetTombstone(); + Debug.Assert(addTombstone, "This block should only be handling tombstoning requests by NCU where the previous record was not elidable."); + newLogRecord.InfoRef.SetModified(); + newLogRecord.InfoRef.SetTombstone(); status = OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.CreatedRecord | StatusCode.Expired); } @@ -553,10 +564,14 @@ private OperationStatus CreateNewRecordRMW= GetMinRevivifiableAddress()) - { - // We need to re-get the old record's length because rmwInfo has the new record's info. If freelist-add fails, it remains Sealed/Invalidated. - var oldRecordLengths = GetRecordLengths(stackCtx.recSrc.PhysicalAddress, ref hlog.GetValue(stackCtx.recSrc.PhysicalAddress), ref srcRecordInfo); - _ = TryTransferToFreeList(sessionFunctions, ref stackCtx, ref srcRecordInfo, oldRecordLengths); - } + _ = TryTransferToFreeList(sessionFunctions, stackCtx.recSrc.LogicalAddress, ref inMemoryLogRecord); + else + OnDispose(ref inMemoryLogRecord, DisposeReason.Elided); } else - srcRecordInfo.Seal(); // The record was not elided, so do not Invalidate + { + // If it is in mutable or fuzzy region, we must Seal + if (stackCtx.recSrc.HasMainLogSrc && stackCtx.recSrc.LogicalAddress > hlogBase.SafeReadOnlyAddress) + srcLogRecord.InfoRef.Seal(); // The record was not elided, so do not Invalidate + } } + Done: stackCtx.ClearNewRecord(); - pendingContext.recordInfo = newRecordInfo; pendingContext.logicalAddress = newLogicalAddress; return status; } // CAS failed - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - ref TValue insertedValue = ref hlog.GetValue(newPhysicalAddress); - ref TKey insertedKey = ref hlog.GetKey(newPhysicalAddress); - storeFunctions.DisposeRecord(ref insertedKey, ref insertedValue, doingCU ? DisposeReason.CopyUpdaterCASFailed : DisposeReason.InitialUpdaterCASFailed); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + OnDispose(ref newLogRecord, doingCU ? DisposeReason.CopyUpdaterCASFailed : DisposeReason.InitialUpdaterCASFailed); - SetExtraValueLength(ref newRecordValue, ref newRecordInfo, rmwInfo.UsedValueLength, rmwInfo.FullValueLength); - SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress, allocatedSize); + SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress); return OperationStatus.RETRY_NOW; // CAS failure does not require epoch refresh } - internal bool ReinitializeExpiredRecord(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref RecordInfo recordInfo, ref RMWInfo rmwInfo, + internal bool ReinitializeExpiredRecord(ref LogRecord logRecord, ref TInput input, ref TOutput output, ref RMWInfo rmwInfo, long logicalAddress, TSessionFunctionsWrapper sessionFunctions, bool isIpu, out OperationStatus status) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // This is called for InPlaceUpdater or CopyUpdater only; CopyUpdater however does not copy an expired record, so we return CreatedRecord. var advancedStatusCode = isIpu ? StatusCode.InPlaceUpdatedRecord : StatusCode.CreatedRecord; advancedStatusCode |= StatusCode.Expired; - if (!sessionFunctions.NeedInitialUpdate(ref key, ref input, ref output, ref rmwInfo)) + + // Dispose the expired record's resources before reinitializing. + // For IPU, skip the heap-tracker decrement because the outer pre/post delta + // tracking in InternalRMW captures the net heap change across the entire IPU. + // We still need the disposal side-effects (trigger + ClearHeapFields). + // For CU, this is the newly allocated record (source was already disposed at the decision site). + if (isIpu) + { + storeFunctions.OnDispose(ref logRecord, DisposeReason.Deleted); + logRecord.ClearHeapFields(clearKey: false); + logRecord.ClearOptionals(); + } + else + { + OnDispose(ref logRecord, DisposeReason.Deleted); + } + + if (!sessionFunctions.NeedInitialUpdate(logRecord, ref input, ref output, ref rmwInfo)) { if (rmwInfo.Action == RMWAction.CancelOperation) { @@ -633,22 +716,23 @@ internal bool ReinitializeExpiredRecord= requiredSize) + if (logRecord.TryReinitializeValueLength(in sizeInfo)) { - if (sessionFunctions.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo)) + if (sessionFunctions.InitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo)) { - // If IPU path, we need to complete PostInitialUpdater as well + // If IPU path, we need to complete PostInitialUpdater as well. + // No explicit heap tracking here — the outer pre/post delta in InternalRMW + // captures the net change (old value → new value). if (isIpu) - sessionFunctions.PostInitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + sessionFunctions.PostInitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, advancedStatusCode); return true; @@ -663,7 +747,7 @@ internal bool ReinitializeExpiredRecordallocate->IU path + // IPU: move to the NIU->allocate->IU path. Set tombstone so CreateNewRecordRMW uses InitialUpdater (doingCU=false). // CU: caller invalidates allocation, retries operation as NIU->allocate->IU + if (isIpu) + logRecord.InfoRef.SetTombstone(); status = OperationStatus.SUCCESS; return false; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRead.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRead.cs index e20c675b985..af304d447ca 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRead.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalRead.cs @@ -6,9 +6,11 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Read operation. Computes the 'output' from 'input' and current value corresponding to 'key'. @@ -50,25 +52,23 @@ public unsafe partial class TsavoriteKV /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalRead(ref TKey key, long keyHash, ref TInput input, ref TOutput output, + internal OperationStatus InternalRead(TKey key, long keyHash, ref TInput input, ref TOutput output, TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - OperationStackContext stackCtx = new(keyHash); + OperationStackContext stackCtx = new(keyHash); pendingContext.keyHash = keyHash; + pendingContext.logicalAddress = kInvalidAddress; if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); - if (!FindTagAndTryTransientSLock(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (!FindTagAndTryEphemeralSLock(sessionFunctions, ref stackCtx, out var status)) return status; - stackCtx.SetRecordSourceToHashEntry(hlogBase); - - // We have to assign a reference on declaration, so assign it here before we know whether LogicalAddress is above or below HeadAddress. - // It must be at this scope so it can be unlocked in 'finally'. - RecordInfo dummyRecordInfo = RecordInfo.InitialValid; - ref RecordInfo srcRecordInfo = ref dummyRecordInfo; - ReadInfo readInfo = new() { Version = sessionFunctions.Ctx.version, @@ -78,116 +78,105 @@ internal OperationStatus InternalRead= hlogBase.SafeReadOnlyAddress) { // Mutable region (even fuzzy region is included here) - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); - pendingContext.recordInfo = srcRecordInfo; - readInfo.SetRecordInfo(ref srcRecordInfo); - - if (srcRecordInfo.IsClosedOrTombstoned(ref status)) + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); + if (srcLogRecord.Info.IsClosedOrTombstoned(ref status)) return status; - if (sessionFunctions.ConcurrentReader(ref key, ref input, ref stackCtx.recSrc.GetValue(), ref output, ref readInfo, ref srcRecordInfo)) - return OperationStatus.SUCCESS; - return CheckFalseActionStatus(readInfo); + return sessionFunctions.Reader(in srcLogRecord, ref input, ref output, ref readInfo) + ? OperationStatus.SUCCESS + : CheckFalseActionStatus(ref readInfo); } + // Pending (and CopyFromImmutable may go pending) must track the latest searched-below addresses. They are the same if there are no readcache records. + pendingContext.initialEntryAddress = stackCtx.hei.Address; + pendingContext.initialLatestLogicalAddress = stackCtx.recSrc.LatestLogicalAddress; + if (stackCtx.recSrc.LogicalAddress >= hlogBase.HeadAddress) { // Immutable region - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); - pendingContext.recordInfo = srcRecordInfo; - readInfo.SetRecordInfo(ref srcRecordInfo); - - if (srcRecordInfo.IsClosedOrTombstoned(ref status)) + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); + if (srcLogRecord.Info.IsClosedOrTombstoned(ref status)) return status; - if (sessionFunctions.SingleReader(ref key, ref input, ref stackCtx.recSrc.GetValue(), ref output, ref readInfo)) + if (sessionFunctions.Reader(in srcLogRecord, ref input, ref output, ref readInfo)) { - if (pendingContext.readCopyOptions.CopyFrom != ReadCopyFrom.AllImmutable) - return OperationStatus.SUCCESS; - return CopyFromImmutable(ref key, ref input, ref output, userContext, ref pendingContext, sessionFunctions, ref stackCtx, ref status, stackCtx.recSrc.GetValue()); + return pendingContext.readCopyOptions.CopyFrom != ReadCopyFrom.AllImmutable + ? OperationStatus.SUCCESS + : CopyFromImmutable(ref srcLogRecord, ref input, ref output, userContext, ref pendingContext, sessionFunctions, ref stackCtx, ref status); } - return CheckFalseActionStatus(readInfo); + return CheckFalseActionStatus(ref readInfo); } if (stackCtx.recSrc.LogicalAddress >= hlogBase.BeginAddress) { // On-Disk Region - Debug.Assert(!sessionFunctions.IsManualLocking || LockTable.IsLocked(ref stackCtx.hei), "A Lockable-session Read() of an on-disk key requires a LockTable lock"); + Debug.Assert(!sessionFunctions.IsTransactionalLocking || LockTable.IsLocked(ref stackCtx.hei), "A Transactional-session Read() of an on-disk key requires a LockTable lock"); // Note: we do not lock here; we wait until reading from disk, then lock in the ContinuePendingRead chain. if (hlogBase.IsNullDevice) return OperationStatus.NOTFOUND; - CreatePendingReadContext(ref key, ref input, output, userContext, ref pendingContext, sessionFunctions, stackCtx.recSrc.LogicalAddress); + CreatePendingReadContext(key, ref input, ref output, userContext, ref pendingContext, sessionFunctions, stackCtx.recSrc.LogicalAddress); return OperationStatus.RECORD_ON_DISK; } // No record found - Debug.Assert(!sessionFunctions.IsManualLocking || LockTable.IsLocked(ref stackCtx.hei), "A Lockable-session Read() of a non-existent key requires a LockTable lock"); + Debug.Assert(!sessionFunctions.IsTransactionalLocking || LockTable.IsLocked(ref stackCtx.hei), "A Transactional-session Read() of a non-existent key requires a LockTable lock"); return OperationStatus.NOTFOUND; } finally { stackCtx.HandleNewRecordOnException(this); - TransientSUnlock(sessionFunctions, ref key, ref stackCtx); + EphemeralSUnlock(sessionFunctions, ref stackCtx); } } - // No AggressiveInlining; this is a less-common function and it may improve inlining of InternalRead to have this be a virtcall. - private OperationStatus CopyFromImmutable(ref TKey key, ref TInput input, ref TOutput output, TContext userContext, + [MethodImpl(MethodImplOptions.NoInlining)] + private OperationStatus CopyFromImmutable(ref LogRecord srcLogRecord, ref TInput input, ref TOutput output, TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref OperationStatus status, TValue recordValue) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + ref OperationStackContext stackCtx, ref OperationStatus status) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.MainLog) { - status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, ref key, ref input, ref recordValue, ref output, userContext, ref stackCtx, - WriteReason.CopyToTail, wantIO: false); - if (status == OperationStatus.ALLOCATE_FAILED && pendingContext.IsAsync) // May happen due to CopyToTailFromReadOnly - CreatePendingReadContext(ref key, ref input, output, userContext, ref pendingContext, sessionFunctions, stackCtx.recSrc.LogicalAddress); + // Plumb source logical address so PostCopyToTail can name per-flush snapshot files. + pendingContext.originalAddress = stackCtx.recSrc.LogicalAddress; + status = ConditionalCopyToTail(sessionFunctions, ref pendingContext, in srcLogRecord, ref stackCtx, wantIO: false); return status; } - if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.ReadCache - && TryCopyToReadCache(sessionFunctions, ref pendingContext, ref key, ref input, ref recordValue, ref stackCtx)) + if (pendingContext.readCopyOptions.CopyTo == ReadCopyTo.ReadCache && TryCopyToReadCache(in srcLogRecord, sessionFunctions, ref pendingContext, ref stackCtx)) { // Copy to read cache is "best effort"; we don't return an error if it fails. return OperationStatus.SUCCESS | OperationStatus.COPIED_RECORD_TO_READ_CACHE; @@ -195,13 +184,15 @@ private OperationStatus CopyFromImmutable /// /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalReadAtAddress(long readAtAddress, ref TKey key, ref TInput input, ref TOutput output, + internal OperationStatus InternalReadAtAddress(long readAtAddress, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { if (readAtAddress < hlogBase.BeginAddress) return OperationStatus.NOTFOUND; - pendingContext.IsReadAtAddress = true; + pendingContext.SetIsReadAtAddress(); // We do things in a different order here than in InternalRead, in part to handle NoKey (especially with Revivification). if (readAtAddress < hlogBase.HeadAddress) { // Do not trace back in the pending callback if it is a key mismatch. - pendingContext.NoKey = true; + pendingContext.SetIsNoKey(); - CreatePendingReadContext(ref key, ref input, output, userContext, ref pendingContext, sessionFunctions, readAtAddress); + CreatePendingReadContext(key, ref input, ref output, userContext, ref pendingContext, sessionFunctions, readAtAddress); return OperationStatus.RECORD_ON_DISK; } // We're in-memory, so it is safe to get the address now. - var physicalAddress = hlog.GetPhysicalAddress(readAtAddress); + var srcLogRecord = hlog.CreateLogRecord(readAtAddress); - TKey defaultKey = default; + // Get the key hash. if (readOptions.KeyHash.HasValue) pendingContext.keyHash = readOptions.KeyHash.Value; - else if (!pendingContext.NoKey) - pendingContext.keyHash = storeFunctions.GetKeyHashCode64(ref key); + else if (!pendingContext.IsNoKey) + pendingContext.keyHash = storeFunctions.GetKeyHashCode64(key); else { // We have NoKey and an in-memory address so we must get the record to get the key to get the hashcode check for index growth, // possibly lock the bucket, etc. - pendingContext.keyHash = storeFunctions.GetKeyHashCode64(ref hlog.GetKey(physicalAddress)); - -#pragma warning disable CS9085 // "This ref-assigns a value that has a narrower escape scope than the target", but we don't return the reference. - // Note: With bucket-based locking the key is not used for Transient locks (only the key's hashcode is used). A key-based locking system - // would require this to be the actual key. We do *not* set this to the record key in case that is reclaimed by revivification. - key = ref defaultKey; -#pragma warning restore CS9085 + pendingContext.keyHash = storeFunctions.GetKeyHashCode64(srcLogRecord); } - OperationStackContext stackCtx = new(pendingContext.keyHash); + OperationStackContext stackCtx = new(pendingContext.keyHash); if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); - if (!FindTagAndTryTransientSLock(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (!FindTagAndTryEphemeralSLock(sessionFunctions, ref stackCtx, out var status)) return status; stackCtx.SetRecordSourceToHashEntry(hlogBase); @@ -299,19 +287,15 @@ internal OperationStatus InternalReadAtAddress= hlogBase.SafeReadOnlyAddress) - { - // Mutable region (even fuzzy region is included here). - sessionFunctions.ConcurrentReader(ref stackCtx.recSrc.GetKey(), ref input, ref stackCtx.recSrc.GetValue(), ref output, ref readInfo, ref srcRecordInfo); - } - else - { - // Immutable region (we tested for < HeadAddress above). - sessionFunctions.SingleReader(ref stackCtx.recSrc.GetKey(), ref input, ref stackCtx.recSrc.GetValue(), ref output, ref readInfo); - } + _ = sessionFunctions.Reader(in srcLogRecord, ref input, ref output, ref readInfo); } finally { stackCtx.HandleNewRecordOnException(this); - TransientSUnlock(sessionFunctions, ref key, ref stackCtx); + EphemeralSUnlock(sessionFunctions, ref stackCtx); } return status; } - // No AggressiveInlining; this is called only for the pending case and may improve inlining of InternalRead in the normal case if the compiler decides not to inline this. - private void CreatePendingReadContext(ref TKey key, ref TInput input, TOutput output, TContext userContext, + [MethodImpl(MethodImplOptions.NoInlining)] + private void CreatePendingReadContext(TKey key, ref TInput input, ref TOutput output, TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, long logicalAddress) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { pendingContext.type = OperationType.READ; - if (!pendingContext.NoKey && pendingContext.key == default) // If this is true, we don't have a valid key - pendingContext.key = hlog.GetKeyContainer(ref key); - if (pendingContext.input == default) - pendingContext.input = sessionFunctions.GetHeapContainer(ref input); - - pendingContext.output = output; - sessionFunctions.ConvertOutputToHeap(ref input, ref pendingContext.output); - - pendingContext.userContext = userContext; + pendingContext.CopyInputsForReadOrRMW(key, ref input, ref output, userContext, sessionFunctions, hlogBase.bufferPool); pendingContext.logicalAddress = logicalAddress; } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalUpsert.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalUpsert.cs index 6d487c660f9..7b012f7d405 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalUpsert.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/InternalUpsert.cs @@ -1,23 +1,26 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Diagnostics; using System.Runtime.CompilerServices; namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// - /// Upsert operation. Replaces the value corresponding to 'key' with provided 'value', if one exists - /// else inserts a new record with 'key' and 'value'. + /// Upsert operation. Replaces the value corresponding to 'key' with provided 'value', if one exists, else inserts a new record with 'key' and 'value'. /// /// key of the record. /// /// input used to update the value. - /// value to be updated to (or inserted if key does not exist). + /// String value to be updated to (or inserted if key does not exist); exclusive with . + /// String value to be updated to (or inserted if key does not exist); exclusive with . /// output where the result of the update can be placed /// User context for the operation, in case it goes pending. /// Pending context used internally to store the context of the operation. @@ -43,34 +46,42 @@ public unsafe partial class TsavoriteKV /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalUpsert(ref TKey key, long keyHash, ref TInput input, ref TValue value, ref TOutput output, + internal OperationStatus InternalUpsert(TKey key, long keyHash, ref TInput input, + ReadOnlySpan srcStringValue, IHeapObject srcObjectValue, in TSourceLogRecord inputLogRecord, ref TOutput output, ref TContext userContext, ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TValueSelector : IUpsertValueSelector + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - OperationStackContext stackCtx = new(keyHash); + OperationStackContext stackCtx = new(keyHash); pendingContext.keyHash = keyHash; + pendingContext.logicalAddress = kInvalidAddress; if (sessionFunctions.Ctx.phase == Phase.IN_PROGRESS_GROW) SplitBuckets(stackCtx.hei.hash); - if (!FindOrCreateTagAndTryTransientXLock(sessionFunctions, ref key, ref stackCtx, out OperationStatus status)) + if (!FindOrCreateTagAndTryEphemeralXLock(sessionFunctions, ref stackCtx, out OperationStatus status)) return status; - RecordInfo dummyRecordInfo = RecordInfo.InitialValid; - ref RecordInfo srcRecordInfo = ref dummyRecordInfo; + LogRecord srcLogRecord = default; UpsertInfo upsertInfo = new() { Version = sessionFunctions.Ctx.version, - SessionID = sessionFunctions.Ctx.sessionID + SessionID = sessionFunctions.Ctx.sessionID, + KeyHash = stackCtx.hei.hash }; // We must use try/finally to ensure unlocking even in the presence of exceptions. The inner try/finally ensures - // the transient X-lock on the hash bucket is released even if a Post*Operation callback (e.g. AOF append) throws. + // the ephemeral X-lock on the hash bucket is released even if a Post*Operation callback (e.g. AOF append) throws. try { // We blindly insert if the key isn't in the mutable region, so only check down to ReadOnlyAddress (minRevivifiableAddress is always >= ReadOnlyAddress). - if (!TryFindRecordForUpdate(ref key, ref stackCtx, hlogBase.ReadOnlyAddress, out status)) + if (!TryFindRecordForUpdate(key, ref stackCtx, hlogBase.ReadOnlyAddress, out status)) return status; // Note: Upsert does not track pendingContext.InitialAddress because we don't have an InternalContinuePendingUpsert @@ -78,7 +89,7 @@ internal OperationStatus InternalUpsert= hlogBase.ReadOnlyAddress) { - srcRecordInfo = ref stackCtx.recSrc.GetInfo(); + srcLogRecord = stackCtx.recSrc.CreateLogRecord(); // Mutable Region: Update the record in-place. We perform mutable updates only if we are in normal processing phase of checkpointing upsertInfo.Address = stackCtx.recSrc.LogicalAddress; - upsertInfo.KeyHash = stackCtx.hei.hash; - upsertInfo.SetRecordInfo(ref srcRecordInfo); - ref TValue recordValue = ref stackCtx.recSrc.GetValue(); - if (srcRecordInfo.Tombstone) + if (srcLogRecord.Info.Tombstone) { // If we're doing revivification and this is in the revivifiable range, try to revivify--otherwise we'll create a new record. if (RevivificationManager.IsEnabled && stackCtx.recSrc.LogicalAddress >= GetMinRevivifiableAddress()) { - if (TryRevivifyInChain(ref key, ref input, ref value, ref output, ref pendingContext, sessionFunctions, ref stackCtx, ref srcRecordInfo, ref upsertInfo, out status, ref recordValue) + if (TryRevivifyInChain( + ref srcLogRecord, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref pendingContext, sessionFunctions, ref stackCtx, ref upsertInfo, out status) || status != OperationStatus.SUCCESS) goto Done; } goto CreateNewRecord; } - // upsertInfo's lengths are filled in and GetValueLengths and SetLength are called inside ConcurrentWriter. - if (sessionFunctions.ConcurrentWriter(stackCtx.recSrc.PhysicalAddress, ref key, ref input, ref value, ref recordValue, ref output, ref upsertInfo, ref srcRecordInfo)) + // Track value heap delta across in-place write. + var sizeTracker = hlogBase.logSizeTracker; + var ipwPreHeap = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() : 0L; + var ipwResult = TValueSelector.InPlaceWriter( + ref srcLogRecord, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref upsertInfo, sessionFunctions); + var ipwDelta = sizeTracker is not null ? srcLogRecord.GetValueHeapMemorySize() - ipwPreHeap : 0L; + if (ipwResult) { - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); - pendingContext.recordInfo = srcRecordInfo; + if (ipwDelta != 0) + sizeTracker.IncrementSize(ipwDelta); + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + status = OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.InPlaceUpdatedRecord); goto Done; } @@ -137,29 +153,30 @@ internal OperationStatus InternalUpsert( + key, ref srcLogRecord, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref pendingContext, sessionFunctions, ref stackCtx, ref upsertInfo); + // We should never return "SUCCESS" for a new record operation: it returns NOTFOUND on success. + Debug.Assert(OperationStatusUtils.IsAppend(status) || OperationStatusUtils.BasicOpCode(status) != OperationStatus.SUCCESS); goto Done; } finally @@ -167,11 +184,12 @@ internal OperationStatus InternalUpsert(key, ref input, + srcStringValue, srcObjectValue, in inputLogRecord, ref upsertInfo, sessionFunctions, epoch); } finally { - TransientXUnlock(sessionFunctions, ref key, ref stackCtx); + EphemeralXUnlock(sessionFunctions, ref stackCtx); } } @@ -179,81 +197,56 @@ internal OperationStatus InternalUpsert(ref TKey key, ref TInput input, ref TValue value, TOutput output, TContext userContext, - ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private bool TryRevivifyInChain(ref LogRecord logRecord, ref TInput input, + ReadOnlySpan srcStringValue, IHeapObject srcObjectValue, in TSourceLogRecord inputLogRecord, ref TOutput output, ref PendingContext pendingContext, + TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref UpsertInfo upsertInfo, out OperationStatus status) + where TValueSelector : IUpsertValueSelector + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - pendingContext.type = OperationType.UPSERT; - if (pendingContext.key == default) - pendingContext.key = hlog.GetKeyContainer(ref key); - if (pendingContext.input == default) - pendingContext.input = sessionFunctions.GetHeapContainer(ref input); - if (pendingContext.value == default) - pendingContext.value = hlog.GetValueContainer(ref value); - - pendingContext.output = output; - sessionFunctions.ConvertOutputToHeap(ref input, ref pendingContext.output); - - pendingContext.userContext = userContext; - pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; - } - - private bool TryRevivifyInChain(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref PendingContext pendingContext, - TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, ref UpsertInfo upsertInfo, - out OperationStatus status, ref TValue recordValue) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - { - if (IsFrozen(sessionFunctions, ref stackCtx, ref srcRecordInfo)) + if (IsFrozen(sessionFunctions, ref stackCtx, logRecord.Info)) goto NeedNewRecord; // This record is safe to revivify even if its PreviousAddress points to a valid record, because it is revivified for the same key. - var ok = true; + var ok = false; try { - if (srcRecordInfo.Tombstone) - { - srcRecordInfo.ClearTombstone(); + logRecord.ClearOptionals(); + logRecord.InfoRef.ClearTombstone(); - if (RevivificationManager.IsFixedLength) - upsertInfo.UsedValueLength = upsertInfo.FullValueLength = RevivificationManager.FixedValueLength; - else - { - var recordLengths = GetRecordLengths(stackCtx.recSrc.PhysicalAddress, ref recordValue, ref srcRecordInfo); - upsertInfo.FullValueLength = recordLengths.fullValueLength; + var sizeInfo = TValueSelector.GetUpsertRecordSize(hlog, logRecord, srcStringValue, srcObjectValue, in inputLogRecord, ref input, sessionFunctions); - // Input is not included in record-length calculations for Upsert - var (requiredSize, _, _) = hlog.GetRecordSize(ref key, ref value); - (ok, upsertInfo.UsedValueLength) = TryReinitializeTombstonedValue(sessionFunctions, - ref srcRecordInfo, ref key, ref recordValue, requiredSize, recordLengths, stackCtx.recSrc.PhysicalAddress); - } + ref RevivificationStats stats = ref sessionFunctions.Ctx.RevivificationStats; - ref RevivificationStats stats = ref sessionFunctions.Ctx.RevivificationStats; - - if (ok && sessionFunctions.SingleWriter(ref key, ref input, ref value, ref recordValue, ref output, ref upsertInfo, WriteReason.Upsert, ref srcRecordInfo)) - { - sessionFunctions.PostSingleWriter(ref key, ref input, ref value, ref recordValue, ref output, ref upsertInfo, WriteReason.Upsert, ref srcRecordInfo); - // Success - MarkPage(stackCtx.recSrc.LogicalAddress, sessionFunctions.Ctx); - pendingContext.recordInfo = srcRecordInfo; - pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; - // Return NOTFOUND OperationStatus to indicate that the operation was successful but a previous record was not found. - status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.InPlaceUpdatedRecord); - stats.inChainSuccesses++; - return true; - } - - // Did not revivify; restore the tombstone and leave the deleted record there. - srcRecordInfo.SetTombstone(); - stats.inChainFailures++; + // Type arg specification is needed because we don't pass TContext + ok = TValueSelector.InitialWriter( + ref logRecord, in sizeInfo, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref upsertInfo, sessionFunctions); + if (ok) + { + TValueSelector.PostInitialWriter( + ref logRecord, in sizeInfo, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref upsertInfo, sessionFunctions); + + // Track revived record's heap — key was already tracked when the tombstone was created, + // but value heap was subtracted at OnDispose(Deleted). Re-add it now. + var valueHeap = logRecord.GetValueHeapMemorySize(); + if (valueHeap != 0) + hlogBase.logSizeTracker?.IncrementSize(valueHeap); + + // Success + pendingContext.logicalAddress = stackCtx.recSrc.LogicalAddress; + + // Return NOTFOUND OperationStatus to indicate that the operation was successful but a previous record was not found. + status = OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.InPlaceUpdatedRecord); + stats.inChainSuccesses++; + return true; } + // Did not revivify; restore the tombstone and leave the deleted record there. + stats.inChainFailures++; } finally { - if (ok) - SetExtraValueLength(ref recordValue, ref srcRecordInfo, upsertInfo.UsedValueLength, upsertInfo.FullValueLength); - else - SetTombstoneAndExtraValueLength(ref recordValue, ref srcRecordInfo, upsertInfo.UsedValueLength, upsertInfo.FullValueLength); // Restore tombstone and ensure default value on inability to update in place + if (!ok) + logRecord.InfoRef.SetTombstone(); } NeedNewRecord: @@ -262,7 +255,7 @@ private bool TryRevivifyInChain stackCtx, ref OperationStatus status) + private LatchDestination CheckCPRConsistencyUpsert(Phase phase, ref OperationStackContext stackCtx, ref OperationStatus status) { // See explanatory comments in CheckCPRConsistencyRMW. @@ -289,99 +282,117 @@ private LatchDestination CheckCPRConsistencyUpsert(Phase phase, ref OperationSta } /// - /// Create a new record for Upsert + /// Create a new record for Upsert from a source Key, Value, and Input /// /// The record Key + /// The source record, if . and + /// it is either too small or is in readonly region, or is in readcache /// Input to the operation - /// The value to insert - /// The result of ISessionFunctions.SingleWriter + /// String value to be set to; exclusive with and . + /// String value to be set to; exclusive with and . + /// Log record to be copied from; exclusive with and . + /// The result of ISessionFunctions operation /// Information about the operation context /// The current session - /// Contains the and structures for this operation, + /// Contains the and structures for this operation, /// and allows passing back the newLogicalAddress for invalidation in the case of exceptions. - /// If ., - /// this is the for /// UpsertInfo - private OperationStatus CreateNewRecordUpsert(ref TKey key, ref TInput input, ref TValue value, ref TOutput output, - ref PendingContext pendingContext, TSessionFunctionsWrapper sessionFunctions, - ref OperationStackContext stackCtx, ref RecordInfo srcRecordInfo, ref UpsertInfo upsertInfo) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private OperationStatus CreateNewRecordUpsert(TKey key, ref LogRecord srcLogRecord, ref TInput input, + ReadOnlySpan srcStringValue, IHeapObject srcObjectValue, in TSourceLogRecord inputLogRecord, ref TOutput output, ref PendingContext pendingContext, + TSessionFunctionsWrapper sessionFunctions, ref OperationStackContext stackCtx, ref UpsertInfo upsertInfo) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TValueSelector : IUpsertValueSelector + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - var (actualSize, allocatedSize, keySize) = hlog.GetUpsertRecordSize(ref key, ref value, ref input, sessionFunctions); + var sizeInfo = TValueSelector.GetUpsertRecordSize(hlog, key, srcStringValue, srcObjectValue, in inputLogRecord, ref input, sessionFunctions); AllocateOptions allocOptions = new() { - Recycle = true, - ElideSourceRecord = stackCtx.recSrc.HasMainLogSrc && CanElide(sessionFunctions, ref stackCtx, ref srcRecordInfo) + recycle = true, + elideSourceRecord = stackCtx.recSrc.HasMainLogSrc && CanElide(sessionFunctions, ref stackCtx, srcLogRecord.Info) }; - if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, actualSize, ref allocatedSize, keySize, allocOptions, - out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status)) + if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, ref sizeInfo, allocOptions, out var newLogicalAddress, out var newPhysicalAddress, out var status)) return status; - ref RecordInfo newRecordInfo = ref WriteNewRecordInfo(ref key, hlogBase, newPhysicalAddress, inNewVersion: sessionFunctions.Ctx.InNewVersion, stackCtx.recSrc.LatestLogicalAddress); - if (allocOptions.ElideSourceRecord) - newRecordInfo.PreviousAddress = srcRecordInfo.PreviousAddress; + var newLogRecord = WriteNewRecordInfo(key, hlogBase, newLogicalAddress, newPhysicalAddress, in sizeInfo, sessionFunctions.Ctx.InNewVersion, previousAddress: stackCtx.recSrc.LatestLogicalAddress); + if (allocOptions.elideSourceRecord) + newLogRecord.InfoRef.PreviousAddress = srcLogRecord.Info.PreviousAddress; stackCtx.SetNewRecord(newLogicalAddress); upsertInfo.Address = newLogicalAddress; upsertInfo.KeyHash = stackCtx.hei.hash; - upsertInfo.SetRecordInfo(ref newRecordInfo); - ref TValue newRecordValue = ref hlog.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize); - (upsertInfo.UsedValueLength, upsertInfo.FullValueLength) = GetNewValueLengths(actualSize, allocatedSize, newPhysicalAddress, ref newRecordValue); - - if (!sessionFunctions.SingleWriter(ref key, ref input, ref value, ref newRecordValue, ref output, ref upsertInfo, WriteReason.Upsert, ref newRecordInfo)) + // Type arg specification is needed because we don't pass TContext + var success = TValueSelector.InitialWriter( + ref newLogRecord, in sizeInfo, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref upsertInfo, sessionFunctions); + if (!success) { // Save allocation for revivification (not retry, because these aren't retry status codes), or abandon it if that fails. - if (RevivificationManager.UseFreeRecordPool && RevivificationManager.TryAdd(newLogicalAddress, newPhysicalAddress, allocatedSize, ref sessionFunctions.Ctx.RevivificationStats)) - stackCtx.ClearNewRecord(); - else - stackCtx.SetNewRecordInvalid(ref newRecordInfo); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + if (!RevivificationManager.UseFreeRecordPool || !TryTransferToFreeList(sessionFunctions, newLogicalAddress, ref newLogRecord)) + OnDispose(ref newLogRecord, DisposeReason.InsertAbandoned); if (upsertInfo.Action == UpsertAction.CancelOperation) return OperationStatus.CANCELED; return OperationStatus.NOTFOUND; // But not CreatedRecord } - SetExtraValueLength(ref newRecordValue, ref newRecordInfo, upsertInfo.UsedValueLength, upsertInfo.FullValueLength); - // Insert the new record by CAS'ing either directly into the hash entry or splicing into the readcache/mainlog boundary. // If the current record can be elided then we can freelist it; detach it by swapping its .PreviousAddress into newRecordInfo. - bool success = CASRecordIntoChain(ref key, ref stackCtx, newLogicalAddress, ref newRecordInfo); + success = CASRecordIntoChain(newLogicalAddress, ref newLogRecord, ref stackCtx); if (success) { - PostCopyToTail(ref key, ref stackCtx, ref srcRecordInfo); + // Track key overflow internally — session functions only track in-place value deltas. + if (newLogRecord.Info.KeyIsOverflow) + hlogBase.logSizeTracker?.IncrementSize(newLogRecord.KeyOverflow.HeapMemorySize); + + PostCopyToTail(in srcLogRecord, ref stackCtx); - sessionFunctions.PostSingleWriter(ref key, ref input, ref value, ref newRecordValue, ref output, ref upsertInfo, WriteReason.Upsert, ref newRecordInfo); + // Type arg specification is needed because we don't pass TContext + TValueSelector.PostInitialWriter( + ref newLogRecord, in sizeInfo, ref input, srcStringValue, srcObjectValue, in inputLogRecord, ref output, ref upsertInfo, sessionFunctions); + + // Track new record's value heap — after PostInitialWriter so the value is fully populated. + { + var valueHeap = newLogRecord.GetValueHeapMemorySize(); + if (valueHeap != 0) + hlogBase.logSizeTracker?.IncrementSize(valueHeap); + } // ElideSourceRecord means we have verified that the old source record is elidable and now that CAS has replaced it in the HashBucketEntry with // the new source record that does not point to the old source record, we have elided it, so try to transfer to freelist. - if (allocOptions.ElideSourceRecord) + if (allocOptions.elideSourceRecord) { // Success should always Seal the old record. This may be readcache, readonly, or the temporary recordInfo, which is OK and saves the cost of an "if". - srcRecordInfo.SealAndInvalidate(); // The record was elided, so Invalidate + srcLogRecord.InfoRef.SealAndInvalidate(); // The record was elided, so Invalidate if (stackCtx.recSrc.LogicalAddress >= GetMinRevivifiableAddress()) - { - // We need to re-get the old record's length because upsertInfo has the new record's info. If freelist-add fails, it remains Sealed/Invalidated. - var oldRecordLengths = GetRecordLengths(stackCtx.recSrc.PhysicalAddress, ref hlog.GetValue(stackCtx.recSrc.PhysicalAddress), ref srcRecordInfo); - TryTransferToFreeList(sessionFunctions, ref stackCtx, ref srcRecordInfo, oldRecordLengths); - } + _ = TryTransferToFreeList(sessionFunctions, stackCtx.recSrc.LogicalAddress, ref srcLogRecord); + else + OnDispose(ref srcLogRecord, DisposeReason.Elided); } else - srcRecordInfo.Seal(); // The record was not elided, so do not Invalidate + { + // If it is in mutable or fuzzy region, we must Seal + if (stackCtx.recSrc.HasMainLogSrc && stackCtx.recSrc.LogicalAddress > hlogBase.SafeReadOnlyAddress) + srcLogRecord.InfoRef.Seal(); // The record was not elided, so do not Invalidate + } stackCtx.ClearNewRecord(); - pendingContext.recordInfo = newRecordInfo; pendingContext.logicalAddress = newLogicalAddress; + return OperationStatusUtils.AdvancedOpCode(OperationStatus.NOTFOUND, StatusCode.CreatedRecord); } // CAS failed - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - storeFunctions.DisposeRecord(ref hlog.GetKey(newPhysicalAddress), ref hlog.GetValue(newPhysicalAddress), DisposeReason.SingleWriterCASFailed); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + OnDispose(ref newLogRecord, DisposeReason.InitialWriterCASFailed); - SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress, allocatedSize); + SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress); return OperationStatus.RETRY_NOW; // CAS failure does not require epoch refresh } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/ILockTable.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/ILockTable.cs index ec3b7c9df89..b8747b12b1a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/ILockTable.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/ILockTable.cs @@ -6,13 +6,12 @@ namespace Tsavorite.core { /// - /// Manual-enabled (both manual and transient) LockTable interface definition + /// LockTable interface definition (for both Transactional and Ephemeral) /// - /// - public interface ILockTable : IDisposable + public interface ILockTable : IDisposable { /// - /// Try to acquire a manual lock for the key. + /// Whether the lock table is enabled. /// public bool IsEnabled { get; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/OverflowBucketLockTable.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/OverflowBucketLockTable.cs index 3d4143847e5..d8840be86e3 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/OverflowBucketLockTable.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/OverflowBucketLockTable.cs @@ -7,17 +7,17 @@ namespace Tsavorite.core { - internal struct OverflowBucketLockTable : ILockTable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal struct OverflowBucketLockTable : ILockTable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - private readonly TsavoriteKV store; + private readonly TsavoriteKV store; internal readonly long NumBuckets => store.state[store.resizeInfo.version].size_mask + 1; public readonly bool IsEnabled => store is not null; - internal OverflowBucketLockTable(TsavoriteKV store) => this.store = store; + internal OverflowBucketLockTable(TsavoriteKV store) => this.store = store; internal readonly long GetSize() => store.state[store.resizeInfo.version].size_mask; @@ -83,16 +83,16 @@ public unsafe LockState GetLockState(ref HashEntryInfo hei) IsLockedExclusive = HashBucket.IsLatchedExclusive(hei.firstBucket) }; - private static int KeyHashComparer(TLockableKey key1, TLockableKey key2, long size_mask) - where TLockableKey : ILockableKey + private static int KeyHashComparer(TTransactionalKey key1, TTransactionalKey key2, long size_mask) + where TTransactionalKey : ITransactionalKey { var idx1 = GetBucketIndex(key1.KeyHash, size_mask); var idx2 = GetBucketIndex(key2.KeyHash, size_mask); return (idx1 != idx2) ? idx1.CompareTo(idx2) : ((byte)key1.LockType).CompareTo((byte)key2.LockType); } - private static int KeyHashComparer(ref TLockableKey key1, ref TLockableKey key2, long size_mask) - where TLockableKey : ILockableKey + private static int KeyHashComparer(ref TTransactionalKey key1, ref TTransactionalKey key2, long size_mask) + where TTransactionalKey : ITransactionalKey { var idx1 = GetBucketIndex(key1.KeyHash, size_mask); var idx2 = GetBucketIndex(key2.KeyHash, size_mask); @@ -100,31 +100,31 @@ private static int KeyHashComparer(ref TLockableKey key1, ref TLoc } /// - internal readonly int CompareKeyHashes(TLockableKey key1, TLockableKey key2) - where TLockableKey : ILockableKey + internal readonly int CompareKeyHashes(TTransactionalKey key1, TTransactionalKey key2) + where TTransactionalKey : ITransactionalKey => KeyHashComparer(key1, key2, store.state[store.resizeInfo.version].size_mask); /// - internal readonly int CompareKeyHashes(ref TLockableKey key1, ref TLockableKey key2) - where TLockableKey : ILockableKey + internal readonly int CompareKeyHashes(ref TTransactionalKey key1, ref TTransactionalKey key2) + where TTransactionalKey : ITransactionalKey => KeyHashComparer(ref key1, ref key2, store.state[store.resizeInfo.version].size_mask); /// - internal readonly void SortKeyHashes(Span keys) - where TLockableKey : ILockableKey - => keys.Sort(new KeyComparer(store.state[store.resizeInfo.version].size_mask)); + internal readonly void SortKeyHashes(Span keys) + where TTransactionalKey : ITransactionalKey + => keys.Sort(new KeyComparer(store.state[store.resizeInfo.version].size_mask)); /// /// Need this struct because the Comparison{T} form of Array.Sort is not available with start and length arguments. /// - struct KeyComparer : IComparer - where TLockableKey : ILockableKey + struct KeyComparer : IComparer + where TTransactionalKey : ITransactionalKey { readonly long size_mask; internal KeyComparer(long s) => size_mask = s; - public int Compare(TLockableKey key1, TLockableKey key2) => KeyHashComparer(key1, key2, size_mask); + public int Compare(TTransactionalKey key1, TTransactionalKey key2) => KeyHashComparer(key1, key2, size_mask); } /// diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/TransientLocking.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/TransientLocking.cs index 34ccc80660e..b3ee4ce7aba 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/TransientLocking.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Locking/TransientLocking.cs @@ -6,17 +6,17 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool TryTransientXLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx, + private bool TryEphemeralXLock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx, out OperationStatus status) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - if (sessionFunctions.TryLockTransientExclusive(ref key, ref stackCtx)) + if (sessionFunctions.TryLockEphemeralExclusive(ref stackCtx)) { status = OperationStatus.SUCCESS; return true; @@ -26,21 +26,21 @@ private bool TryTransientXLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + private static void EphemeralXUnlock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - if (stackCtx.recSrc.HasTransientXLock) - sessionFunctions.UnlockTransientExclusive(ref key, ref stackCtx); + if (stackCtx.recSrc.HasEphemeralXLock) + sessionFunctions.UnlockEphemeralExclusive(ref stackCtx); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryTransientSLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx, + internal bool TryEphemeralSLock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx, out OperationStatus status) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - if (sessionFunctions.TryLockTransientShared(ref key, ref stackCtx)) + if (sessionFunctions.TryLockEphemeralShared(ref stackCtx)) { status = OperationStatus.SUCCESS; return true; @@ -50,36 +50,40 @@ internal bool TryTransientSLock(TSessionFunctionsWrapper sessionFunctions, ref TKey key, - ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal static void EphemeralSUnlock(TSessionFunctionsWrapper sessionFunctions, + ref OperationStackContext stackCtx) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - if (stackCtx.recSrc.HasTransientSLock) - sessionFunctions.UnlockTransientShared(ref key, ref stackCtx); + if (stackCtx.recSrc.HasEphemeralSLock) + sessionFunctions.UnlockEphemeralShared(ref stackCtx); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void LockForScan(ref OperationStackContext stackCtx, ref TKey key) + internal void LockForScan(ref OperationStackContext stackCtx, TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { Debug.Assert(!stackCtx.recSrc.HasLock, $"Should not call LockForScan if recSrc already has a lock ({stackCtx.recSrc.LockStateString()})"); - // This will always be a transient lock as it is not session-based - stackCtx = new(storeFunctions.GetKeyHashCode64(ref key)); + // This will always be an Ephemeral lock as it is not session-based + stackCtx = new(storeFunctions.GetKeyHashCode64(key)); _ = FindTag(ref stackCtx.hei); stackCtx.SetRecordSourceToHashEntry(hlogBase); while (!LockTable.TryLockShared(ref stackCtx.hei)) epoch.ProtectAndDrain(); - stackCtx.recSrc.SetHasTransientSLock(); + stackCtx.recSrc.SetHasEphemeralSLock(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void UnlockForScan(ref OperationStackContext stackCtx) + internal void UnlockForScan(ref OperationStackContext stackCtx) { - if (stackCtx.recSrc.HasTransientSLock) + if (stackCtx.recSrc.HasEphemeralSLock) { LockTable.UnlockShared(ref stackCtx.hei); - stackCtx.recSrc.ClearHasTransientSLock(); + stackCtx.recSrc.ClearHasEphemeralSLock(); } } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ModifiedBitOperation.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ModifiedBitOperation.cs index b7aacffc3af..7c0eb93bc1f 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ModifiedBitOperation.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ModifiedBitOperation.cs @@ -6,9 +6,9 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// if reset is true it simply resets the modified bit for the key @@ -18,24 +18,27 @@ public unsafe partial class TsavoriteKVRecordInfo of the key for checkModified. /// Operation Type, whether it is reset or check [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal OperationStatus InternalModifiedBitOperation(ref TKey key, out RecordInfo modifiedInfo, bool reset = true) + internal OperationStatus InternalModifiedBitOperation(TKey key, out RecordInfo modifiedInfo, bool reset = true) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { Debug.Assert(epoch.ThisInstanceProtected()); - HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(ref key)); ; + HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(key)); ; #region Trace back for record in in-memory HybridLog _ = FindTag(ref hei); var logicalAddress = hei.Address; - var physicalAddress = hlog.GetPhysicalAddress(logicalAddress); if (logicalAddress >= hlogBase.HeadAddress) { - ref RecordInfo recordInfo = ref hlog.GetInfo(physicalAddress); - if (recordInfo.Invalid || !storeFunctions.KeysEqual(ref key, ref hlog.GetKey(physicalAddress))) + var logRecord = hlog.CreateLogRecord(logicalAddress); + if (logRecord.Info.Invalid || !storeFunctions.KeysEqual(key, logRecord)) { - logicalAddress = recordInfo.PreviousAddress; - TraceBackForKeyMatch(ref key, logicalAddress, hlogBase.HeadAddress, out logicalAddress, out physicalAddress); + logicalAddress = logRecord.Info.PreviousAddress; + TraceBackForKeyMatch(key, logicalAddress, hlogBase.HeadAddress, out logicalAddress, out _); } } #endregion @@ -43,7 +46,7 @@ internal OperationStatus InternalModifiedBitOperation(ref TKey key, out RecordIn modifiedInfo = default; if (logicalAddress >= hlogBase.HeadAddress) { - ref RecordInfo recordInfo = ref hlog.GetInfo(physicalAddress); + ref var recordInfo = ref LogRecord.GetInfoRef(hlogBase.GetPhysicalAddress(logicalAddress)); if (reset) { if (!recordInfo.TryResetModifiedAtomic()) diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/OperationStackContext.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/OperationStackContext.cs index bbe24ff24a4..8285fd5f156 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/OperationStackContext.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/OperationStackContext.cs @@ -6,13 +6,15 @@ namespace Tsavorite.core { - public struct OperationStackContext - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public struct OperationStackContext + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Note: Cannot use ref fields because they are not supported before net7.0. internal HashEntryInfo hei; - internal RecordSource recSrc; + internal RecordSource recSrc; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal OperationStackContext(long keyHash) => hei = new(keyHash); @@ -23,21 +25,21 @@ public struct OperationStackContext /// /// The TsavoriteKV's hlog [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetRecordSourceToHashEntry(AllocatorBase srcLog) => recSrc.Set(hei.Address, srcLog); + internal void SetRecordSourceToHashEntry(AllocatorBase srcLog) => recSrc.Set(hei.Address, srcLog); /// /// Sets to the current ., which is the current address /// in the hash table. This is the same effect as calling . /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void UpdateRecordSourceToCurrentHashEntry(AllocatorBase hlog) + internal void UpdateRecordSourceToCurrentHashEntry(AllocatorBase hlog) { hei.SetToCurrent(); SetRecordSourceToHashEntry(hlog); } /// - /// If this is not , it is the logical Address allocated by CreateNewRecord*; if an exception + /// If this is not , it is the logical Address allocated by CreateNewRecord*; if an exception /// occurs, this needs to be set invalid and non-tentative by the caller's 'finally' (to avoid another try/finally overhead). /// private long newLogicalAddress; @@ -56,25 +58,25 @@ internal void SetNewRecordInvalid(ref RecordInfo newRecordInfo) { Debug.Assert(newRecordInfo.Invalid, "Records should be invalidated until successfully inserted"); // TODO: If this does not fire, remove the following line that sets it newRecordInfo.SetInvalid(); - newLogicalAddress = Constants.kInvalidAddress; + newLogicalAddress = kInvalidAddress; } /// /// Called during normal operations when a record insertion succeeds, to set the new record non-tentative (permanent). /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ClearNewRecord() => newLogicalAddress = Constants.kInvalidAddress; + internal void ClearNewRecord() => newLogicalAddress = kInvalidAddress; /// /// Called during InternalXxx 'finally' handler, to set the new record invalid if an exception or other error occurred. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void HandleNewRecordOnException(TsavoriteKV store) + internal void HandleNewRecordOnException(TsavoriteKV store) { - if (newLogicalAddress != Constants.kInvalidAddress) + if (newLogicalAddress != kInvalidAddress) { store.SetRecordInvalid(newLogicalAddress); - newLogicalAddress = Constants.kInvalidAddress; + newLogicalAddress = kInvalidAddress; } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ReadCache.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ReadCache.cs index 77824d80be3..84e9ac62dc3 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ReadCache.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/ReadCache.cs @@ -3,22 +3,28 @@ using System.Diagnostics; using System.Runtime.CompilerServices; -using static Tsavorite.core.Utility; namespace Tsavorite.core { +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + // Partial file for readcache functions - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.NoInlining)] - internal bool FindInReadCache(ref TKey key, ref OperationStackContext stackCtx, long minAddress = Constants.kInvalidAddress, bool alwaysFindLatestLA = true) + internal bool FindInReadCache(TKey key, ref OperationStackContext stackCtx, long minAddress = kInvalidAddress, bool alwaysFindLatestLA = true) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { Debug.Assert(UseReadCache, "Should not call FindInReadCache if !UseReadCache"); // minAddress, if present, comes from the pre-pendingIO entry.Address; there may have been no readcache entries then. - minAddress = IsReadCache(minAddress) ? AbsoluteAddress(minAddress) : readCacheBase.HeadAddress; + minAddress = IsReadCache(minAddress) ? AbsoluteAddress(minAddress) : readcacheBase.HeadAddress; RestartChain: @@ -27,43 +33,53 @@ internal bool FindInReadCache(ref TKey key, ref OperationStackContext= minAddress && !stackCtx.recSrc.HasReadCacheSrc - && storeFunctions.KeysEqual(ref key, ref readcache.GetKey(stackCtx.recSrc.LowestReadCachePhysicalAddress))) + if (!recordInfo.Invalid && stackCtx.recSrc.LatestLogicalAddress >= minAddress && !stackCtx.recSrc.HasReadCacheSrc) { - // Keep these at the current readcache location; they'll be the caller's source record. - stackCtx.recSrc.LogicalAddress = stackCtx.recSrc.LowestReadCacheLogicalAddress; - stackCtx.recSrc.PhysicalAddress = stackCtx.recSrc.LowestReadCachePhysicalAddress; - stackCtx.recSrc.SetHasReadCacheSrc(); - stackCtx.recSrc.SetAllocator(readCacheBase); - - // Read() does not need to continue past the found record; updaters need to continue to find latestLogicalAddress and lowestReadCache*Address. - if (!alwaysFindLatestLA) - return true; + // TODO: Can we avoid always creating the log record here? + var logRecord = readcache.CreateLogRecord(stackCtx.recSrc.LowestReadCacheLogicalAddress); + if (storeFunctions.KeysEqual(key, logRecord)) + { + // Keep these at the current readcache location; they'll be the caller's source record. + stackCtx.recSrc.LogicalAddress = stackCtx.recSrc.LowestReadCacheLogicalAddress; + stackCtx.recSrc.PhysicalAddress = stackCtx.recSrc.LowestReadCachePhysicalAddress; + stackCtx.recSrc.SetAllocator(readcacheBase); + stackCtx.recSrc.SetHasReadCacheSrc(); + + // Read() does not need to continue past the found record; updaters need to continue to find latestLogicalAddress and lowestReadCache*Address. + if (!alwaysFindLatestLA) + return true; + } + } + + // If a readcache record was evicted while we were processing it here, its .PreviousAddress will be kTempInvalidAddress. + // This should not be the case otherwise; we should always find a valid main-log address after the readcache prefix chain. + if (recordInfo.PreviousAddress <= kTempInvalidAddress) + { + _ = ReadCacheNeedToWaitForEviction(ref stackCtx); + goto RestartChain; } // Update the leading LatestLogicalAddress to recordInfo.PreviousAddress, and if that is a main log record, break out. - stackCtx.recSrc.LatestLogicalAddress = recordInfo.PreviousAddress & ~Constants.kReadCacheBitMask; - if (!IsReadCache(recordInfo.PreviousAddress)) + stackCtx.recSrc.LatestLogicalAddress = recordInfo.PreviousAddress; + if (!IsReadCache(stackCtx.recSrc.LatestLogicalAddress)) goto InMainLog; } @@ -73,18 +89,21 @@ internal bool FindInReadCache(ref TKey key, ref OperationStackContext Constants.kTempInvalidAddress, "Must have a main-log address after readcache"); + Debug.Assert(stackCtx.recSrc.LatestLogicalAddress > kTempInvalidAddress, "Must have a main-log address after readcache"); stackCtx.recSrc.LogicalAddress = stackCtx.recSrc.LatestLogicalAddress; stackCtx.recSrc.PhysicalAddress = 0; // do *not* call hlog.GetPhysicalAddress(); LogicalAddress may be below hlog.HeadAddress. Let the caller decide when to do this. return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool ReadCacheNeedToWaitForEviction(ref OperationStackContext stackCtx) + bool ReadCacheNeedToWaitForEviction(ref OperationStackContext stackCtx) { - if (stackCtx.recSrc.LatestLogicalAddress < readCacheBase.HeadAddress) + Debug.Assert(stackCtx.hei.IsReadCache, "Can only call ReadCacheNeedToWaitForEviction when hei.IsReadCache is true"); + Debug.Assert(IsReadCache(stackCtx.recSrc.LatestLogicalAddress), "Can only call ReadCacheNeedToWaitForEviction when stackCtx.recSrc.LatestLogicalAddress is readcache"); + var logicalAddress = AbsoluteAddress(stackCtx.recSrc.LatestLogicalAddress); + if (logicalAddress < readcacheBase.HeadAddress) { - SpinWaitUntilRecordIsClosed(stackCtx.recSrc.LatestLogicalAddress, readCacheBase); + SpinWaitUntilRecordIsClosed(logicalAddress, readcacheBase); // Restore to hlog; we may have set readcache into Log and continued the loop, had to restart, and the matching readcache record was evicted. stackCtx.UpdateRecordSourceToCurrentHashEntry(hlogBase); @@ -94,21 +113,21 @@ bool ReadCacheNeedToWaitForEviction(ref OperationStackContext stackCtx, long newLogicalAddress) + private bool SpliceIntoHashChainAtReadCacheBoundary(ref OperationStackContext stackCtx, long newLogicalAddress) { // Splice into the gap of the last readcache/first main log entries. - Debug.Assert(stackCtx.recSrc.LowestReadCacheLogicalAddress >= readCacheBase.ClosedUntilAddress, - $"{nameof(VerifyInMemoryAddresses)} should have ensured LowestReadCacheLogicalAddress ({stackCtx.recSrc.LowestReadCacheLogicalAddress}) >= readcache.ClosedUntilAddress ({readCacheBase.ClosedUntilAddress})"); + Debug.Assert(stackCtx.recSrc.LowestReadCacheLogicalAddress >= readcacheBase.ClosedUntilAddress, + $"{nameof(VerifyInMemoryAddresses)} should have ensured LowestReadCacheLogicalAddress ({stackCtx.recSrc.LowestReadCacheLogicalAddress}) >= readcache.ClosedUntilAddress ({readcacheBase.ClosedUntilAddress})"); // If the LockTable is enabled, then we either have an exclusive lock and thus cannot have a competing insert to the readcache, or we are doing a // Read() so we allow a momentary overlap of records because they're the same value (no update is being done). - ref RecordInfo rcri = ref readcache.GetInfo(stackCtx.recSrc.LowestReadCachePhysicalAddress); + ref var rcri = ref LogRecord.GetInfoRef(stackCtx.recSrc.LowestReadCachePhysicalAddress); return rcri.TryUpdateAddress(stackCtx.recSrc.LatestLogicalAddress, newLogicalAddress); } // Skip over all readcache records in this key's chain, advancing stackCtx.recSrc to the first non-readcache record we encounter. [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SkipReadCache(ref OperationStackContext stackCtx, out bool didRefresh) + internal void SkipReadCache(ref OperationStackContext stackCtx, out bool didRefresh) { Debug.Assert(UseReadCache, "Should not call SkipReadCache if !UseReadCache"); didRefresh = false; @@ -119,11 +138,9 @@ internal void SkipReadCache(ref OperationStackContextbucket_entries[index]; + var entry = (HashBucketEntry*)&bucket->bucket_entries[index]; if (0 == entry->word) continue; - if (!entry->ReadCache) continue; + if (!entry->IsReadCache) continue; var logicalAddress = entry->Address; - var physicalAddress = readcache.GetPhysicalAddress(AbsoluteAddress(logicalAddress)); + var physicalAddress = readcacheBase.GetPhysicalAddress(logicalAddress); while (true) { - logicalAddress = readcache.GetInfo(physicalAddress).PreviousAddress; + logicalAddress = LogRecord.GetInfo(physicalAddress).PreviousAddress; entry->Address = logicalAddress; - if (!entry->ReadCache) + if (!entry->IsReadCache) break; - physicalAddress = readcache.GetPhysicalAddress(AbsoluteAddress(logicalAddress)); + physicalAddress = readcacheBase.GetPhysicalAddress(logicalAddress); } } } // Called after a readcache insert, to make sure there was no race with another session that added a main-log record at the same time. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool EnsureNoNewMainLogRecordWasSpliced(ref TKey key, RecordSource recSrc, long highestSearchedAddress, ref OperationStatus failStatus) + private bool EnsureNoNewMainLogRecordWasSpliced(TKey key, ref OperationStackContext stackCtx, long highestSearchedAddress, ref OperationStatus failStatus) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - bool success = true; - ref RecordInfo lowest_rcri = ref readcache.GetInfo(recSrc.LowestReadCachePhysicalAddress); + Debug.Assert(!IsReadCache(highestSearchedAddress), "highestSearchedAddress should be a main-log address"); + var success = true; + Debug.Assert(AbsoluteAddress(stackCtx.recSrc.LowestReadCacheLogicalAddress) >= readcacheBase.ClosedUntilAddress, "recSrc.LowestReadCachePhysicalAddress should be above ClosedUntilAddress"); + var lowest_rcri = LogRecord.GetInfo(stackCtx.recSrc.LowestReadCachePhysicalAddress); Debug.Assert(!IsReadCache(lowest_rcri.PreviousAddress), "lowest-rcri.PreviousAddress should be a main-log address"); if (lowest_rcri.PreviousAddress > highestSearchedAddress) { // Someone added a new record in the splice region. It won't be readcache; that would've been added at tail. See if it's our key. var minAddress = highestSearchedAddress > hlogBase.HeadAddress ? highestSearchedAddress : hlogBase.HeadAddress; - if (TraceBackForKeyMatch(ref key, lowest_rcri.PreviousAddress, minAddress + 1, out long prevAddress, out _)) + if (TraceBackForKeyMatch(key, lowest_rcri.PreviousAddress, minAddress + 1, out var prevAddress, out _)) success = false; else if (prevAddress > highestSearchedAddress && prevAddress < hlogBase.HeadAddress) { @@ -206,7 +229,11 @@ private bool EnsureNoNewMainLogRecordWasSpliced(ref TKey key, RecordSource(TKey key, ref HashEntryInfo hei, long highestReadCacheAddressChecked) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { Debug.Assert(UseReadCache, "Should not call ReadCacheCheckTailAfterSplice if !UseReadCache"); @@ -215,11 +242,11 @@ private void ReadCacheCheckTailAfterSplice(ref TKey key, ref HashEntryInfo hei, HashBucketEntry untilEntry = new() { word = highestReadCacheAddressChecked }; // Traverse for the key above untilAddress (which may not be in the readcache if there were no readcache records when it was retrieved). - while (entry.ReadCache && (entry.Address > untilEntry.Address || !untilEntry.ReadCache)) + while (entry.IsReadCache && (!!untilEntry.IsReadCache || entry.Address > untilEntry.Address)) { - var physicalAddress = readcache.GetPhysicalAddress(entry.AbsoluteAddress); - ref RecordInfo recordInfo = ref readcache.GetInfo(physicalAddress); - if (!recordInfo.Invalid && storeFunctions.KeysEqual(ref key, ref readcache.GetKey(physicalAddress))) + var logRecord = readcache.CreateLogRecord(entry.Address); + ref var recordInfo = ref logRecord.InfoRef; + if (!recordInfo.Invalid && storeFunctions.KeysEqual(key, logRecord)) { recordInfo.SetInvalidAtomic(); return; @@ -232,26 +259,31 @@ private void ReadCacheCheckTailAfterSplice(ref TKey key, ref HashEntryInfo hei, } [MethodImpl(MethodImplOptions.AggressiveInlining)] - void ReadCacheAbandonRecord(long physicalAddress) + static void ReadCacheAbandonRecord(long physicalAddress) { // TODO: We currently don't save readcache allocations for retry, but we could - ref var ri = ref readcache.GetInfo(physicalAddress); + ref var ri = ref LogRecord.GetInfoRef(physicalAddress); ri.SetInvalid(); - ri.PreviousAddress = Constants.kTempInvalidAddress; // Necessary for ReadCacheEvict, but cannot be kInvalidAddress or we have recordInfo.IsNull + ri.PreviousAddress = kTempInvalidAddress; // Necessary for ReadCacheEvict, but cannot be kInvalidAddress or we have recordInfo.IsNull } internal void ReadCacheEvict(long rcLogicalAddress, long rcToLogicalAddress) { // Iterate readcache entries in the range rcFrom/ToLogicalAddress, and remove them from the hash chain. + // First make sure we're not trying to process a logical address that's in a page header. + var offsetOnPage = readcacheBase.GetOffsetOnPage(rcLogicalAddress); + if (offsetOnPage < PageHeader.Size) + rcLogicalAddress += PageHeader.Size - offsetOnPage; + while (rcLogicalAddress < rcToLogicalAddress) { - var rcPhysicalAddress = readcache.GetPhysicalAddress(rcLogicalAddress); - var (_, rcAllocatedSize) = readcache.GetRecordSize(rcPhysicalAddress); - var rcRecordInfo = readcache.GetInfo(rcPhysicalAddress); + var logRecord = new LogRecord(readcacheBase.GetPhysicalAddress(rcLogicalAddress)); + var rcAllocatedSize = logRecord.AllocatedSize; + var rcRecordInfo = logRecord.Info; // Check PreviousAddress for null to handle the info.IsNull() "partial record at end of page" case as well as readcache CAS failures // (such failed records are not in the hash chain, so we must not process them here). We do process other Invalid records here. - if (rcRecordInfo.PreviousAddress <= Constants.kTempInvalidAddress) + if (rcRecordInfo.PreviousAddress <= kTempInvalidAddress) goto NextRecord; // If there are any readcache entries for this key, the hash chain will always be of the form: @@ -267,17 +299,16 @@ internal void ReadCacheEvict(long rcLogicalAddress, long rcToLogicalAddress) Debug.Assert(!IsReadCache(rcRecordInfo.PreviousAddress) || AbsoluteAddress(rcRecordInfo.PreviousAddress) < rcLogicalAddress, "Invalid record ordering in readcache"); // Find the hash index entry for the key in the store's hash table. - ref TKey key = ref readcache.GetKey(rcPhysicalAddress); - HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(ref key)); + HashEntryInfo hei = new(storeFunctions.GetKeyHashCode64(logRecord)); if (!FindTag(ref hei)) goto NextRecord; ReadCacheEvictChain(rcToLogicalAddress, ref hei); NextRecord: - if ((rcLogicalAddress & readCacheBase.PageSizeMask) + rcAllocatedSize > readCacheBase.PageSize) + if (readcacheBase.GetOffsetOnPage(rcLogicalAddress) + rcAllocatedSize >= readcacheBase.PageSize - RecordInfo.Size) { - rcLogicalAddress = (1 + (rcLogicalAddress >> readCacheBase.LogPageSizeBits)) << readCacheBase.LogPageSizeBits; + rcLogicalAddress = readcacheBase.GetFirstValidLogicalAddressOnPage(1 + readcacheBase.GetPage(rcLogicalAddress)); continue; } rcLogicalAddress += rcAllocatedSize; @@ -289,44 +320,49 @@ private void ReadCacheEvictChain(long rcToLogicalAddress, ref HashEntryInfo hei) // Traverse the chain of readcache entries for this key, looking "ahead" to .PreviousAddress to see if it is less than readcache.HeadAddress. // nextPhysicalAddress remains Constants.kInvalidAddress if hei.Address is < HeadAddress; othrwise, it is the lowest-address readcache record // remaining following this eviction, and its .PreviousAddress is updated to each lower record in turn until we hit a non-readcache record. - long nextPhysicalAddress = Constants.kInvalidAddress; + var nextPhysicalAddress = kInvalidAddress; HashBucketEntry entry = new() { word = hei.entry.word }; - while (entry.ReadCache) + while (entry.IsReadCache) { - var la = entry.AbsoluteAddress; - var pa = readcache.GetPhysicalAddress(la); - ref RecordInfo ri = ref readcache.GetInfo(pa); + var logRecord = new LogRecord(readcacheBase.GetPhysicalAddress(entry.Address)); + ref var recordInfo = ref logRecord.InfoRef; #if DEBUG // Due to collisions, we can compare the hash code *mask* (i.e. the hash bucket index), not the key var mask = state[resizeInfo.version].size_mask; var rc_mask = hei.hash & mask; - var pa_mask = storeFunctions.GetKeyHashCode64(ref readcache.GetKey(pa)) & mask; + var pa_mask = storeFunctions.GetKeyHashCode64(logRecord) & mask; Debug.Assert(rc_mask == pa_mask, "The keyHash mask of the hash-chain ReadCache entry does not match the one obtained from the initial readcache address"); #endif // If the record's address is above the eviction range, leave it there and track nextPhysicalAddress. - if (la >= rcToLogicalAddress) + if (AbsoluteAddress(entry.Address) >= rcToLogicalAddress) { - nextPhysicalAddress = pa; - entry.word = ri.PreviousAddress; + Debug.Assert(!IsReadCache(recordInfo.PreviousAddress) || entry.Address > recordInfo.PreviousAddress, "Invalid ordering in readcache chain"); + + nextPhysicalAddress = logRecord.physicalAddress; + entry.word = recordInfo.PreviousAddress; continue; } - // The record is being evicted. If we have a higher readcache record that is not being evicted, unlink 'la' by setting - // (nextPhysicalAddress).PreviousAddress to (la).PreviousAddress. - if (nextPhysicalAddress != Constants.kInvalidAddress) + // The record is being evicted. If we have a higher readcache record that is not being evicted, unlink 'entry.Address' by setting + // (nextPhysicalAddress).PreviousAddress to (entry.Address).PreviousAddress. + if (nextPhysicalAddress != kInvalidAddress) { - ref RecordInfo nextri = ref readcache.GetInfo(nextPhysicalAddress); - if (nextri.TryUpdateAddress(entry.Address, ri.PreviousAddress)) - ri.PreviousAddress = Constants.kTempInvalidAddress; // The record is no longer in the chain - entry.word = nextri.PreviousAddress; + ref var nextri = ref LogRecord.GetInfoRef(nextPhysicalAddress); + if (nextri.TryUpdateAddress(entry.Address, recordInfo.PreviousAddress)) + { + recordInfo.PreviousAddress = kTempInvalidAddress; // The record is no longer in the chain + entry.word = nextri.PreviousAddress; + } + else + Debug.Assert(entry.word == nextri.PreviousAddress, "We should be about to retry nextri.PreviousAddress"); continue; } // We are evicting the record whose address is in the hash bucket; unlink 'la' by setting the hash bucket to point to (la).PreviousAddress. - if (hei.TryCAS(ri.PreviousAddress)) - ri.PreviousAddress = Constants.kTempInvalidAddress; // The record is no longer in the chain + if (hei.TryCAS(recordInfo.PreviousAddress)) + recordInfo.PreviousAddress = kTempInvalidAddress; // The record is no longer in the chain else hei.SetToCurrent(); entry.word = hei.entry.word; diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/RecordSource.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/RecordSource.cs index 5b8e0b2b6c8..fd4fa1b2330 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/RecordSource.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/RecordSource.cs @@ -1,19 +1,21 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System.Diagnostics; using System.Runtime.CompilerServices; -using static Tsavorite.core.Utility; namespace Tsavorite.core { + using static LogAddress; + /// /// Carries various addresses and accompanying values corresponding to source records for the current InternalXxx or InternalContinuePendingR* /// operations, where "source" is a copy source for RMW and/or a locked record. This is passed to functions that create records, such as /// TsavoriteKV.CreateNewRecord*() or TsavoriteKV.InternalTryCopyToTail(), and to unlocking utilities. /// - internal struct RecordSource - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal struct RecordSource + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// If valid, this is the logical address of a record. As "source", it may be copied from for RMW or pending Reads, @@ -54,15 +56,16 @@ internal struct RecordSource /// /// If , this is the allocator base (hlog or readcache) that is in. /// - internal AllocatorBase AllocatorBase { get; private set; } + internal AllocatorBase AllocatorBase { get; private set; } struct InternalStates { internal const int None = 0; - internal const int TransientSLock = 0x0001; // LockTable - internal const int TransientXLock = 0x0002; // LockTable - internal const int LockBits = TransientSLock | TransientXLock; + internal const int EphemeralSLock = 0x0001; // LockTable + internal const int EphemeralXLock = 0x0002; // LockTable + internal const int LockBits = EphemeralSLock | EphemeralXLock; + // These are separate from the AddressType in LogicalAddress because we need to know if that LogicalAddress matched the key. internal const int MainLogSrc = 0x0100; internal const int ReadCacheSrc = 0x0200; internal const int InMemSrcBits = MainLogSrc | ReadCacheSrc; @@ -79,13 +82,13 @@ void append(int value, string name) if ((state & value) != 0) { if (sb.Length > 0) - sb.Append(", "); - sb.Append(name); + _ = sb.Append(", "); + _ = sb.Append(name); } } - append(TransientSLock, nameof(TransientSLock)); - append(TransientXLock, nameof(TransientXLock)); + append(EphemeralSLock, nameof(EphemeralSLock)); + append(EphemeralXLock, nameof(EphemeralXLock)); append(MainLogSrc, nameof(MainLogSrc)); append(ReadCacheSrc, nameof(ReadCacheSrc)); return sb.ToString(); @@ -95,22 +98,22 @@ void append(int value, string name) int internalState; /// - /// Set (and cleared) by caller to indicate whether we have a LockTable-based Transient Shared lock (does not include Manual locks; this is per-operation only). + /// Set (and cleared) by caller to indicate whether we have a LockTable-based Ephemeral Shared lock (does not include Manual locks; this is per-operation only). /// - internal readonly bool HasTransientSLock => (internalState & InternalStates.TransientSLock) != 0; + internal readonly bool HasEphemeralSLock => (internalState & InternalStates.EphemeralSLock) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetHasTransientSLock() => internalState |= InternalStates.TransientSLock; + internal void SetHasEphemeralSLock() => internalState |= InternalStates.EphemeralSLock; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ClearHasTransientSLock() => internalState &= ~InternalStates.TransientSLock; + internal void ClearHasEphemeralSLock() => internalState &= ~InternalStates.EphemeralSLock; /// - /// Set (and cleared) by caller to indicate whether we have a LockTable-based Transient Exclusive lock (does not include Manual locks; this is per-operation only). + /// Set (and cleared) by caller to indicate whether we have a LockTable-based Ephemeral Exclusive lock (does not include Manual locks; this is per-operation only). /// - internal readonly bool HasTransientXLock => (internalState & InternalStates.TransientXLock) != 0; + internal readonly bool HasEphemeralXLock => (internalState & InternalStates.EphemeralXLock) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetHasTransientXLock() => internalState |= InternalStates.TransientXLock; + internal void SetHasEphemeralXLock() => internalState |= InternalStates.EphemeralXLock; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ClearHasTransientXLock() => internalState &= ~InternalStates.TransientXLock; + internal void ClearHasEphemeralXLock() => internalState &= ~InternalStates.EphemeralXLock; /// /// Indicates whether we have any type of non-Manual lock. @@ -122,65 +125,72 @@ void append(int value, string name) /// internal readonly bool HasMainLogSrc => (internalState & InternalStates.MainLogSrc) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetHasMainLogSrc() => internalState |= InternalStates.MainLogSrc; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ClearHasMainLogSrc() => internalState &= ~InternalStates.MainLogSrc; + internal void SetHasMainLogSrc() + { + Debug.Assert(!IsReadCache(LogicalAddress), "LogicalAddress must be a main log address to set HasMainLogSrc"); + internalState |= InternalStates.MainLogSrc; + } /// /// Set by caller to indicate whether the is an in-memory record in the readcache, being used as a copy source and/or a lock. /// internal readonly bool HasReadCacheSrc => (internalState & InternalStates.ReadCacheSrc) != 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetHasReadCacheSrc() => internalState |= InternalStates.ReadCacheSrc; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ClearHasReadCacheSrc() => internalState &= ~InternalStates.ReadCacheSrc; + internal void SetHasReadCacheSrc() + { + Debug.Assert(IsReadCache(LogicalAddress), "LogicalAddress must be a readcache address to set HasReadCacheSrc"); + internalState |= InternalStates.ReadCacheSrc; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal long SetPhysicalAddress() => PhysicalAddress = Allocator.GetPhysicalAddress(LogicalAddress); + internal long SetPhysicalAddress() => PhysicalAddress = AllocatorBase.GetPhysicalAddress(LogicalAddress); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal readonly ref RecordInfo GetInfo() => ref Allocator.GetInfo(PhysicalAddress); + internal readonly ref RecordInfo GetInfoRef() => ref LogRecord.GetInfoRef(PhysicalAddress); [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal readonly ref TKey GetKey() => ref Allocator.GetKey(PhysicalAddress); + internal readonly RecordInfo GetInfo() => LogRecord.GetInfo(PhysicalAddress); + [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal readonly ref TValue GetValue() => ref Allocator.GetValue(PhysicalAddress); + internal readonly LogRecord CreateLogRecord() + { + // If we have a physical address we must be in the in-memory log. + Debug.Assert(PhysicalAddress != 0, "Cannot CreateLogRecord until PhysicalAddress is set"); + return Allocator.CreateLogRecord(LogicalAddress, PhysicalAddress); + } internal readonly bool HasInMemorySrc => (internalState & (InternalStates.MainLogSrc | InternalStates.ReadCacheSrc)) != 0; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ClearHasInMemorySrc() => internalState &= ~(InternalStates.MainLogSrc | InternalStates.ReadCacheSrc); + /// /// Initialize to the latest logical address from the caller. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void Set(long latestLogicalAddress, AllocatorBase srcAllocatorBase) + internal void Set(long latestLogicalAddress, AllocatorBase srcAllocatorBase) { PhysicalAddress = default; LowestReadCacheLogicalAddress = default; LowestReadCachePhysicalAddress = default; - ClearHasMainLogSrc(); - ClearHasReadCacheSrc(); + ClearHasInMemorySrc(); - // HasTransientLock = ...; Do not clear this; it is in the LockTable and must be preserved until unlocked + // DO NOT clear locks; we call SetRecordSourceToHashEntry() after we've acquired the lock. - LatestLogicalAddress = LogicalAddress = AbsoluteAddress(latestLogicalAddress); + LatestLogicalAddress = LogicalAddress = latestLogicalAddress; SetAllocator(srcAllocatorBase); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetAllocator(AllocatorBase srcAllocatorBase) + internal void SetAllocator(AllocatorBase srcAllocatorBase) { - this.AllocatorBase = srcAllocatorBase; - this.Allocator = AllocatorBase._wrapper; + AllocatorBase = srcAllocatorBase; + Allocator = AllocatorBase._wrapper; } [MethodImpl(MethodImplOptions.AggressiveInlining)] internal readonly string LockStateString() => InternalStates.ToString(internalState & InternalStates.LockBits); public override readonly string ToString() - { - var isRC = "(rc)"; - var llaRC = IsReadCache(LatestLogicalAddress) ? isRC : string.Empty; - var laRC = IsReadCache(LogicalAddress) ? isRC : string.Empty; - return $"lla {AbsoluteAddress(LatestLogicalAddress)}{llaRC}, la {AbsoluteAddress(LogicalAddress)}{laRC}, lrcla {AbsoluteAddress(LowestReadCacheLogicalAddress)}," - + $" hasInMemorySrc {InternalStates.ToString(internalState & InternalStates.InMemSrcBits)}, hasLocks {LockStateString()}"; - } + => $"lla {AddressString(LatestLogicalAddress)}, la {AddressString(LogicalAddress)}, lrcla {AddressString(LowestReadCacheLogicalAddress)}," + + $" inMemSrc {InternalStates.ToString(internalState & InternalStates.InMemSrcBits)}, locks {LockStateString()}"; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/CheckEmptyWorker.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/CheckEmptyWorker.cs index 8d3bfca58b3..1376454f77b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/CheckEmptyWorker.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/CheckEmptyWorker.cs @@ -8,9 +8,9 @@ namespace Tsavorite.core { - internal sealed class CheckEmptyWorker - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class CheckEmptyWorker + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // State control variables. internal struct State @@ -33,9 +33,9 @@ internal static string ToString(long state) CancellationTokenSource cts = new(); - readonly FreeRecordPool recordPool; + readonly FreeRecordPool recordPool; - internal CheckEmptyWorker(FreeRecordPool recordPool) => this.recordPool = recordPool; + internal CheckEmptyWorker(FreeRecordPool recordPool) => this.recordPool = recordPool; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal unsafe void Start() @@ -54,7 +54,7 @@ private async void LaunchWorker() { try { - await Task.Delay(1000, cts.Token); + await Task.Delay(1000, cts.Token).ConfigureAwait(false); if (disposed) break; recordPool.ScanForEmpty(cts.Token); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/FreeRecordPool.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/FreeRecordPool.cs index 92cea4dacf1..8fa9ac1d936 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/FreeRecordPool.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/FreeRecordPool.cs @@ -7,50 +7,57 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; -using static Tsavorite.core.Utility; namespace Tsavorite.core { +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + using static Utility; + [StructLayout(LayoutKind.Explicit, Size = sizeof(long))] internal struct FreeRecord { - internal const int kSizeBits = 64 - RecordInfo.kPreviousAddressBits; - const int kSizeShiftInWord = RecordInfo.kPreviousAddressBits; +#pragma warning disable IDE1006 // Naming Styles + internal const int kSizeBits = 64 - kAddressBits; // 16 currently + const int kSizeShiftInWord = kAddressBits; const long kSizeMask = RevivificationBin.MaxInlineRecordSize - 1; const long kSizeMaskInWord = kSizeMask << kSizeShiftInWord; +#pragma warning restore IDE1006 // Naming Styles // This is the empty word we replace the current word with on Reads. - private const long emptyWord = 0; + private const long EmptyWord = 0; - // 'word' contains the reclaimable logicalAddress and the size of the record at that address. + // 'word' contains the reclaimable logicalAddress and the size of the recordPtr at that address. [FieldOffset(0)] private long word; internal const int StructSize = sizeof(long); + /// LogicalAddress of the recordPtr. public long Address { - readonly get => word & RecordInfo.kPreviousAddressMaskInWord; - set => word = (word & ~RecordInfo.kPreviousAddressMaskInWord) | (value & RecordInfo.kPreviousAddressMaskInWord); + readonly get => word & kAddressBitMask; + set => word = (word & ~kAddressBitMask) | (value & kAddressBitMask); } + /// Inline size of the recordPtr. May contain overflow allocations. public readonly int Size => (int)((word & kSizeMaskInWord) >> kSizeShiftInWord); /// public override readonly string ToString() => $"address {Address}, size {Size}"; - internal readonly bool IsSet => word != emptyWord; + internal readonly bool IsSet => word != EmptyWord; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool Set(long address, long recordSize, long minAddress) { - // If the record is empty or the address is below minAddress, set the new address into it. + // If the recordPtr is empty or the address is below minAddress, set the new address into it. var oldRecord = this; if (oldRecord.IsSet && oldRecord.Address >= minAddress) return false; - long newWord = (recordSize << kSizeShiftInWord) | (address & RecordInfo.kPreviousAddressMaskInWord); + var newWord = (recordSize << kSizeShiftInWord) | (address & kAddressBitMask); return Interlocked.CompareExchange(ref word, newWord, oldRecord.word) == oldRecord.word; } @@ -58,15 +65,15 @@ internal bool Set(long address, long recordSize, long minAddress) void SetEmptyAtomic(long oldWord) { // Ignore the result; this is just to clear an obsolete value, so if another thread already updated it, that's by design. - Interlocked.CompareExchange(ref word, emptyWord, oldWord); + _ = Interlocked.CompareExchange(ref word, EmptyWord, oldWord); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryPeek(long recordSize, TsavoriteKV store, bool oversize, long minAddress, out int thisRecordSize) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal bool TryPeek(long recordSize, TsavoriteKV store, bool oversize, long minAddress, out int thisRecordSize) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - FreeRecord oldRecord = this; + var oldRecord = this; thisRecordSize = 0; if (!oldRecord.IsSet) return false; @@ -75,6 +82,7 @@ internal bool TryPeek(long recordSize SetEmptyAtomic(oldRecord.word); return false; } + var thisSize = oversize ? GetRecordSize(store, oldRecord.Address) : oldRecord.Size; if (thisSize < recordSize) return false; @@ -107,11 +115,11 @@ internal readonly void MergeTo(ref RevivificationStats revivStats) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal bool TryTake(int recordSize, long minAddress, out long address, ref TakeResult takeResult) + internal bool TryTake(int actualInlineRecordSize, long minAddress, out long address, ref TakeResult takeResult) { address = 0; - FreeRecord oldRecord = this; + var oldRecord = this; while (true) { if (!oldRecord.IsSet) @@ -119,45 +127,43 @@ internal bool TryTake(int recordSize, long minAddress, out long address, ref Tak takeResult.isEmpty = false; if (oldRecord.Address < minAddress) return false; - else - takeResult.addressOk = true; - if (oldRecord.Size < recordSize) + + takeResult.addressOk = true; + if (oldRecord.Size < actualInlineRecordSize) return false; - else - takeResult.recordSizeOk = true; + takeResult.recordSizeOk = true; - // If we're here, the record was set and size and address were adequate. - if (Interlocked.CompareExchange(ref word, emptyWord, oldRecord.word) == oldRecord.word) + // If we're here, the recordPtr was set and size and address were adequate. + if (Interlocked.CompareExchange(ref word, EmptyWord, oldRecord.word) == oldRecord.word) { address = oldRecord.Address; return true; } - // Failed to CAS. Loop again to see if someone else put in a different, but still good, record. + // Failed to CAS. Loop again to see if someone else put in a different, but still good, recordPtr. oldRecord = this; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetRecordSize(TsavoriteKV store, long logicalAddress) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + private static int GetRecordSize(TsavoriteKV store, long logicalAddress) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - // Because this is oversize, we need hlog to get the length out of the record's value (it won't fit in FreeRecord.kSizeBits) - long physicalAddress = store.hlog.GetPhysicalAddress(logicalAddress); - return store.GetFreeRecordSize(physicalAddress, ref store.hlog.GetInfo(physicalAddress)); + // This is called for oversize, so we need hlog to get the length out of the recordPtr (it won't fit in FreeRecord.kSizeBits) + return LogRecord.GetAllocatedSize(store.hlogBase.GetPhysicalAddress(logicalAddress)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal unsafe bool TryTakeOversize(long recordSize, long minAddress, TsavoriteKV store, out long address, ref TakeResult takeResult) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal bool TryTakeOversize(int actualInlineRecordSize, long minAddress, TsavoriteKV store, out long address, ref TakeResult takeResult) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { address = 0; // The difference in this oversize version is that we delay checking size until after the CAS, because we have // go go the slow route of getting the physical address. - FreeRecord oldRecord = this; + var oldRecord = this; while (true) { if (!oldRecord.IsSet) @@ -165,26 +171,22 @@ internal unsafe bool TryTakeOversize( takeResult.isEmpty = false; if (oldRecord.Address < minAddress) return false; - else - takeResult.addressOk = true; + takeResult.addressOk = true; - // Because this is oversize, we need hlog to get the length out of the record's value (it won't fit in FreeRecord.kSizeBits) - long physicalAddress = store.hlog.GetPhysicalAddress(oldRecord.Address); - long thisSize = store.GetFreeRecordSize(physicalAddress, ref store.hlog.GetInfo(physicalAddress)); - - if (thisSize < recordSize) + // Because this is oversize, we need hlog to get the length out of the recordPtr's value (it won't fit in FreeRecord.kSizeBits) + long thisSize = GetRecordSize(store, oldRecord.Address); + if (thisSize < actualInlineRecordSize) return false; - else - takeResult.recordSizeOk = true; + takeResult.recordSizeOk = true; - // If we're here, the record was set and size and address were adequate. - if (Interlocked.CompareExchange(ref word, emptyWord, oldRecord.word) == oldRecord.word) + // If we're here, the recordPtr was set and size and address were adequate. + if (Interlocked.CompareExchange(ref word, EmptyWord, oldRecord.word) == oldRecord.word) { address = oldRecord.Address; return true; } - // Failed to CAS. Loop again to see if someone else put in a different, but still good, record. + // Failed to CAS. Loop again to see if someone else put in a different, but still good, recordPtr. oldRecord = this; } } @@ -210,7 +212,7 @@ internal unsafe class FreeRecordBin /// public override string ToString() { - string scanStr = bestFitScanLimit switch + var scanStr = bestFitScanLimit switch { RevivificationBin.BestFitScanAll => "ScanAll", RevivificationBin.UseFirstFit => "FirstFit", @@ -219,28 +221,28 @@ public override string ToString() return $"isEmpty {isEmpty}, recSizes {minRecordSize}..{maxRecordSize}, recSizeInc {segmentRecordSizeIncrement}, #recs {recordCount}; segments: segSize {segmentSize}, #segs {segmentCount}; scanLimit {scanStr}"; } - internal FreeRecordBin(ref RevivificationBin binDef, int prevBinRecordSize, bool isFixedLength) + internal FreeRecordBin(ref RevivificationBin binDef, int prevBinRecordSize) { - // If the record size range is too much for the number of records in the bin, we must allow multiple record sizes per segment. - // prevBinRecordSize is already verified to be a multiple of 8. - var bindefRecordSize = RoundUp(binDef.RecordSize, 8); - if (isFixedLength || bindefRecordSize == prevBinRecordSize + 8) + // If the recordPtr size range is too much for the number of records in the bin, we must allow multiple recordPtr sizes per segment. + // prevBinRecordSize is already verified to be a multiple of Constants.kRecordAlignment. + var bindefRecordSize = RoundUp(binDef.RecordSize, Constants.kRecordAlignment); + if (bindefRecordSize == prevBinRecordSize + Constants.kRecordAlignment) { bestFitScanLimit = RevivificationBin.UseFirstFit; segmentSize = RoundUp(binDef.NumberOfRecords, MinSegmentSize); segmentCount = 1; segmentRecordSizeIncrement = 1; // For the division and multiplication in GetSegmentStart - minRecordSize = maxRecordSize = isFixedLength ? prevBinRecordSize : bindefRecordSize; + minRecordSize = maxRecordSize = bindefRecordSize; } else { bestFitScanLimit = binDef.BestFitScanLimit; - // minRecordSize is already verified to be a multiple of 8. + // minRecordSize is already verified to be a multiple of Constants.kRecordAlignment. var sizeRange = bindefRecordSize - prevBinRecordSize; - segmentCount = sizeRange / 8; + segmentCount = sizeRange / Constants.kRecordAlignment; segmentSize = (int)Math.Ceiling(binDef.NumberOfRecords / (double)segmentCount); if (segmentSize >= MinSegmentSize) @@ -251,15 +253,15 @@ internal FreeRecordBin(ref RevivificationBin binDef, int prevBinRecordSize, bool segmentCount = (int)Math.Ceiling(binDef.NumberOfRecords / (double)segmentSize); } - segmentRecordSizeIncrement = RoundUp(sizeRange / segmentCount, 8); - maxRecordSize = prevBinRecordSize + segmentRecordSizeIncrement * segmentCount; + segmentRecordSizeIncrement = RoundUp(sizeRange / segmentCount, Constants.kRecordAlignment); + maxRecordSize = prevBinRecordSize + (segmentRecordSizeIncrement * segmentCount); minRecordSize = prevBinRecordSize + segmentRecordSizeIncrement; } recordCount = segmentSize * segmentCount; // Overallocate the GCHandle by one cache line so we have room to offset the returned pointer to make it cache-aligned. - recordsArray = GC.AllocateArray(recordCount + Constants.kCacheLineBytes / FreeRecord.StructSize, pinned: true); - long p = (long)Unsafe.AsPointer(ref recordsArray[0]); + recordsArray = GC.AllocateArray(recordCount + (Constants.kCacheLineBytes / FreeRecord.StructSize), pinned: true); + var p = (long)Unsafe.AsPointer(ref recordsArray[0]); // Force the pointer to align to cache boundary. records = (FreeRecord*)RoundUp(p, Constants.kCacheLineBytes); @@ -270,8 +272,7 @@ internal FreeRecordBin(ref RevivificationBin binDef, int prevBinRecordSize, bool [MethodImpl(MethodImplOptions.AggressiveInlining)] internal int GetSegmentStart(int recordSize) { - // recordSize and segmentSizeIncrement are rounded up to 8, unless IsFixedLength in which case segmentSizeIncrement is 1. - // sizeOffset will be negative if we are searching the next-highest bin. + // recordSize and segmentSizeIncrement are rounded up to Constants.kRecordAlignment. sizeOffset will be negative if we are searching the next-highest bin. var sizeOffset = recordSize - minRecordSize; if (sizeOffset < 0) sizeOffset = 0; @@ -284,16 +285,16 @@ internal int GetSegmentStart(int recordSize) private FreeRecord* GetRecord(int recordIndex) => records + (recordIndex >= recordCount ? recordIndex - recordCount : recordIndex); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryAdd(long address, int recordSize, TsavoriteKV store, long minAddress, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public bool TryAdd(long logicalAddress, int recordSize, TsavoriteKV store, long minAddress, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { var segmentStart = GetSegmentStart(recordSize); - for (var ii = 0; ii < recordCount; ++ii) + for (var ii = 0; ii < recordCount; ii++) { - FreeRecord* record = GetRecord(segmentStart + ii); - if (record->Set(address, recordSize, minAddress)) + var recordPtr = GetRecord(segmentStart + ii); + if (recordPtr->Set(logicalAddress, recordSize, minAddress)) { ++revivStats.successfulAdds; isEmpty = false; @@ -305,41 +306,46 @@ public bool TryAdd(long address, int } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTake(int recordSize, long minAddress, TsavoriteKV store, out long address, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => TryTake(recordSize, minAddress, store, oversize: false, out address, ref revivStats); + public bool TryTake(int actualInlineRecordSize, long minAddress, TsavoriteKV store, out long address, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => TryTake(actualInlineRecordSize, minAddress, store, oversize: false, out address, ref revivStats); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTake(int recordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public bool TryTake(int actualInlineRecordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { if (isEmpty) { address = 0; return false; } - return (bestFitScanLimit == RevivificationBin.UseFirstFit) - ? TryTakeFirstFit(recordSize, minAddress, store, oversize, out address, ref revivStats) - : TryTakeBestFit(recordSize, minAddress, store, oversize, out address, ref revivStats); + var result = (bestFitScanLimit == RevivificationBin.UseFirstFit) + ? TryTakeFirstFit(actualInlineRecordSize, minAddress, store, oversize, out address, ref revivStats) + : TryTakeBestFit(actualInlineRecordSize, minAddress, store, oversize, out address, ref revivStats); + if (result) + ++revivStats.successfulTakes; + else + ++revivStats.failedTakes; + return result; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTakeFirstFit(int recordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public bool TryTakeFirstFit(int actualInlineRecordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - var segmentStart = GetSegmentStart(recordSize); + var segmentStart = GetSegmentStart(actualInlineRecordSize); - int retryCount = recordCount; + var retryCount = recordCount; FreeRecord.TakeResult takeResult = new(); while (true) { - for (var ii = 0; ii < recordCount; ++ii) + for (var ii = 0; ii < recordCount; ii++) { - FreeRecord* record = GetRecord(segmentStart + ii); - if (oversize ? record->TryTakeOversize(recordSize, minAddress, store, out address, ref takeResult) : record->TryTake(recordSize, minAddress, out address, ref takeResult)) + var recordPtr = GetRecord(segmentStart + ii); + if (oversize ? recordPtr->TryTakeOversize(actualInlineRecordSize, minAddress, store, out address, ref takeResult) : recordPtr->TryTake(actualInlineRecordSize, minAddress, out address, ref takeResult)) { takeResult.MergeTo(ref revivStats); return true; @@ -355,27 +361,27 @@ public bool TryTakeFirstFit(int recor } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTakeBestFit(int recordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public bool TryTakeBestFit(int actualInlineRecordSize, long minAddress, TsavoriteKV store, bool oversize, out long address, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Retry as long as we find a candidate, but reduce the best fit scan limit each retry. - int localBestFitScanLimit = bestFitScanLimit; - var segmentStart = GetSegmentStart(recordSize); + var localBestFitScanLimit = bestFitScanLimit; + var segmentStart = GetSegmentStart(actualInlineRecordSize); FreeRecord.TakeResult takeResult = new(); while (true) { - int bestFitSize = int.MaxValue; // Comparison is "if record.Size < bestFitSize", hence initialized to int.MaxValue - int bestFitIndex = -1; // Will be compared to >= 0 on exit from the best-fit scan loop - int firstFitIndex = int.MaxValue; // Subtracted from loop control var and tested for >= bestFitScanLimit; int.MaxValue produces a negative result + var bestFitSize = int.MaxValue; // Comparison is "if recordPtr.Size < bestFitSize", hence initialized to int.MaxValue + var bestFitIndex = -1; // Will be compared to >= 0 on exit from the best-fit scan loop + var firstFitIndex = int.MaxValue; // Subtracted from loop control var and tested for >= bestFitScanLimit; int.MaxValue produces a negative result FreeRecord* record; - for (var ii = 0; ii < recordCount; ++ii) + for (var ii = 0; ii < recordCount; ii++) { // For best-fit we must peek first without taking. record = GetRecord(segmentStart + ii); - if (record->TryPeek(recordSize, store, oversize, minAddress, out var thisRecordSize)) + if (record->TryPeek(actualInlineRecordSize, store, oversize, minAddress, out var thisRecordSize)) { bestFitIndex = ii; // Found exact match break; @@ -400,7 +406,7 @@ record = GetRecord(segmentStart + ii); } record = GetRecord(segmentStart + bestFitIndex); - if (oversize ? record->TryTakeOversize(recordSize, minAddress, store, out address, ref takeResult) : record->TryTake(recordSize, minAddress, out address, ref takeResult)) + if (oversize ? record->TryTakeOversize(actualInlineRecordSize, minAddress, store, out address, ref takeResult) : record->TryTake(actualInlineRecordSize, minAddress, out address, ref takeResult)) { takeResult.MergeTo(ref revivStats); return true; @@ -409,22 +415,22 @@ record = GetRecord(segmentStart + bestFitIndex); // We found a candidate but CAS failed. Reduce the best fit scan length and continue. localBestFitScanLimit /= 2; if (localBestFitScanLimit <= 1) - return TryTakeFirstFit(recordSize, minAddress, store, oversize, out address, ref revivStats); + return TryTakeFirstFit(actualInlineRecordSize, minAddress, store, oversize, out address, ref revivStats); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void ScanForEmpty(FreeRecordPool recordPool, CancellationToken cancellationToken) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal void ScanForEmpty(FreeRecordPool recordPool, CancellationToken cancellationToken) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // Add() always sets isEmpty to false and we do not clear isEmpty on Take() because that could lead to more lost "isEmpty = false". // So this routine is called only if the bin is marked not-empty. - for (var ii = 0; ii < recordCount; ++ii) + for (var ii = 0; ii < recordCount; ii++) { if (cancellationToken.IsCancellationRequested) break; - FreeRecord record = *(records + ii); + var record = *(records + ii); if (record.IsSet) { // Still not empty; only the CheckEmptyWorker thread should set this.isEmpty, so the value should not have changed. @@ -437,60 +443,52 @@ internal void ScanForEmpty(FreeRecord } } - internal unsafe class FreeRecordPool : IDisposable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal unsafe class FreeRecordPool : IDisposable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - internal readonly TsavoriteKV store; + internal readonly TsavoriteKV store; internal readonly FreeRecordBin[] bins; internal int numberOfBinsToSearch; - internal bool IsFixedLength; internal readonly int[] sizeIndexArray; private readonly int* sizeIndex; private readonly int numBins; - internal readonly CheckEmptyWorker checkEmptyWorker; + internal readonly CheckEmptyWorker checkEmptyWorker; /// public override string ToString() - => $"isFixedLen {IsFixedLength}, numBins {numBins}, searchNextBin {numberOfBinsToSearch}, checkEmptyWorker: {checkEmptyWorker}"; + => $"numBins {numBins}, searchNextBin {numberOfBinsToSearch}, checkEmptyWorker: {checkEmptyWorker}"; - internal FreeRecordPool(TsavoriteKV store, RevivificationSettings settings, int fixedRecordLength) + internal FreeRecordPool(TsavoriteKV store, RevivificationSettings settings) { this.store = store; - IsFixedLength = fixedRecordLength > 0; checkEmptyWorker = new(this); - if (IsFixedLength) - { - numBins = 1; - bins = [new FreeRecordBin(ref settings.FreeRecordBins[0], fixedRecordLength, isFixedLength: true)]; - return; - } - // First create the "size index": a cache-aligned vector of int bin sizes. This way searching for the bin - // for a record size will stay in a single cache line (unless there are more than 16 bins). + // for a recordPtr size will stay in a single cache line (unless there are more than 16 bins). var sizeIndexCount = RoundUp(settings.FreeRecordBins.Length * sizeof(int), Constants.kCacheLineBytes) / sizeof(int); // Overallocate the GCHandle by one cache line so we have room to offset the returned pointer to make it cache-aligned. - sizeIndexArray = GC.AllocateArray(sizeIndexCount + Constants.kCacheLineBytes / sizeof(int), pinned: true); - long p = (long)Unsafe.AsPointer(ref sizeIndexArray[0]); + sizeIndexArray = GC.AllocateArray(sizeIndexCount + (Constants.kCacheLineBytes / sizeof(int)), pinned: true); + var p = (long)Unsafe.AsPointer(ref sizeIndexArray[0]); // Force the pointer to align to cache boundary. - long p2 = RoundUp(p, Constants.kCacheLineBytes); + var p2 = RoundUp(p, Constants.kCacheLineBytes); sizeIndex = (int*)p2; - // Create the bins. - List binList = new(); - int prevBinRecordSize = RevivificationBin.MinRecordSize - 8; // The minimum record size increment is 8, so the first bin will set this to MinRecordSize or more - for (var ii = 0; ii < settings.FreeRecordBins.Length; ++ii) + // Create the bins. The minimum recordPtr size increment is Constants.kRecordAlignment, so set prevBinRecordSize to + // MinRecordSize - Constants.kRecordAlignment and the first bin will set its minimum recordsize to MinRecordSize or more. + List binList = []; + var prevBinRecordSize = RevivificationBin.MinRecordSize - Constants.kRecordAlignment; + for (var ii = 0; ii < settings.FreeRecordBins.Length; ii++) { if (prevBinRecordSize >= settings.FreeRecordBins[ii].RecordSize) continue; - FreeRecordBin bin = new(ref settings.FreeRecordBins[ii], prevBinRecordSize, isFixedLength: false); + FreeRecordBin bin = new(ref settings.FreeRecordBins[ii], prevBinRecordSize); sizeIndex[binList.Count] = bin.maxRecordSize; binList.Add(bin); prevBinRecordSize = bin.maxRecordSize; @@ -503,10 +501,8 @@ internal FreeRecordPool(TsavoriteKV s [MethodImpl(MethodImplOptions.AggressiveInlining)] internal bool GetBinIndex(int size, out int binIndex) { - Debug.Assert(!IsFixedLength, "Should only search bins if !IsFixedLength"); - // Sequential search in the sizeIndex for the requested size. - for (var ii = 0; ii < numBins; ++ii) + for (var ii = 0; ii < numBins; ii++) { if (sizeIndex[ii] >= size) { @@ -519,22 +515,22 @@ internal bool GetBinIndex(int size, out int binIndex) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryAdd(long logicalAddress, int size, ref RevivificationStats revivStats) + private bool TryAddToBin(long logicalAddress, ref LogRecord logRecord, ref RevivificationStats revivStats) { var minAddress = store.GetMinRevivifiableAddress(); - int binIndex = 0; - if (logicalAddress < minAddress || (!IsFixedLength && !GetBinIndex(size, out binIndex))) + var recordSize = logRecord.AllocatedSize; + if (logicalAddress < minAddress || (!GetBinIndex(recordSize, out var binIndex))) return false; - if (!bins[binIndex].TryAdd(logicalAddress, size, store, minAddress, ref revivStats)) + if (!bins[binIndex].TryAdd(logicalAddress, recordSize, store, minAddress, ref revivStats)) return false; - // We've added a record, so now start the worker thread that periodically checks to see if Take() has emptied the bins. + // We've added a recordPtr, so now start the worker thread that periodically checks to see if Take() has emptied the bins. checkEmptyWorker.Start(); return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryAdd(long logicalAddress, long physicalAddress, int allocatedSize, ref RevivificationStats revivStats) + public bool TryAdd(long logicalAddress, ref LogRecord logRecord, ref RevivificationStats revivStats) { var minAddress = store.GetMinRevivifiableAddress(); if (logicalAddress < minAddress) @@ -542,37 +538,24 @@ public bool TryAdd(long logicalAddress, long physicalAddress, int allocatedSize, ++revivStats.failedAdds; return false; } - var recordInfo = store.hlog.GetInfo(physicalAddress); - recordInfo.TrySeal(invalidate: true); - store.SetFreeRecordSize(physicalAddress, ref recordInfo, allocatedSize); - bool result = TryAdd(logicalAddress, allocatedSize, ref revivStats); - - if (result) - ++revivStats.successfulAdds; - else - ++revivStats.failedAdds; - return result; + _ = logRecord.InfoRef.TrySeal(invalidate: true); + return TryAddToBin(logicalAddress, ref logRecord, ref revivStats); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTake(int recordSize, long minAddress, out long address, ref RevivificationStats revivStats) + public bool TryTake(int actualInlineRecordSize, long minAddress, out long address, ref RevivificationStats revivStats) { address = 0; - bool result = false; - if (IsFixedLength) - result = bins[0].TryTake(recordSize, minAddress, store, out address, ref revivStats); - else if (GetBinIndex(recordSize, out int index)) + var result = false; + + var allocatedInlineRecordSize = RoundUp(actualInlineRecordSize, Constants.kRecordAlignment); + if (GetBinIndex(allocatedInlineRecordSize, out var index)) { // Try to Take from the initial bin and if unsuccessful, try the next-highest bin if requested. - result = bins[index].TryTake(recordSize, minAddress, store, oversize: sizeIndex[index] > RevivificationBin.MaxInlineRecordSize, out address, ref revivStats); - for (int ii = 0; !result && ii < numberOfBinsToSearch && index < numBins - 1; ++ii) - result = bins[++index].TryTake(recordSize, minAddress, store, oversize: sizeIndex[index] > RevivificationBin.MaxInlineRecordSize, out address, ref revivStats); + result = bins[index].TryTake(allocatedInlineRecordSize, minAddress, store, oversize: sizeIndex[index] > RevivificationBin.MaxInlineRecordSize, out address, ref revivStats); + for (var ii = 0; !result && ii < numberOfBinsToSearch && index < numBins - 1; ii++) + result = bins[++index].TryTake(allocatedInlineRecordSize, minAddress, store, oversize: sizeIndex[index] > RevivificationBin.MaxInlineRecordSize, out address, ref revivStats); } - - if (result) - ++revivStats.successfulTakes; - else - ++revivStats.failedTakes; return result; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RecordLengths.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RecordLengths.cs deleted file mode 100644 index a77405cc234..00000000000 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RecordLengths.cs +++ /dev/null @@ -1,241 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace Tsavorite.core -{ - using static Utility; - - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal long GetMinRevivifiableAddress() - => RevivificationManager.GetMinRevivifiableAddress(hlogBase.GetTailAddress(), hlogBase.ReadOnlyAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetValueOffset(long physicalAddress, ref TValue recordValue) => (int)((long)Unsafe.AsPointer(ref recordValue) - physicalAddress); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe int* GetExtraValueLengthPointer(ref TValue recordValue, int usedValueLength) - { - Debug.Assert(RoundUp(usedValueLength, sizeof(int)) == usedValueLength, "GetLiveFullValueLengthPointer: usedValueLength should have int-aligned length"); - return (int*)((long)Unsafe.AsPointer(ref recordValue) + usedValueLength); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal unsafe void SetExtraValueLength(ref TValue recordValue, ref RecordInfo recordInfo, int usedValueLength, int fullValueLength) - { - if (RevivificationManager.IsFixedLength) - recordInfo.ClearHasFiller(); - else - SetVarLenExtraValueLength(ref recordValue, ref recordInfo, usedValueLength, fullValueLength); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static unsafe void SetVarLenExtraValueLength(ref TValue recordValue, ref RecordInfo recordInfo, int usedValueLength, int fullValueLength) - { - usedValueLength = RoundUp(usedValueLength, sizeof(int)); - Debug.Assert(fullValueLength >= usedValueLength, $"SetFullValueLength: usedValueLength {usedValueLength} cannot be > fullValueLength {fullValueLength}"); - int extraValueLength = fullValueLength - usedValueLength; - if (extraValueLength >= sizeof(int)) - { - var extraValueLengthPtr = GetExtraValueLengthPointer(ref recordValue, usedValueLength); - Debug.Assert(*extraValueLengthPtr == 0 || *extraValueLengthPtr == extraValueLength, "existing ExtraValueLength should be 0 or the same value"); - - // We always store the "extra" as the difference between the aligned usedValueLength and the fullValueLength. - // However, the UpdateInfo structures use the unaligned usedValueLength; aligned usedValueLength is not visible to the user. - *extraValueLengthPtr = extraValueLength; - recordInfo.SetHasFiller(); - return; - } - recordInfo.ClearHasFiller(); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal (int usedValueLength, int fullValueLength, int fullRecordLength) GetRecordLengths(long physicalAddress, ref TValue recordValue, ref RecordInfo recordInfo) - { - // FixedLen may be GenericAllocator which does not point physicalAddress to the actual record location, so calculate fullRecordLength via GetAverageRecordSize(). - if (RevivificationManager.IsFixedLength) - return (RevivificationManager.FixedValueLength, RevivificationManager.FixedValueLength, hlog.GetAverageRecordSize()); - - int usedValueLength, fullValueLength, allocatedSize, valueOffset = GetValueOffset(physicalAddress, ref recordValue); - if (recordInfo.HasFiller) - { - usedValueLength = hlog.GetValueLength(ref recordValue); - var alignedUsedValueLength = RoundUp(usedValueLength, sizeof(int)); - fullValueLength = alignedUsedValueLength + *GetExtraValueLengthPointer(ref recordValue, alignedUsedValueLength); - Debug.Assert(fullValueLength >= usedValueLength, $"GetLengthsFromFiller: fullValueLength {fullValueLength} should be >= usedValueLength {usedValueLength}"); - allocatedSize = valueOffset + fullValueLength; - } - else - { - // Live VarLen record with no stored sizes; we always have a Key and Value (even if defaults). Return the full record length (including recordInfo and Key). - (int actualSize, allocatedSize) = hlog.GetRecordSize(physicalAddress); - usedValueLength = actualSize - valueOffset; - fullValueLength = allocatedSize - valueOffset; - } - - Debug.Assert(usedValueLength >= 0, $"GetLiveRecordLengths: usedValueLength {usedValueLength}"); - Debug.Assert(fullValueLength >= 0, $"GetLiveRecordLengths: fullValueLength {fullValueLength}"); - Debug.Assert(allocatedSize >= 0, $"GetLiveRecordLengths: fullRecordLength {allocatedSize}"); - return (usedValueLength, fullValueLength, allocatedSize); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private (int usedValueLength, int fullValueLength) GetNewValueLengths(int actualSize, int allocatedSize, long newPhysicalAddress, ref TValue recordValue) - { - // Called after a new record is allocated - if (RevivificationManager.IsFixedLength) - return (RevivificationManager.FixedValueLength, RevivificationManager.FixedValueLength); - - int valueOffset = GetValueOffset(newPhysicalAddress, ref recordValue); - int usedValueLength = actualSize - valueOffset; - int fullValueLength = allocatedSize - valueOffset; - Debug.Assert(usedValueLength >= 0, $"GetNewValueLengths: usedValueLength {usedValueLength}"); - Debug.Assert(fullValueLength >= 0, $"GetNewValueLengths: fullValueLength {fullValueLength}"); - Debug.Assert(fullValueLength >= RoundUp(usedValueLength, sizeof(int)), $"GetNewValueLengths: usedValueLength {usedValueLength} cannot be > fullValueLength {fullValueLength}"); - - return (usedValueLength, fullValueLength); - } - - // A "free record" is one on the FreeList. - #region FreeRecords - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetFreeRecordSize(long physicalAddress, ref RecordInfo recordInfo, int allocatedSize) - { - // Skip the valuelength calls if we are not VarLen. - if (RevivificationManager.IsFixedLength) - { - recordInfo.ClearHasFiller(); - return; - } - - // Store the full value length. Defer clearing the Key until the record is revivified (it may never be). - ref TValue recordValue = ref hlog.GetValue(physicalAddress); - int usedValueLength = hlog.GetValueLength(ref recordValue); - int fullValueLength = allocatedSize - GetValueOffset(physicalAddress, ref recordValue); - SetVarLenExtraValueLength(ref recordValue, ref recordInfo, usedValueLength, fullValueLength); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal int GetFreeRecordSize(long physicalAddress, ref RecordInfo recordInfo) - => RevivificationManager.IsFixedLength - ? hlog.GetAverageRecordSize() - : GetRecordLengths(physicalAddress, ref hlog.GetValue(physicalAddress), ref recordInfo).fullRecordLength; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void ClearExtraValueSpace(ref RecordInfo recordInfo, ref TValue recordValue, int usedValueLength, int fullValueLength) - { - // SpanByte's implementation of GetAndInitializeValue does not clear the space after usedValueLength. This may be - // considerably less than the previous value length, so we clear it here before DisposeForRevivification. This space - // includes the extra value length if Filler is set, so we must clear the space before clearing the Filler bit so - // log-scan traversal does not see nonzero values past Value (it's fine if we see the Filler and extra length is 0). - int extraValueLength = fullValueLength - usedValueLength; // do not round up usedValueLength; we must clear all extra bytes - if (extraValueLength > 0) - { - // Even though this says "SpanByte" it is just a utility function to zero space; no actual SpanByte instance is assumed - SpanByte.Clear((byte*)Unsafe.AsPointer(ref recordValue) + usedValueLength, extraValueLength); - } - recordInfo.ClearHasFiller(); - } - - // Do not try to inline this; it causes TryAllocateRecord to bloat and slow - bool TryTakeFreeRecord(TSessionFunctionsWrapper sessionFunctions, int requiredSize, ref int allocatedSize, int newKeySize, long minRevivAddress, - out long logicalAddress, out long physicalAddress) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - { - // Caller checks for UseFreeRecordPool - if (RevivificationManager.TryTake(allocatedSize, minRevivAddress, out logicalAddress, ref sessionFunctions.Ctx.RevivificationStats)) - { - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - ref RecordInfo recordInfo = ref hlog.GetInfo(physicalAddress); - Debug.Assert(recordInfo.IsSealed, "TryTakeFreeRecord: recordInfo should still have the revivification Seal"); - - // If IsFixedLengthReviv, the allocatedSize will be unchanged - if (!RevivificationManager.IsFixedLength) - { - var (usedValueLength, fullValueLength, fullRecordLength) = GetRecordLengths(physicalAddress, ref hlog.GetValue(physicalAddress), ref recordInfo); - - // ClearExtraValueSpace has already been called (at freelist-add time) to zero the end of the value space between used and full value lengths and clear the Filler. - // Now we use the newKeySize to find out how much space is actually required. - var valueOffset = fullRecordLength - fullValueLength; - var requiredValueLength = requiredSize - valueOffset; - var minValueLength = requiredValueLength < usedValueLength ? requiredValueLength : usedValueLength; - ref var recordValue = ref hlog.GetValue(physicalAddress); - Debug.Assert(valueOffset == (long)Unsafe.AsPointer(ref recordValue) - physicalAddress); - - // Clear any no-longer-needed space, then call DisposeForRevivification again with newKeySize so SpanByte can be efficient about zeroinit. - ClearExtraValueSpace(ref recordInfo, ref recordValue, minValueLength, fullValueLength); - storeFunctions.DisposeRecord(ref hlog.GetKey(physicalAddress), ref recordValue, DisposeReason.RevivificationFreeList, newKeySize); - - Debug.Assert(fullRecordLength >= allocatedSize, $"TryTakeFreeRecord: fullRecordLength {fullRecordLength} should be >= allocatedSize {allocatedSize}"); - allocatedSize = fullRecordLength; - } - - // Preserve the Sealed bit due to checkpoint/recovery; see RecordInfo.WriteInfo. - return true; - } - - // No free record available. - logicalAddress = physicalAddress = default; - return false; - } - - #endregion FreeRecords - - // TombstonedRecords are in the tag chain with the tombstone bit set (they are not in the freelist). They preserve the key (they mark that key as deleted, - // which is important if there is a subsequent record for that key), and store the full Value length after the used value data (if there is room). - #region TombstonedRecords - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SetTombstoneAndExtraValueLength(ref TValue recordValue, ref RecordInfo recordInfo, int usedValueLength, int fullValueLength) - { - recordInfo.SetTombstone(); - if (RevivificationManager.IsFixedLength) - { - recordInfo.ClearHasFiller(); - return; - } - - Debug.Assert(usedValueLength == hlog.GetValueLength(ref recordValue)); - SetVarLenExtraValueLength(ref recordValue, ref recordInfo, usedValueLength, fullValueLength); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal (bool ok, int usedValueLength) TryReinitializeTombstonedValue(TSessionFunctionsWrapper sessionFunctions, - ref RecordInfo srcRecordInfo, ref TKey key, ref TValue recordValue, int requiredSize, (int usedValueLength, int fullValueLength, int allocatedSize) recordLengths, long physicalAddress) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper - { - if (RevivificationManager.IsFixedLength || recordLengths.allocatedSize < requiredSize) - return (false, recordLengths.usedValueLength); - - // Zero the end of the value space between required and full value lengths and clear the Filler. - var valueOffset = recordLengths.allocatedSize - recordLengths.fullValueLength; - var requiredValueLength = requiredSize - valueOffset; - var minValueLength = requiredValueLength < recordLengths.usedValueLength ? requiredValueLength : recordLengths.usedValueLength; - - // clears out the minimum space possible. So let's say we are shrinking our usage from 8 bytes to 3 bytes. This will clear only the bytes 4-8. - // if we are expanding this will not clear anything. - ClearExtraValueSpace(ref srcRecordInfo, ref recordValue, minValueLength, recordLengths.fullValueLength); - - srcRecordInfo.ClearTombstone(); - - // for SpanByte, this will set the new length (payload + metadata). - hlog.GetAndInitializeValue(physicalAddress, physicalAddress + requiredSize); - // since the above sets the Length, we can use the below to get TotalSize which represents the UsedLength of a record - var newUsedValueLength = hlog.GetValueLength(ref recordValue); - - // potentially sets filler, if the used value length is going to be under the full length by more than 4 bytes. - SetExtraValueLength(ref recordValue, ref srcRecordInfo, usedValueLength: newUsedValueLength, recordLengths.fullValueLength); - - return (true, newUsedValueLength); - } - - #endregion TombstonedRecords - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationManager.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationManager.cs index bf4b3eb7749..80244f9592a 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationManager.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationManager.cs @@ -6,22 +6,19 @@ namespace Tsavorite.core { - internal struct RevivificationManager - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal struct RevivificationManager + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - internal FreeRecordPool FreeRecordPool; - internal readonly bool UseFreeRecordPool => FreeRecordPool is not null; + internal FreeRecordPool freeRecordPool; + internal readonly bool UseFreeRecordPool => freeRecordPool is not null; internal RevivificationStats stats = new(); - internal bool IsEnabled => revivSuspendCount == 0; - internal static int FixedValueLength => Unsafe.SizeOf(); + internal readonly bool IsEnabled => revivSuspendCount == 0; internal bool restoreDeletedRecordsIfBinIsFull; internal bool useFreeRecordPoolForCTT; - internal readonly bool IsFixedLength { get; } - internal double revivifiableFraction; internal int revivSuspendCount = -1; @@ -30,47 +27,42 @@ internal struct RevivificationManager public void ResumeRevivification() => Interlocked.Increment(ref revivSuspendCount); - public RevivificationManager(TsavoriteKV store, bool isFixedLen, RevivificationSettings revivSettings, LogSettings logSettings) + public RevivificationManager(TsavoriteKV store, RevivificationSettings revivSettings, LogSettings logSettings) { - IsFixedLength = isFixedLen; - revivifiableFraction = revivSettings is null || revivSettings.RevivifiableFraction == RevivificationSettings.DefaultRevivifiableFraction - ? logSettings.MutableFraction - : revivSettings.RevivifiableFraction; - if (revivSettings is null) + { + revivifiableFraction = RevivificationSettings.DefaultRevivifiableFraction; return; + } + revivifiableFraction = revivSettings.RevivifiableFraction; - revivSettings.Verify(IsFixedLength, logSettings.MutableFraction); + revivSettings.Verify(logSettings.MutableFraction); if (!revivSettings.EnableRevivification) return; revivSuspendCount = 0; if (revivSettings.FreeRecordBins?.Length > 0) { - FreeRecordPool = new FreeRecordPool(store, revivSettings, IsFixedLength ? store.hlog.GetAverageRecordSize() : -1); + freeRecordPool = new FreeRecordPool(store, revivSettings); restoreDeletedRecordsIfBinIsFull = revivSettings.RestoreDeletedRecordsIfBinIsFull; useFreeRecordPoolForCTT = revivSettings.UseFreeRecordPoolForCopyToTail; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal long GetMinRevivifiableAddress(long tailAddress, long readOnlyAddress) + internal readonly long GetMinRevivifiableAddress(long tailAddress, long readOnlyAddress) => tailAddress - (long)((tailAddress - readOnlyAddress) * revivifiableFraction); // Method redirectors [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryAdd(long logicalAddress, int size, ref RevivificationStats revivStats) - => UseFreeRecordPool && FreeRecordPool.TryAdd(logicalAddress, size, ref revivStats); + public readonly bool TryAdd(long logicalAddress, ref LogRecord logRecord, ref RevivificationStats revivStats) + => UseFreeRecordPool && freeRecordPool.TryAdd(logicalAddress, ref logRecord, ref revivStats); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryAdd(long logicalAddress, long physicalAddress, int allocatedSize, ref RevivificationStats revivStats) - => UseFreeRecordPool && FreeRecordPool.TryAdd(logicalAddress, physicalAddress, allocatedSize, ref revivStats); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryTake(int recordSize, long minAddress, out long address, ref RevivificationStats revivStats) + public readonly bool TryTake(int actualInlineRecordSize, long minAddress, out long address, ref RevivificationStats revivStats) { if (UseFreeRecordPool) - return FreeRecordPool.TryTake(recordSize, minAddress, out address, ref revivStats); + return freeRecordPool.TryTake(actualInlineRecordSize, minAddress, out address, ref revivStats); address = 0; return false; } @@ -78,9 +70,7 @@ public bool TryTake(int recordSize, long minAddress, out long address, ref Reviv public void Dispose() { if (UseFreeRecordPool) - { - FreeRecordPool.Dispose(); - } + freeRecordPool.Dispose(); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationSettings.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationSettings.cs index 632036026b1..75cb541d7a5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/Revivification/RevivificationSettings.cs @@ -19,12 +19,12 @@ public class RevivificationSettings /// /// Indicates whether deleted record space should be reused. /// - ///
  • If this is true, then tombstoned records in the hashtable chain are revivified if possible, and a FreeList is maintained if + /// If this is true, then tombstoned records in the hashtable chain are revivified if possible, and a FreeList is maintained if /// is non-null and non-empty. - ///
    • - ///
  • If this is false, then tombstoned records in the hashtable chain will not be revivified, and no FreeList is used (regardless + /// + /// If this is false, then tombstoned records in the hashtable chain will not be revivified, and no FreeList is used (regardless /// of the setting of ). - ///
    • + /// ///     ///     public bool EnableRevivification = true; @@ -40,11 +40,6 @@ public class RevivificationSettings /// /// Bin definitions for the free list (in addition to any in the hash chains). These must be ordered by . /// - /// - /// If the Key and Value are both fixed-length datatypes (either blittable or object), this must contain a single bin whose - /// is ignored. Otherwise, one or both of the Key and Value are variable-length, - /// and this usually contains multiple bins. - /// public RevivificationBin[] FreeRecordBins; /// @@ -73,21 +68,6 @@ public class RevivificationSettings /// public static PowerOf2BinsRevivificationSettings PowerOf2Bins { get; } = new(); - /// - /// Default bin for fixed-length. - /// - public static RevivificationSettings DefaultFixedLength { get; } = new() - { - FreeRecordBins = - [ - new RevivificationBin() - { - RecordSize = RevivificationBin.MaxRecordSize, - BestFitScanLimit = RevivificationBin.UseFirstFit - } - ] - }; - /// /// Enable only in-tag-chain revivification; do not use FreeList /// @@ -98,12 +78,10 @@ public class RevivificationSettings ///     public static RevivificationSettings None { get; } = new() { EnableRevivification = false }; - internal void Verify(bool isFixedRecordLength, double mutableFraction) + internal void Verify(double mutableFraction) { if (!EnableRevivification || FreeRecordBins?.Length == 0) return; - if (isFixedRecordLength && FreeRecordBins?.Length > 1) - throw new TsavoriteException($"Only 1 bin may be specified with fixed-length datatypes (blittable or object)"); if (RevivifiableFraction != DefaultRevivifiableFraction) { if (RevivifiableFraction <= 0) @@ -114,7 +92,7 @@ internal void Verify(bool isFixedRecordLength, double mutableFraction) if (FreeRecordBins is not null) { foreach (var bin in FreeRecordBins) - bin.Verify(isFixedRecordLength); + bin.Verify(); } } @@ -188,21 +166,17 @@ public struct RevivificationBin public const int DefaultRecordsPerBin = 256; /// - /// The maximum size of records in this partition. This should be partitioned for your app. Ignored if this is the single bin - /// for fixed-length records. + /// The maximum size of records in this partition. This should be partitioned for your app. /// public int RecordSize; /// /// The number of records for each partition. This count will be adjusted upward so the partition is cache-line aligned. /// - /// - /// The first record is not available; its space is used to store the circular buffer read and write pointers - /// public int NumberOfRecords = DefaultRecordsPerBin; /// - /// The maximum number of entries to scan for best fit after finding first fit. Ignored for fixed-length datatypes. + /// The maximum number of entries to scan for best fit after finding first fit. /// public int BestFitScanLimit = UseFirstFit; @@ -213,10 +187,8 @@ public RevivificationBin() { } - internal void Verify(bool isFixedLength) + internal void Verify() { - if (!isFixedLength && (RecordSize < MinRecordSize || RecordSize > MaxRecordSize)) - throw new TsavoriteException($"Invalid RecordSize {RecordSize}; must be >= {MinRecordSize} and <= {MaxRecordSize}"); if (NumberOfRecords < MinRecordsPerBin) throw new TsavoriteException($"Invalid NumberOfRecords {NumberOfRecords}; must be > {MinRecordsPerBin}"); if (BestFitScanLimit < 0) diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/SplitIndex.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/SplitIndex.cs index d8b6986fbf0..fe056d470f8 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/SplitIndex.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/SplitIndex.cs @@ -2,13 +2,16 @@ // Licensed under the MIT license. using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Threading; namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + using static LogAddress; + + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal void SplitAllBuckets() { @@ -30,6 +33,7 @@ internal void SplitAllBuckets() overflowBucketsAllocatorResize = null; } + [MethodImpl(MethodImplOptions.NoInlining)] internal void SplitBuckets(long hash) { long masked_bucket_index = hash & state[1 - resizeInfo.version].size_mask; @@ -114,27 +118,25 @@ private void SplitChunk( HashBucketEntry entry = default; do { - for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) + for (int index = 0; index < Constants.kOverflowBucketIndex; index++) { entry.word = *(((long*)src_start) + index); if (Constants.kInvalidEntry == entry.word) - { continue; - } - var logicalAddress = entry.Address; - long physicalAddress = 0; - - if (entry.ReadCache && entry.AbsoluteAddress >= readCacheBase.HeadAddress) - physicalAddress = readcache.GetPhysicalAddress(entry.AbsoluteAddress); - else if (logicalAddress >= hlogBase.HeadAddress) - physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + LogRecord logRecord = default; + if (entry.IsReadCache) + { + if (entry.Address >= readcacheBase.HeadAddress) + logRecord = readcache.CreateLogRecord(entry.Address); + } + else if (entry.Address >= hlogBase.HeadAddress) + logRecord = hlog.CreateLogRecord(entry.Address); - // It is safe to always use hlog instead of readcache for some calls such - // as GetKey and GetInfo - if (physicalAddress != 0) + if (logRecord.IsSet) { - var hash = storeFunctions.GetKeyHashCode64(ref hlog.GetKey(physicalAddress)); + var physicalAddress = logRecord.physicalAddress; + var hash = storeFunctions.GetKeyHashCode64(logRecord); if ((hash & state[resizeInfo.version].size_mask) >> (state[resizeInfo.version].size_bits - 1) == 0) { // Insert in left @@ -151,8 +153,8 @@ private void SplitChunk( left++; // Insert previous address in right - entry.Address = TraceBackForOtherChainStart(hlog.GetInfo(physicalAddress).PreviousAddress, 1); - if ((entry.Address != Constants.kInvalidAddress) && (entry.Address != Constants.kTempInvalidAddress)) + entry.Address = TraceBackForOtherChainStart(LogRecord.GetInfo(physicalAddress).PreviousAddress, 1); + if ((entry.Address != kInvalidAddress) && (entry.Address != kTempInvalidAddress)) { if (right == right_end) { @@ -183,8 +185,8 @@ private void SplitChunk( right++; // Insert previous address in left - entry.Address = TraceBackForOtherChainStart(hlog.GetInfo(physicalAddress).PreviousAddress, 0); - if ((entry.Address != Constants.kInvalidAddress) && (entry.Address != Constants.kTempInvalidAddress)) + entry.Address = TraceBackForOtherChainStart(LogRecord.GetInfo(physicalAddress).PreviousAddress, 0); + if ((entry.Address != kInvalidAddress) && (entry.Address != kTempInvalidAddress)) { if (left == left_end) { @@ -242,32 +244,24 @@ private long TraceBackForOtherChainStart(long logicalAddress, int bit) { while (true) { - HashBucketEntry entry = default; - entry.Address = logicalAddress; - if (entry.ReadCache) + LogRecord logRecord; + if (IsReadCache(logicalAddress)) { - if (logicalAddress < readCacheBase.HeadAddress) + if (logicalAddress < readcacheBase.HeadAddress) break; - var physicalAddress = readcache.GetPhysicalAddress(logicalAddress); - var hash = storeFunctions.GetKeyHashCode64(ref readcache.GetKey(physicalAddress)); - if ((hash & state[resizeInfo.version].size_mask) >> (state[resizeInfo.version].size_bits - 1) == bit) - { - return logicalAddress; - } - logicalAddress = readcache.GetInfo(physicalAddress).PreviousAddress; + logRecord = new LogRecord(readcacheBase.GetPhysicalAddress(logicalAddress)); } else { if (logicalAddress < hlogBase.HeadAddress) break; - var physicalAddress = hlog.GetPhysicalAddress(logicalAddress); - var hash = storeFunctions.GetKeyHashCode64(ref hlog.GetKey(physicalAddress)); - if ((hash & state[resizeInfo.version].size_mask) >> (state[resizeInfo.version].size_bits - 1) == bit) - { - return logicalAddress; - } - logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; + logRecord = new LogRecord(hlogBase.GetPhysicalAddress(logicalAddress)); } + + var hash = storeFunctions.GetKeyHashCode64(logRecord); + if ((hash & state[resizeInfo.version].size_mask) >> (state[resizeInfo.version].size_bits - 1) == bit) + return logicalAddress; + logicalAddress = logRecord.Info.PreviousAddress; } return logicalAddress; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToReadCache.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToReadCache.cs index a1209777217..de5d6d93aea 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToReadCache.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToReadCache.cs @@ -3,91 +3,90 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator +#pragma warning disable IDE0065 // Misplaced using directive + using static LogAddress; + + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Copy a record from the disk to the read cache. /// /// - /// - /// - /// - /// Contains the and structures for this operation, + /// Input log record that was IO'd from disk + /// Contains the and structures for this operation, /// and allows passing back the newLogicalAddress for invalidation in the case of exceptions. /// /// True if copied to readcache, else false; readcache is "best effort", and we don't fail the read process, or slow it down by retrying. /// - internal bool TryCopyToReadCache(TSessionFunctionsWrapper sessionFunctions, ref PendingContext pendingContext, - ref TKey key, ref TInput input, ref TValue recordValue, ref OperationStackContext stackCtx) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal bool TryCopyToReadCache(in TSourceLogRecord inputLogRecord, TSessionFunctionsWrapper sessionFunctions, + ref PendingContext pendingContext, ref OperationStackContext stackCtx) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - var (actualSize, allocatedSize, _) = hlog.GetRecordSize(ref key, ref recordValue); + var sizeInfo = new RecordSizeInfo() { FieldInfo = inputLogRecord.GetRecordFieldInfo() }; + hlog.PopulateRecordSizeInfo(ref sizeInfo); - if (!TryAllocateRecordReadCache(ref pendingContext, ref stackCtx, allocatedSize, out long newLogicalAddress, out long newPhysicalAddress, out _)) + if (!TryAllocateRecordReadCache(ref pendingContext, ref stackCtx, in sizeInfo, out var newLogicalAddress, out var newPhysicalAddress, out _ /*status*/)) return false; - ref var newRecordInfo = ref WriteNewRecordInfo(ref key, readCacheBase, newPhysicalAddress, inNewVersion: false, stackCtx.hei.Address); - stackCtx.SetNewRecord(newLogicalAddress | Constants.kReadCacheBitMask); - - UpsertInfo upsertInfo = new() - { - Version = sessionFunctions.Ctx.version, - SessionID = sessionFunctions.Ctx.sessionID, - Address = Constants.kInvalidAddress, // We do not expose readcache addresses - KeyHash = stackCtx.hei.hash, - }; - upsertInfo.SetRecordInfo(ref newRecordInfo); + var newLogRecord = WriteNewRecordInfo(inputLogRecord, readcacheBase, newLogicalAddress, newPhysicalAddress, in sizeInfo, inNewVersion: false, previousAddress: stackCtx.hei.Address); - // Even though readcache records are immutable, we have to initialize the lengths - ref TValue newRecordValue = ref readcache.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize); - (upsertInfo.UsedValueLength, upsertInfo.FullValueLength) = GetNewValueLengths(actualSize, allocatedSize, newPhysicalAddress, ref newRecordValue); - - TOutput output = default; - if (!sessionFunctions.SingleWriter(ref key, ref input, ref recordValue, ref readcache.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize), - ref output, ref upsertInfo, WriteReason.CopyToReadCache, ref newRecordInfo)) - { - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - return false; - } + stackCtx.SetNewRecord(newLogicalAddress | RecordInfo.kIsReadCacheBitMask); + _ = newLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); // Insert the new record by CAS'ing directly into the hash entry (readcache records are always CAS'd into the HashBucketEntry, never spliced). // It is possible that we will successfully CAS but subsequently fail due to a main log entry having been spliced in. - var success = stackCtx.hei.TryCAS(newLogicalAddress | Constants.kReadCacheBitMask); + var success = stackCtx.hei.TryCAS(newLogicalAddress | RecordInfo.kIsReadCacheBitMask); var casSuccess = success; - OperationStatus failStatus = OperationStatus.RETRY_NOW; // Default to CAS-failed status, which does not require an epoch refresh - if (success && stackCtx.recSrc.LowestReadCacheLogicalAddress != Constants.kInvalidAddress) + var failStatus = OperationStatus.RETRY_NOW; // Default to CAS-failed status, which does not require an epoch refresh + + // If lowestReadCacheLogicalAddress was previously set, see if there was a conflicting splice into the readcache/mainlog gap. + // (If lowestReadCacheLogicalAddress wasn't set, i.e. !hei.IsReadCache, then our CAS would have failed if there had been a conflicting insert at tail.) + if (success && stackCtx.recSrc.LowestReadCacheLogicalAddress != kInvalidAddress) { - // If someone added a main-log entry for this key from a CTT while we were inserting the new readcache record, then the new - // readcache record is obsolete and must be Invalidated. (If LowestReadCacheLogicalAddress == kInvalidAddress, then the CAS would have - // failed in this case.) If this was the first readcache record in the chain, then once we CAS'd it in someone could have spliced into - // it, but then that splice will call ReadCacheCheckTailAfterSplice and invalidate it if it's the same key. - // Consistency Notes: - // - This is only a concern for CTT; an update would take an XLock which means the ReadCache insert could not be done until that XLock was released. - // a. Therefore there is no "momentary inconsistency", because the value inserted at the splice would not be changed. - // b. It is not possible for another thread to update the "at tail" value to introduce inconsistency until we have released the current SLock. - // - If there are two ReadCache inserts for the same key, one will fail the CAS because it will see the other's update which changed hei.entry. - success = EnsureNoNewMainLogRecordWasSpliced(ref key, stackCtx.recSrc, pendingContext.InitialLatestLogicalAddress, ref failStatus); + // We may have forced ReadCacheEvict by the TryAllocate above; in that case, stackCtx.recSrc.LowestReadCacheLogicalAddress, or even the entire + // readcache chain, may have been evicted. Therefore, SkipReadCache here; reinitialize the recSrc to hei and the hlog (not readcache). + if (AbsoluteAddress(stackCtx.recSrc.LowestReadCacheLogicalAddress) < readcacheBase.ClosedUntilAddress) + { + stackCtx.SetRecordSourceToHashEntry(hlogBase); + SkipReadCache(ref stackCtx, out _); // No need to track didRefresh; we already know it was refreshed once. + } + + if (stackCtx.hei.IsReadCache) + { + // If someone added a main-log entry for this key from a CTT while we were inserting the new readcache record, then the new + // readcache record is obsolete and must be Invalidated. (If LowestReadCacheLogicalAddress == kInvalidAddress, then the CAS would have + // failed in this case.) If this was the first readcache record in the chain, then once we CAS'd it in someone could have spliced into + // it, but then that splice will call ReadCacheCheckTailAfterSplice and invalidate it if it's the same key. + // Consistency Notes: + // - This is only a concern for CTT; an update would take an XLock which means the ReadCache insert could not be done until that XLock was released. + // a. Therefore there is no "momentary inconsistency", because the value inserted at the splice would not be changed. + // b. It is not possible for another thread to update the "at tail" value to introduce inconsistency until we have released the current SLock. + // - If there are two ReadCache inserts for the same key, one will fail the CAS because it will see the other's update which changed hei.entry. + success = EnsureNoNewMainLogRecordWasSpliced(inputLogRecord, ref stackCtx, pendingContext.initialLatestLogicalAddress, ref failStatus); + } } if (success) { - if (success) - newRecordInfo.UnsealAndValidate(); - pendingContext.recordInfo = newRecordInfo; - pendingContext.logicalAddress = upsertInfo.Address; - sessionFunctions.PostSingleWriter(ref key, ref input, ref recordValue, ref readcache.GetValue(newPhysicalAddress), ref output, ref upsertInfo, WriteReason.CopyToReadCache, ref newRecordInfo); + // We don't call PostInitialWriter here so we must do the size tracking separately. + readcacheBase.logSizeTracker?.UpdateSize(in newLogRecord, add: true); + + newLogRecord.InfoRef.UnsealAndValidate(); + // Do not clear pendingContext.logicalAddress; we've already set it to the requested address, which is valid. We don't expose readcache + // addresses, but here we found it in the main log address space, so retain that address. stackCtx.ClearNewRecord(); return true; } // CAS failure, or another record was spliced in. - stackCtx.SetNewRecordInvalid(ref newRecordInfo); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); if (!casSuccess) { - storeFunctions.DisposeRecord(ref readcache.GetKey(newPhysicalAddress), ref readcache.GetValue(newPhysicalAddress), DisposeReason.SingleWriterCASFailed); - newRecordInfo.PreviousAddress = Constants.kTempInvalidAddress; // Necessary for ReadCacheEvict, but cannot be kInvalidAddress or we have recordInfo.IsNull + OnDispose(ref newLogRecord, DisposeReason.InitialWriterCASFailed); + newLogRecord.InfoRef.PreviousAddress = kTempInvalidAddress; // Necessary for ReadCacheEvict, but cannot be kInvalidAddress or we have recordInfo.IsNull } return false; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToTail.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToTail.cs index de38e40c1ad..9fa836e2de2 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToTail.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/TryCopyToTail.cs @@ -3,83 +3,71 @@ namespace Tsavorite.core { - public unsafe partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public unsafe partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Copy a record from the immutable region of the log, from the disk, or from ConditionalCopyToTail to the tail of the log (or splice into the log/readcache boundary). /// + /// The source record, either from readonly region of the in-memory log, or from disk + /// /// - /// - /// - /// - /// - /// Contains the and structures for this operation, + /// Contains the and structures for this operation, /// and allows passing back the newLogicalAddress for invalidation in the case of exceptions. - /// if ., the recordInfo to close, if transferring. - /// - /// The reason for this operation. /// /// /// RETRY_NOW: failed CAS, so no copy done. This routine deals entirely with new records, so will not encounter Sealed records /// SUCCESS: copy was done /// /// - internal OperationStatus TryCopyToTail(ref PendingContext pendingContext, - ref TKey key, ref TInput input, ref TValue value, ref TOutput output, ref OperationStackContext stackCtx, - ref RecordInfo srcRecordInfo, TSessionFunctionsWrapper sessionFunctions, WriteReason reason) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal OperationStatus TryCopyToTail(in TSourceLogRecord inputLogRecord, + TSessionFunctionsWrapper sessionFunctions, ref PendingContext pendingContext, + ref OperationStackContext stackCtx) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { - var (actualSize, allocatedSize, keySize) = hlog.GetRecordSize(ref key, ref value); - if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, actualSize, ref allocatedSize, keySize, new AllocateOptions() { Recycle = true }, - out long newLogicalAddress, out long newPhysicalAddress, out OperationStatus status)) - return status; - ref var newRecordInfo = ref WriteNewRecordInfo(ref key, hlogBase, newPhysicalAddress, inNewVersion: sessionFunctions.Ctx.InNewVersion, stackCtx.recSrc.LatestLogicalAddress); - stackCtx.SetNewRecord(newLogicalAddress); - - UpsertInfo upsertInfo = new() - { - Version = sessionFunctions.Ctx.version, - SessionID = sessionFunctions.Ctx.sessionID, - Address = newLogicalAddress, - KeyHash = stackCtx.hei.hash, - }; - upsertInfo.SetRecordInfo(ref newRecordInfo); + var sizeInfo = new RecordSizeInfo() { FieldInfo = inputLogRecord.GetRecordFieldInfo() }; + hlog.PopulateRecordSizeInfo(ref sizeInfo); - ref TValue newRecordValue = ref hlog.GetAndInitializeValue(newPhysicalAddress, newPhysicalAddress + actualSize); - (upsertInfo.UsedValueLength, upsertInfo.FullValueLength) = GetNewValueLengths(actualSize, allocatedSize, newPhysicalAddress, ref newRecordValue); + var allocOptions = new AllocateOptions() { recycle = true }; + if (!TryAllocateRecord(sessionFunctions, ref pendingContext, ref stackCtx, ref sizeInfo, allocOptions, out var newLogicalAddress, out var newPhysicalAddress, out var status)) + return status; + var newLogRecord = WriteNewRecordInfo(inputLogRecord, hlogBase, newLogicalAddress, newPhysicalAddress, in sizeInfo, inNewVersion: sessionFunctions.Ctx.InNewVersion, previousAddress: stackCtx.recSrc.LatestLogicalAddress); - if (!sessionFunctions.SingleWriter(ref key, ref input, ref value, ref newRecordValue, ref output, ref upsertInfo, reason, ref newRecordInfo)) - { - // Save allocation for revivification (not retry, because we won't retry here), or abandon it if that fails. - if (RevivificationManager.UseFreeRecordPool && RevivificationManager.TryAdd(newLogicalAddress, newPhysicalAddress, allocatedSize, ref sessionFunctions.Ctx.RevivificationStats)) - stackCtx.ClearNewRecord(); - else - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - return (upsertInfo.Action == UpsertAction.CancelOperation) ? OperationStatus.CANCELED : OperationStatus.SUCCESS; - } - SetExtraValueLength(ref newRecordValue, ref srcRecordInfo, upsertInfo.UsedValueLength, upsertInfo.FullValueLength); + stackCtx.SetNewRecord(newLogicalAddress); + _ = newLogRecord.TryCopyFrom(in inputLogRecord, in sizeInfo); // Insert the new record by CAS'ing either directly into the hash entry or splicing into the readcache/mainlog boundary. - bool success = CASRecordIntoChain(ref key, ref stackCtx, newLogicalAddress, ref newRecordInfo); + var success = CASRecordIntoChain(newLogicalAddress, ref newLogRecord, ref stackCtx); if (success) { - newRecordInfo.UnsealAndValidate(); - PostCopyToTail(ref key, ref stackCtx, ref srcRecordInfo, pendingContext.InitialEntryAddress); + // We don't call PostInitialWriter here so we must do the size tracking separately. + hlogBase.logSizeTracker?.UpdateSize(in newLogRecord, add: true); + + // Fire the application-level post-CAS hook BEFORE unsealing dst, so concurrent readers + // observing dst still see SkipOnScan and retry. The hook may mutate dst's value bytes + // and/or the source record (similar to RIPROMOTE PostCopyUpdater for live transfers). + if (storeFunctions.CallPostCopyToTail) + { + var srcLogicalAddress = stackCtx.recSrc.HasMainLogSrc + ? stackCtx.recSrc.LogicalAddress + : pendingContext.originalAddress; + storeFunctions.PostCopyToTail(in inputLogRecord, srcLogicalAddress, ref newLogRecord, newLogicalAddress); + } + + newLogRecord.InfoRef.UnsealAndValidate(); - pendingContext.recordInfo = newRecordInfo; - pendingContext.logicalAddress = upsertInfo.Address; - sessionFunctions.PostSingleWriter(ref key, ref input, ref value, ref hlog.GetValue(newPhysicalAddress), ref output, ref upsertInfo, reason, ref newRecordInfo); + pendingContext.logicalAddress = newLogicalAddress; stackCtx.ClearNewRecord(); return OperationStatusUtils.AdvancedOpCode(OperationStatus.SUCCESS, StatusCode.Found | StatusCode.CopiedRecord); } // CAS failed - stackCtx.SetNewRecordInvalid(ref newRecordInfo); - storeFunctions.DisposeRecord(ref hlog.GetKey(newPhysicalAddress), ref hlog.GetValue(newPhysicalAddress), DisposeReason.SingleWriterCASFailed); + stackCtx.SetNewRecordInvalid(ref newLogRecord.InfoRef); + OnDispose(ref newLogRecord, DisposeReason.InitialWriterCASFailed); - SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress, allocatedSize); + SaveAllocationForRetry(ref pendingContext, newLogicalAddress, newPhysicalAddress); return OperationStatus.RETRY_NOW; // CAS failure does not require epoch refresh } } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/UpsertValueSelector.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/UpsertValueSelector.cs new file mode 100644 index 00000000000..aadf30dc3fc --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Implementation/UpsertValueSelector.cs @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Tsavorite.core +{ + public unsafe partial class TsavoriteKV where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + /// + /// Eliminates switching on type of value or another variable to determine which overloaded value-taking method to call. + /// + internal interface IUpsertValueSelector + { + static abstract RecordSizeInfo GetUpsertRecordSize(TAllocator allocator, TKey key, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput; + + static abstract bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper; + + static abstract void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper; + + static abstract bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper; + + static abstract void PostUpsertOperation(TKey key, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TEpochAccessor : IEpochAccessor; + } + + internal struct SpanUpsertValueSelector : IUpsertValueSelector + { + public static RecordSizeInfo GetUpsertRecordSize(TAllocator allocator, TKey key, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => allocator.GetUpsertRecordSize(key, valueSpan, ref input, varlenInput); + + public static bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InitialWriter(ref logRecord, in sizeInfo, ref input, valueSpan, ref output, ref upsertInfo); + + public static void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.PostInitialWriter(ref logRecord, in sizeInfo, ref input, valueSpan, ref output, ref upsertInfo); + + public static bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InPlaceWriter(ref logRecord, ref input, valueSpan, ref output, ref upsertInfo); + + public static void PostUpsertOperation(TKey key, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TEpochAccessor : IEpochAccessor + => sessionFunctions.PostUpsertOperation(key, ref input, valueSpan, ref upsertInfo, epochAccessor); + } + + internal struct ObjectUpsertValueSelector : IUpsertValueSelector + { + public static RecordSizeInfo GetUpsertRecordSize(TAllocator allocator, TKey key, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => allocator.GetUpsertRecordSize(key, valueObject, ref input, varlenInput); + + public static bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InitialWriter(ref logRecord, in sizeInfo, ref input, valueObject, ref output, ref upsertInfo); + + public static void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.PostInitialWriter(ref logRecord, in sizeInfo, ref input, valueObject, ref output, ref upsertInfo); + + public static bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InPlaceWriter(ref logRecord, ref input, valueObject, ref output, ref upsertInfo); + + public static void PostUpsertOperation(TKey key, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TEpochAccessor : IEpochAccessor + => sessionFunctions.PostUpsertOperation(key, ref input, valueObject, ref upsertInfo, epochAccessor); + } + + internal struct LogRecordUpsertValueSelector : IUpsertValueSelector + { + public static RecordSizeInfo GetUpsertRecordSize(TAllocator allocator, TKey key, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TInput input, TVariableLengthInput varlenInput) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TVariableLengthInput : IVariableLengthInput + => allocator.GetUpsertRecordSize(key, in inputLogRecord, ref input, varlenInput); + + public static bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InitialWriter(ref logRecord, in sizeInfo, ref input, in inputLogRecord, ref output, ref upsertInfo); + + public static void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.PostInitialWriter(ref logRecord, in sizeInfo, ref input, in inputLogRecord, ref output, ref upsertInfo); + + public static bool InPlaceWriter(ref LogRecord logRecord, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref TOutput output, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions) + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + => sessionFunctions.InPlaceWriter(ref logRecord, ref input, in inputLogRecord, ref output, ref upsertInfo); + + public static void PostUpsertOperation(TKey key, ref TInput input, + ReadOnlySpan valueSpan, IHeapObject valueObject, in TSourceLogRecord inputLogRecord, ref UpsertInfo upsertInfo, TSessionFunctionsWrapper sessionFunctions, TEpochAccessor epochAccessor) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TEpochAccessor : IEpochAccessor + { + if (!inputLogRecord.Info.ValueIsObject) + sessionFunctions.PostUpsertOperation(key, ref input, inputLogRecord.ValueSpan, ref upsertInfo, epochAccessor); + else + sessionFunctions.PostUpsertOperation(key, ref input, inputLogRecord.ValueObject, ref upsertInfo, epochAccessor); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs index a100a3529ce..f128d44d94b 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs @@ -3,27 +3,26 @@ using System; using System.Runtime.CompilerServices; -using System.Threading; namespace Tsavorite.core { /// /// Wrapper to process log-related commands /// - public sealed class LogAccessor : IObservable> - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public sealed class LogAccessor : IObservable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - private readonly TsavoriteKV store; + private readonly TsavoriteKV store; private readonly TAllocator allocator; - private readonly AllocatorBase allocatorBase; + internal readonly AllocatorBase allocatorBase; /// /// Constructor /// /// /// - internal LogAccessor(TsavoriteKV store, TAllocator allocator) + internal LogAccessor(TsavoriteKV store, TAllocator allocator) { this.store = store; this.allocator = allocator; @@ -56,60 +55,34 @@ internal LogAccessor(TsavoriteKV stor public long BeginAddress => allocatorBase.BeginAddress; /// - /// Get the bytes used on the primary log by every record. Does not include - /// the size of variable-length inline data. Note that class objects occupy - /// 8 bytes (reference) on the main log (i.e., the heap space occupied by - /// class objects is not included in the result of this call). + /// for serializing/deserializing . /// - public int FixedRecordSize => allocator.GetFixedRecordSize(); + public ObjectIdMap TransientObjectIdMap => allocatorBase.transientObjectIdMap; /// - /// Number of pages left empty or unallocated in the in-memory buffer (between 0 and BufferSize-1) + /// Total in-memory circular buffer capacity (in number of pages) /// - public int EmptyPageCount - { - get => allocatorBase.EmptyPageCount; - set { allocatorBase.EmptyPageCount = value; } - } + public int BufferSize => allocatorBase.BufferSize; /// - /// Maximum possible number of empty pages in Allocator + /// The log size tracker (currently used only by test) /// - public int MaxEmptyPageCount => allocatorBase.MaxEmptyPageCount; + public LogSizeTracker LogSizeTracker => allocatorBase.logSizeTracker; /// - /// Minimum possible number of empty pages in Allocator + /// Actual memory used by log (not including heap objects) /// - public int MinEmptyPageCount - { - get => allocatorBase.MinEmptyPageCount; - set { allocatorBase.MinEmptyPageCount = value; } - } + public long MemorySizeBytes => allocatorBase.AllocatedPageCount << allocatorBase.LogPageSizeBits; /// - /// Set empty page count in allocator + /// Actual memory used by log (not including heap objects), including overflow pages /// - /// New empty page count - /// Whether to wait for shift addresses to complete - public void SetEmptyPageCount(int pageCount, bool wait = false) - { - allocatorBase.EmptyPageCount = pageCount; - if (wait) - { - long newHeadAddress = (allocatorBase.GetTailAddress() & ~allocatorBase.PageSizeMask) - allocatorBase.HeadAddressLagOffset; - ShiftHeadAddress(newHeadAddress, wait); - } - } + public long MemorySizeBytesIncludingOverflowPages => (allocatorBase.AllocatedPageCount + allocator.OverflowPageCount) << allocatorBase.LogPageSizeBits; /// - /// Total in-memory circular buffer capacity (in number of pages) + /// Heap memory used /// - public int BufferSize => allocatorBase.BufferSize; - - /// - /// Actual memory used by log (not including heap objects) and overflow pages - /// - public long MemorySizeBytes => ((long)(allocatorBase.AllocatedPageCount + allocator.OverflowPageCount)) << allocatorBase.LogPageSizeBits; + public long HeapSizeBytes => allocatorBase.logSizeTracker is null ? 0 : allocatorBase.logSizeTracker.LogHeapSizeBytes; /// /// Maximum memory size in bytes @@ -121,6 +94,10 @@ public void SetEmptyPageCount(int pageCount, bool wait = false) /// public int AllocatedPageCount => allocatorBase.AllocatedPageCount; + /// Get record size required to allocate a new record. Includes allocator-specific information such as key and value overflow. + /// Requires to be populated already. + public void PopulateRecordSizeInfo(ref RecordSizeInfo sizeInfo) => allocator.PopulateRecordSizeInfo(ref sizeInfo); + /// /// Shift begin address to the provided untilAddress. Make sure address corresponds to record boundary if snapToPageStart is set to /// false. Destructive operation if truncateLog is set to true. @@ -132,18 +109,16 @@ public void SetEmptyPageCount(int pageCount, bool wait = false) public void ShiftBeginAddress(long untilAddress, bool snapToPageStart = false, bool truncateLog = false) { if (snapToPageStart) - untilAddress &= ~allocatorBase.PageSizeMask; + untilAddress = allocatorBase.GetAddressOfStartOfPageOfAddress(untilAddress); - var epochProtected = store.epoch.ThisInstanceProtected(); + var epochAcquired = store.epoch.ResumeIfNotProtected(); try { - if (!epochProtected) - store.epoch.Resume(); allocatorBase.ShiftBeginAddress(untilAddress, truncateLog); } finally { - if (!epochProtected) + if (epochAcquired) store.epoch.Suspend(); } } @@ -156,45 +131,27 @@ public void ShiftBeginAddress(long untilAddress, bool snapToPageStart = false, b public void Truncate() => ShiftBeginAddress(BeginAddress, truncateLog: true); /// - /// Shift log head address to prune memory foorprint of hybrid log + /// Shift log head address to prune memory footprint of hybrid log /// /// Address to shift head until /// Wait for operation to complete (may involve page flushing and closing) - public void ShiftHeadAddress(long newHeadAddress, bool wait) - { - // First shift read-only - // Force wait so that we do not close unflushed page - ShiftReadOnlyAddress(newHeadAddress, true); + public void ShiftHeadAddress(long newHeadAddress, bool wait) => ShiftAddresses(newHeadAddress, newHeadAddress, wait); - // Then shift head address - if (!store.epoch.ThisInstanceProtected()) - { - try - { - store.epoch.Resume(); - allocatorBase.ShiftHeadAddress(newHeadAddress); - } - finally - { - store.epoch.Suspend(); - } - - while (wait && allocatorBase.SafeHeadAddress < newHeadAddress) - _ = Thread.Yield(); - } - else - { - allocatorBase.ShiftHeadAddress(newHeadAddress); - while (wait && allocatorBase.SafeHeadAddress < newHeadAddress) - store.epoch.ProtectAndDrain(); - } - } + /// + /// Shift log read-only address, with an optional wait + /// + /// Address to shift read-only until + /// Wait to ensure shift is complete (may involve page flushing) + public void ShiftReadOnlyAddress(long newReadOnlyAddress, bool wait) => allocatorBase.ShiftReadOnlyAddressWithWait(newReadOnlyAddress, wait); - public Func IsSizeBeyondLimit - { - get => allocatorBase.IsSizeBeyondLimit; - set => allocatorBase.IsSizeBeyondLimit = value; - } + /// + /// Shift log readonly and head addresses, with an optional wait on the head address shift + /// + /// New ReadOnlyAddress + /// New HeadAddress + /// Wait for operation to complete (may involve page flushing and closing) + public void ShiftAddresses(long newReadOnlyAddress, long newHeadAddress, bool waitForEviction) + => allocatorBase.ShiftAddressesWithWait(newReadOnlyAddress, newHeadAddress, waitForEviction); /// /// Subscribe to records (in batches) as they become read-only in the log @@ -203,9 +160,9 @@ public Func IsSizeBeyondLimit /// To scan the historical part of the log, use the Scan(...) method /// /// Observer to which scan iterator is pushed - public IDisposable Subscribe(IObserver> readOnlyObserver) + public IDisposable Subscribe(IObserver readOnlyObserver) { - allocatorBase.OnReadOnlyObserver = readOnlyObserver; + allocatorBase.onReadOnlyObserver = readOnlyObserver; return new LogSubscribeDisposable(allocatorBase, isReadOnly: true); } @@ -216,71 +173,35 @@ public IDisposable Subscribe(IObserver> rea /// To scan the historical part of the log, use the Scan(...) method /// /// Observer to which scan iterator is pushed - public IDisposable SubscribeEvictions(IObserver> evictionObserver) + public IDisposable SubscribeEvictions(IObserver evictionObserver) { - allocatorBase.OnEvictionObserver = evictionObserver; + allocatorBase.onEvictionObserver = evictionObserver; return new LogSubscribeDisposable(allocatorBase, isReadOnly: false); } - public IDisposable SubscribeDeserializations(IObserver> deserializationObserver) - { - allocatorBase.OnDeserializationObserver = deserializationObserver; - return new LogSubscribeDisposable(allocatorBase, isReadOnly: false); - } + /// + /// Set the Log Size Tracker to track log size for operations that do not surface to the caller's level + /// (e.g. implementations). This includes + /// internal copies to log or readcache tail, memory trimming calculations, recovery eviction, etc. + /// + /// The tracker to record operations + public void SetLogSizeTracker(LogSizeTracker logSizeTracker) + => allocatorBase.logSizeTracker = logSizeTracker; /// /// Wrapper to help dispose the subscription /// - class LogSubscribeDisposable : IDisposable + class LogSubscribeDisposable(AllocatorBase allocator, bool isReadOnly) : IDisposable { - private readonly AllocatorBase allocator; - private readonly bool readOnly; - - public LogSubscribeDisposable(AllocatorBase allocator, bool isReadOnly) - { - this.allocator = allocator; - readOnly = isReadOnly; - } + private readonly AllocatorBase allocator = allocator; + private readonly bool readOnly = isReadOnly; public void Dispose() { if (readOnly) - allocator.OnReadOnlyObserver = null; + allocator.onReadOnlyObserver = null; else - allocator.OnEvictionObserver = null; - } - } - - /// - /// Shift log read-only address - /// - /// Address to shift read-only until - /// Wait to ensure shift is complete (may involve page flushing) - public void ShiftReadOnlyAddress(long newReadOnlyAddress, bool wait) - { - if (!store.epoch.ThisInstanceProtected()) - { - try - { - store.epoch.Resume(); - _ = allocatorBase.ShiftReadOnlyAddress(newReadOnlyAddress); - } - finally - { - store.epoch.Suspend(); - } - - // Wait for flush to complete - while (wait && allocatorBase.FlushedUntilAddress < newReadOnlyAddress) - _ = Thread.Yield(); - } - else - { - _ = allocatorBase.ShiftReadOnlyAddress(newReadOnlyAddress); - - // Wait for flush to complete - while (wait && allocatorBase.FlushedUntilAddress < newReadOnlyAddress) - store.epoch.ProtectAndDrain(); + allocator.onEvictionObserver = null; } } @@ -289,114 +210,61 @@ public void ShiftReadOnlyAddress(long newReadOnlyAddress, bool wait) ///     /// Scan iterator instance [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ITsavoriteScanIterator Scan(long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering, bool includeClosedRecords = false) + public ITsavoriteScanIterator Scan(long beginAddress, long endAddress, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool includeClosedRecords = false) => allocatorBase.Scan(store: null, beginAddress, endAddress, scanBufferingMode, includeClosedRecords); /// /// Push-scan the log given address range; returns all records with address less than endAddress /// /// True if Scan completed; false if Scan ended early due to one of the TScanIterator reader functions returning false - public bool Scan(ref TScanFunctions scanFunctions, long beginAddress, long endAddress, ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering) - where TScanFunctions : IScanIteratorFunctions + public bool Scan(ref TScanFunctions scanFunctions, long beginAddress, long endAddress, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering) + where TScanFunctions : IScanIteratorFunctions => allocatorBase.Scan(store, beginAddress, endAddress, ref scanFunctions, scanBufferingMode); /// /// Iterate versions of the specified key, starting with most recent /// /// True if Scan completed; false if Scan ended early due to one of the TScanIterator reader functions returning false - public bool IterateKeyVersions(ref TScanFunctions scanFunctions, ref TKey key) - where TScanFunctions : IScanIteratorFunctions - => allocatorBase.IterateKeyVersions(store, ref key, ref scanFunctions); + public bool IterateKeyVersions(ref TScanFunctions scanFunctions, TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TScanFunctions : IScanIteratorFunctions + => allocatorBase.IterateKeyVersions(store, key, ref scanFunctions); /// /// Flush log until current tail (records are still retained in memory) /// /// Synchronous wait for operation to complete - public void Flush(bool wait) - { - ShiftReadOnlyAddress(allocatorBase.GetTailAddress(), wait); - } + public void Flush(bool wait) => ShiftReadOnlyAddress(allocatorBase.GetTailAddress(), wait); /// /// Flush log and evict all records from memory /// /// Wait for operation to complete - public void FlushAndEvict(bool wait) - { - ShiftHeadAddress(allocatorBase.GetTailAddress(), wait); - } - - /// - /// Delete log entirely from memory. Cannot allocate on the log - /// after this point. This is a synchronous operation. - /// - public void DisposeFromMemory() - { - // Ensure we have flushed and evicted - FlushAndEvict(true); - - // Delete from memory - allocatorBase.DeleteFromMemory(); - } - - /// - /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log - /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate - /// - /// Functions used to manage key-values during compaction - /// Compact log until this address - /// Compaction type (whether we lookup records or scan log for liveness checking) - /// Address until which compaction was done - public long Compact(TFunctions functions, long untilAddress, CompactionType compactionType) - where TFunctions : ISessionFunctions - => Compact>(functions, default, untilAddress, compactionType); + public void FlushAndEvict(bool wait) => ShiftHeadAddress(allocatorBase.GetTailAddress(), wait); /// /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate /// - /// Functions used to manage key-values during compaction - /// Input for SingleWriter - /// Output from SingleWriter; it will be called all records that are moved, before Compact() returns, so the user must supply buffering or process each output completely /// Compact log until this address /// Compaction type (whether we lookup records or scan log for liveness checking) /// Address until which compaction was done - public long Compact(TFunctions functions, ref TInput input, ref TOutput output, long untilAddress, CompactionType compactionType) - where TFunctions : ISessionFunctions - => Compact>(functions, default, ref input, ref output, untilAddress, compactionType); - - /// - /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log - /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate - /// - /// Functions used to manage key-values during compaction - /// User provided compaction functions (see ) - /// Compact log until this address - /// Compaction type (whether we lookup records or scan log for liveness checking) - /// Address until which compaction was done - public long Compact(TFunctions functions, TCompactionFunctions cf, long untilAddress, CompactionType compactionType) - where TFunctions : ISessionFunctions - where TCompactionFunctions : ICompactionFunctions - { - TInput input = default; - TOutput output = default; - return Compact(functions, cf, ref input, ref output, untilAddress, compactionType); - } + public long Compact(long untilAddress, CompactionType compactionType) + => Compact(default, untilAddress, compactionType); /// /// Compact the log until specified address, moving active records to the tail of the log. BeginAddress is shifted, but the physical log /// is not deleted from disk. Caller is responsible for truncating the physical log on disk by taking a checkpoint or calling Log.Truncate /// - /// Functions used to manage key-values during compaction - /// User provided compaction functions (see ) - /// Input for SingleWriter - /// Output from SingleWriter; it will be called all records that are moved, before Compact() returns, so the user must supply buffering or process each output completely + /// User provided compaction functions (see ) /// Compact log until this address /// Compaction type (whether we lookup records or scan log for liveness checking) /// Address until which compaction was done - public long Compact(TFunctions functions, TCompactionFunctions cf, ref TInput input, ref TOutput output, long untilAddress, CompactionType compactionType) - where TFunctions : ISessionFunctions - where TCompactionFunctions : ICompactionFunctions - => store.Compact(functions, cf, ref input, ref output, untilAddress, compactionType); + public long Compact(TCompactionFunctions cf, long untilAddress, CompactionType compactionType) + where TCompactionFunctions : ICompactionFunctions + => store.Compact(cf, untilAddress, compactionType); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Tsavorite.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Tsavorite.cs index b41620afcca..924bcb87b76 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Tsavorite.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/Tsavorite.cs @@ -10,20 +10,21 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using static Tsavorite.core.LogAddress; namespace Tsavorite.core { /// /// The Tsavorite Key/Value store class /// - public partial class TsavoriteKV : TsavoriteBase, IDisposable - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase, IDisposable + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { internal readonly TAllocator hlog; - internal readonly AllocatorBase hlogBase; + internal readonly AllocatorBase hlogBase; internal readonly TAllocator readcache; - internal readonly AllocatorBase readCacheBase; + internal readonly AllocatorBase readcacheBase; internal readonly TStoreFunctions storeFunctions; @@ -32,15 +33,20 @@ public partial class TsavoriteKV : Ts internal readonly int sectorSize; internal readonly StateMachineDriver stateMachineDriver; + /// + /// ObjectIdMap to be used by operations that map it transiently, such as RENAME + /// + public ObjectIdMap TransientObjectIdMap => hlogBase.transientObjectIdMap; + /// /// Number of active entries in hash index (does not correspond to total records, due to hash collisions) /// public long EntryCount => GetEntryCount(); /// - /// Maximum number of memory pages ever allocated + /// High-water mark of the number of memory pages that were allocated in the circular buffer /// - public long MaxAllocatedPageCount => hlogBase.MaxAllocatedPageCount; + public long HighWaterAllocatedPageCount => hlogBase.HighWaterAllocatedPageCount; /// /// Size of index in #cache lines (64 bytes each) @@ -58,20 +64,25 @@ public partial class TsavoriteKV : Ts /// /// Hybrid log used by this Tsavorite instance /// - public LogAccessor Log { get; } + public LogAccessor Log { get; } + + /// + /// Readonly accessor for StoreFunctions + /// + public TStoreFunctions StoreFunctions => storeFunctions; /// /// Read cache used by this Tsavorite instance /// - public LogAccessor ReadCache { get; } + public LogAccessor ReadCache { get; } int maxSessionID; - internal readonly OverflowBucketLockTable LockTable; + internal readonly OverflowBucketLockTable LockTable; internal readonly int ThrottleCheckpointFlushDelayMs = -1; - internal RevivificationManager RevivificationManager; + internal RevivificationManager RevivificationManager; internal Func allocatorFactory; @@ -119,7 +130,7 @@ public void ResumeRevivification() /// Config settings /// Store-level user function implementations /// Func to call to create the allocator(s, if doing readcache) - public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFunctions, Func allocatorFactory) + public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFunctions, Func allocatorFactory) : base(kvSettings.Epoch, kvSettings.logger ?? kvSettings.loggerFactory?.CreateLogger("TsavoriteKV Index Overflow buckets")) { this.allocatorFactory = allocatorFactory; @@ -157,8 +168,6 @@ public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFun if (ReadCopyOptions.CopyFrom == ReadCopyFrom.Inherit) ReadCopyOptions.CopyFrom = ReadCopyFrom.Device; - bool isFixedLenReviv = hlog.IsFixedLength; - // Create the allocator var allocatorSettings = new AllocatorSettings(logSettings, epoch, kvSettings.logger ?? kvSettings.loggerFactory?.CreateLogger(typeof(TAllocator).Name)); hlog = allocatorFactory(allocatorSettings, storeFunctions); @@ -172,24 +181,25 @@ public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFun { LogDevice = new NullDevice(), ObjectLogDevice = hlog.HasObjectLog ? new NullDevice() : null, + MemorySize = logSettings.ReadCacheSettings.MemorySize, PageSizeBits = logSettings.ReadCacheSettings.PageSizeBits, - MemorySizeBits = logSettings.ReadCacheSettings.MemorySizeBits, - SegmentSizeBits = logSettings.ReadCacheSettings.MemorySizeBits, + SegmentSizeBits = logSettings.ReadCacheSettings.PageSizeBits + 1, // Not used by readcache but make sure it passes validation MutableFraction = 1 - logSettings.ReadCacheSettings.SecondChanceFraction }; allocatorSettings.logger = kvSettings.logger ?? kvSettings.loggerFactory?.CreateLogger($"{typeof(TAllocator).Name} ReadCache"); allocatorSettings.evictCallback = ReadCacheEvict; + allocatorSettings.IsReadCache = true; readcache = allocatorFactory(allocatorSettings, storeFunctions); - readCacheBase = readcache.GetBase(); - readCacheBase.Initialize(); + readcacheBase = readcache.GetBase(); + readcacheBase.Initialize(); ReadCache = new(this, readcache); } sectorSize = (int)logSettings.LogDevice.SectorSize; Initialize(kvSettings.GetIndexSizeCacheLines(), sectorSize); - LockTable = new OverflowBucketLockTable(this); - RevivificationManager = new(this, isFixedLenReviv, kvSettings.RevivificationSettings, logSettings); + LockTable = new OverflowBucketLockTable(this); + RevivificationManager = new(this, kvSettings.RevivificationSettings, logSettings); stateMachineDriver = kvSettings.StateMachineDriver ?? new(epoch, kvSettings.logger ?? kvSettings.loggerFactory?.CreateLogger($"StateMachineDriver")); @@ -204,10 +214,12 @@ public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFun } /// Get the hashcode for a key. - public long GetKeyHash(TKey key) => storeFunctions.GetKeyHashCode64(ref key); - - /// Get the hashcode for a key. - public long GetKeyHash(ref TKey key) => storeFunctions.GetKeyHashCode64(ref key); + public long GetKeyHash(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => storeFunctions.GetKeyHashCode64(key); /// /// Initiate full checkpoint @@ -221,7 +233,7 @@ public TsavoriteKV(KVSettings kvSettings, TStoreFunctions storeFun /// fail if we are already taking a checkpoint or performing some other /// operation such as growing the index). Use CompleteCheckpointAsync to wait completion. /// - public bool TryInitiateFullCheckpoint(out Guid token, CheckpointType checkpointType, IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null, CancellationToken cancellationToken = default) + public bool TryInitiateFullCheckpoint(out Guid token, CheckpointType checkpointType, IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null, CancellationToken cancellationToken = default) { IStateMachine stateMachine; @@ -254,7 +266,7 @@ public bool TryInitiateFullCheckpoint(out Guid token, CheckpointType checkpointT /// Await task to complete checkpoint, if initiated successfully /// public async ValueTask<(bool success, Guid token)> TakeFullCheckpointAsync(CheckpointType checkpointType, - CancellationToken cancellationToken = default, IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null) + CancellationToken cancellationToken = default, IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null) { var success = TryInitiateFullCheckpoint(out Guid token, checkpointType, streamingSnapshotIteratorFunctions, cancellationToken); @@ -268,6 +280,7 @@ public bool TryInitiateFullCheckpoint(out Guid token, CheckpointType checkpointT /// Initiate index-only checkpoint /// /// Checkpoint token + /// Caller's cancellation token /// Whether we could initiate the checkpoint. Use CompleteCheckpointAsync to wait completion. public bool TryInitiateIndexCheckpoint(out Guid token, CancellationToken cancellationToken = default) { @@ -302,10 +315,11 @@ public bool TryInitiateIndexCheckpoint(out Guid token, CancellationToken cancell /// /// Checkpoint token /// Checkpoint type - /// For snapshot, try to store as incremental delta over last snapshot + /// Iterator functions + /// Caller's cancellation token /// Whether we could initiate the checkpoint. Use CompleteCheckpointAsync to wait completion. - public bool TryInitiateHybridLogCheckpoint(out Guid token, CheckpointType checkpointType, bool tryIncremental = false, - IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null, CancellationToken cancellationToken = default) + public bool TryInitiateHybridLogCheckpoint(out Guid token, CheckpointType checkpointType, + IStreamingSnapshotIteratorFunctions streamingSnapshotIteratorFunctions = null, CancellationToken cancellationToken = default) { IStateMachine stateMachine; @@ -318,44 +332,15 @@ public bool TryInitiateHybridLogCheckpoint(out Guid token, CheckpointType checkp } else { - token = _lastSnapshotCheckpoint.info.guid; - var incremental = tryIncremental - && checkpointType == CheckpointType.Snapshot - && token != default - && _lastSnapshotCheckpoint.info.finalLogicalAddress > hlogBase.FlushedUntilAddress - && !hlog.HasObjectLog; - if (incremental) - { - stateMachine = Checkpoint.IncrementalHybridLogOnly(this, token); - } - else - { - stateMachine = Checkpoint.HybridLogOnly(this, checkpointType, out token); - } + stateMachine = Checkpoint.HybridLogOnly(this, checkpointType, out token); } return stateMachineDriver.Register(stateMachine, cancellationToken); } - /// - /// Whether we can take an incremental snapshot checkpoint given current state of the store - /// - /// - /// - public bool CanTakeIncrementalCheckpoint(CheckpointType checkpointType, out Guid guid) - { - guid = _lastSnapshotCheckpoint.info.guid; - return - checkpointType == CheckpointType.Snapshot - && guid != default - && _lastSnapshotCheckpoint.info.finalLogicalAddress > hlogBase.FlushedUntilAddress - && !hlog.HasObjectLog; - } - /// /// Take log-only checkpoint /// /// Checkpoint type - /// For snapshot, try to store as incremental delta over last snapshot /// Cancellation token /// /// (bool success, Guid token) @@ -366,9 +351,9 @@ public bool CanTakeIncrementalCheckpoint(CheckpointType checkpointType, out Guid /// Await task to complete checkpoint, if initiated successfully /// public async ValueTask<(bool success, Guid token)> TakeHybridLogCheckpointAsync(CheckpointType checkpointType, - bool tryIncremental = false, CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - var success = TryInitiateHybridLogCheckpoint(out Guid token, checkpointType, tryIncremental, cancellationToken: cancellationToken); + var success = TryInitiateHybridLogCheckpoint(out Guid token, checkpointType, cancellationToken: cancellationToken); if (success) await CompleteCheckpointAsync(cancellationToken).ConfigureAwait(false); @@ -381,15 +366,11 @@ public bool CanTakeIncrementalCheckpoint(CheckpointType checkpointType, out Guid ///     /// Number of pages to preload into memory (beyond what needs to be read for recovery) /// Whether records with versions beyond checkpoint version need to be undone (and invalidated on log) - /// specific version requested or -1 for latest version. Tsavorite will recover to the largest version number checkpointed that's smaller than the required version. /// Version we actually recovered to - public long Recover(int numPagesToPreload = -1, bool undoNextVersion = true, long recoverTo = -1) + public long Recover(int numPagesToPreload = -1, bool undoNextVersion = true) { - // Do not recover - if (recoverTo == 0) - return 0; - FindRecoveryInfo(recoverTo, out var recoveredHlcInfo, out var recoveredIcInfo); - return InternalRecover(recoveredIcInfo, recoveredHlcInfo, numPagesToPreload, undoNextVersion, recoverTo); + FindRecoveryInfo(-1, out var recoveredHlcInfo, out var recoveredIcInfo); + return InternalRecover(recoveredIcInfo, recoveredHlcInfo, numPagesToPreload, undoNextVersion); } /// @@ -416,17 +397,13 @@ public long GetRecoverVersion(long recoverTo = -1) /// /// Number of pages to preload into memory (beyond what needs to be read for recovery) /// Whether records with versions beyond checkpoint version need to be undone (and invalidated on log) - /// specific version requested or -1 for latest version. Tsavorite will recover to the largest version number checkpointed that's smaller than the required version. /// Cancellation token /// Version we actually recovered to - public ValueTask RecoverAsync(int numPagesToPreload = -1, bool undoNextVersion = true, long recoverTo = -1, + public ValueTask RecoverAsync(int numPagesToPreload = -1, bool undoNextVersion = true, CancellationToken cancellationToken = default) { - // Do not recover - if (recoverTo == 0) - return ValueTask.FromResult(0L); - FindRecoveryInfo(recoverTo, out var recoveredHlcInfo, out var recoveredIcInfo); - return InternalRecoverAsync(recoveredIcInfo, recoveredHlcInfo, numPagesToPreload, undoNextVersion, recoverTo, cancellationToken); + FindRecoveryInfo(-1, out var recoveredHlcInfo, out var recoveredIcInfo); + return InternalRecoverAsync(recoveredIcInfo, recoveredHlcInfo, numPagesToPreload, undoNextVersion, cancellationToken); } /// @@ -437,9 +414,7 @@ public ValueTask RecoverAsync(int numPagesToPreload = -1, bool undoNextVer /// Whether records with versions beyond checkpoint version need to be undone (and invalidated on log) /// Version we actually recovered to public long Recover(Guid fullCheckpointToken, int numPagesToPreload = -1, bool undoNextVersion = true) - { - return InternalRecover(fullCheckpointToken, fullCheckpointToken, numPagesToPreload, undoNextVersion, -1); - } + => InternalRecover(fullCheckpointToken, fullCheckpointToken, numPagesToPreload, undoNextVersion); /// /// Asynchronously recover from specific token (blocking operation) @@ -450,7 +425,7 @@ public long Recover(Guid fullCheckpointToken, int numPagesToPreload = -1, bool u /// Cancellation token /// Version we actually recovered to public ValueTask RecoverAsync(Guid fullCheckpointToken, int numPagesToPreload = -1, bool undoNextVersion = true, CancellationToken cancellationToken = default) - => InternalRecoverAsync(fullCheckpointToken, fullCheckpointToken, numPagesToPreload, undoNextVersion, -1, cancellationToken); + => InternalRecoverAsync(fullCheckpointToken, fullCheckpointToken, numPagesToPreload, undoNextVersion, cancellationToken); /// /// Recover from specific index and log token (blocking operation) @@ -461,7 +436,7 @@ public ValueTask RecoverAsync(Guid fullCheckpointToken, int numPagesToPrel /// Whether records with versions beyond checkpoint version need to be undone (and invalidated on log) /// Version we actually recovered to public long Recover(Guid indexCheckpointToken, Guid hybridLogCheckpointToken, int numPagesToPreload = -1, bool undoNextVersion = true) - => InternalRecover(indexCheckpointToken, hybridLogCheckpointToken, numPagesToPreload, undoNextVersion, -1); + => InternalRecover(indexCheckpointToken, hybridLogCheckpointToken, numPagesToPreload, undoNextVersion); /// /// Asynchronously recover from specific index and log token (blocking operation) @@ -473,7 +448,7 @@ public long Recover(Guid indexCheckpointToken, Guid hybridLogCheckpointToken, in /// Cancellation token /// Version we actually recovered to public ValueTask RecoverAsync(Guid indexCheckpointToken, Guid hybridLogCheckpointToken, int numPagesToPreload = -1, bool undoNextVersion = true, CancellationToken cancellationToken = default) - => InternalRecoverAsync(indexCheckpointToken, hybridLogCheckpointToken, numPagesToPreload, undoNextVersion, -1, cancellationToken); + => InternalRecoverAsync(indexCheckpointToken, hybridLogCheckpointToken, numPagesToPreload, undoNextVersion, cancellationToken); /// /// Wait for ongoing checkpoint to complete @@ -487,7 +462,7 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default token.ThrowIfCancellationRequested(); try { - await stateMachineDriver.CompleteAsync(token); + await stateMachineDriver.CompleteAsync(token).ConfigureAwait(false); } catch { @@ -499,15 +474,19 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextRead(ref TKey key, ref TInput input, ref TOutput output, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextRead(TKey key, ref TInput input, ref TOutput output, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions); OperationStatus internalStatus; - var keyHash = storeFunctions.GetKeyHashCode64(ref key); + var keyHash = storeFunctions.GetKeyHashCode64(key); do - internalStatus = InternalRead(ref key, keyHash, ref input, ref output, context, ref pcontext, sessionFunctions); + internalStatus = InternalRead(key, keyHash, ref input, ref output, context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); @@ -515,8 +494,12 @@ internal Status ContextRead [MethodImpl(MethodImplOptions.AggressiveInlining)] [SkipLocalsInit] // Span in here can be sizeable, so 0-init'ing isn't free - internal unsafe void ContextReadWithPrefetch(ref TBatch batch, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal unsafe void ContextReadWithPrefetch(ref TBatch batch, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper where TBatch : IReadArgBatch #if NET9_0_OR_GREATER , allows ref struct @@ -530,13 +513,13 @@ internal unsafe void ContextReadWithPrefetch(sessionFunctions.Ctx.ReadCopyOptions); OperationStatus internalStatus; do - internalStatus = InternalRead(ref key, hash, ref input, ref output, context, ref pcontext, sessionFunctions); + internalStatus = InternalRead(key, hash, ref input, ref output, context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); batch.SetStatus(0, HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus)); @@ -566,7 +549,7 @@ internal unsafe void ContextReadWithPrefetch= hlogBase.HeadAddress) { - Sse.Prefetch0((void*)hlog.GetPhysicalAddress(hei.Address)); + Sse.Prefetch0((void*)hlogBase.GetPhysicalAddress(hei.Address)); } } @@ -601,7 +584,7 @@ internal unsafe void ContextReadWithPrefetch(sessionFunctions.Ctx.ReadCopyOptions); OperationStatus internalStatus; do - internalStatus = InternalRead(ref key, hash, ref input, ref output, context, ref pcontext, sessionFunctions); + internalStatus = InternalRead(key, hash, ref input, ref output, context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); batch.SetStatus(i, HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus)); @@ -637,117 +620,171 @@ internal unsafe void ContextReadWithPrefetch(ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, + internal Status ContextRead(TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions, ref readOptions); OperationStatus internalStatus; - var keyHash = readOptions.KeyHash ?? storeFunctions.GetKeyHashCode64(ref key); + var keyHash = readOptions.KeyHash ?? storeFunctions.GetKeyHashCode64(key); do - internalStatus = InternalRead(ref key, keyHash, ref input, ref output, context, ref pcontext, sessionFunctions); + internalStatus = InternalRead(key, keyHash, ref input, ref output, context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - recordMetadata = status.IsCompletedSuccessfully ? new(pcontext.recordInfo, pcontext.logicalAddress) : default; - return status; + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextReadAtAddress(long address, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions, ref readOptions, noKey: true); - TKey key = default; - return ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, context, ref pcontext, sessionFunctions); + var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions, ref readOptions); + pcontext.SetIsNoKey(); + return ContextReadAtAddress(address, key: default(TKey), ref input, ref output, ref readOptions, out recordMetadata, context, ref pcontext, sessionFunctions); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions, ref readOptions, noKey: false); - return ContextReadAtAddress(address, ref key, ref input, ref output, ref readOptions, out recordMetadata, context, ref pcontext, sessionFunctions); + var pcontext = new PendingContext(sessionFunctions.Ctx.ReadCopyOptions, ref readOptions); + return ContextReadAtAddress(address, key, ref input, ref output, ref readOptions, out recordMetadata, context, ref pcontext, sessionFunctions); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private Status ContextReadAtAddress(long address, ref TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, + private Status ContextReadAtAddress(long address, TKey key, ref TInput input, ref TOutput output, ref ReadOptions readOptions, out RecordMetadata recordMetadata, TContext context, ref PendingContext pcontext, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + { + OperationStatus internalStatus; + do + internalStatus = InternalReadAtAddress(address, key, ref input, ref output, ref readOptions, context, ref pcontext, sessionFunctions); + while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); + + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextUpsert(TKey key, long keyHash, ref TInput input, + ReadOnlySpan srcStringValue, ref TOutput output, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { + var pcontext = default(PendingContext); OperationStatus internalStatus; + DiskLogRecord emptyLogRecord = default; + do - internalStatus = InternalReadAtAddress(address, ref key, ref input, ref output, ref readOptions, context, ref pcontext, sessionFunctions); + internalStatus = InternalUpsert( + key, keyHash, ref input, srcStringValue, srcObjectValue: null, in emptyLogRecord, ref output, ref context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - recordMetadata = status.IsCompletedSuccessfully ? new(pcontext.recordInfo, pcontext.logicalAddress) : default; - return status; + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextUpsert(ref TKey key, long keyHash, ref TInput input, ref TValue value, ref TOutput output, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextUpsert(TKey key, long keyHash, ref TInput input, + IHeapObject srcObjectValue, ref TOutput output, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var pcontext = default(PendingContext); OperationStatus internalStatus; + DiskLogRecord emptyLogRecord = default; do - internalStatus = InternalUpsert(ref key, keyHash, ref input, ref value, ref output, ref context, ref pcontext, sessionFunctions); + internalStatus = InternalUpsert( + key, keyHash, ref input, srcStringValue: default, srcObjectValue, in emptyLogRecord, ref output, ref context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - return status; + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextUpsert(ref TKey key, long keyHash, ref TInput input, ref TValue value, ref TOutput output, out RecordMetadata recordMetadata, - TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextUpsert(TKey key, long keyHash, ref TInput input, + in TSourceLogRecord inputLogRecord, ref TOutput output, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSourceLogRecord : ISourceLogRecord { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalUpsert(ref key, keyHash, ref input, ref value, ref output, ref context, ref pcontext, sessionFunctions); + internalStatus = InternalUpsert( + key, keyHash, ref input, srcStringValue: default, srcObjectValue: default, in inputLogRecord, ref output, ref context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - recordMetadata = status.IsCompletedSuccessfully ? new(pcontext.recordInfo, pcontext.logicalAddress) : default; - return status; + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextRMW(ref TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, + internal Status ContextRMW(TKey key, long keyHash, ref TInput input, ref TOutput output, out RecordMetadata recordMetadata, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalRMW(ref key, keyHash, ref input, ref output, ref context, ref pcontext, sessionFunctions); + internalStatus = InternalRMW(key, keyHash, ref input, ref output, ref context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - recordMetadata = status.IsCompletedSuccessfully ? new(pcontext.recordInfo, pcontext.logicalAddress) : default; - return status; + recordMetadata = new(pcontext.logicalAddress); + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal Status ContextDelete(ref TKey key, long keyHash, TContext context, TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal Status ContextDelete(TKey key, long keyHash, TContext context, TSessionFunctionsWrapper sessionFunctions) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalDelete(ref key, keyHash, ref context, ref pcontext, sessionFunctions); + internalStatus = InternalDelete(key, keyHash, ref context, ref pcontext, sessionFunctions); while (HandleImmediateRetryStatus(internalStatus, sessionFunctions, ref pcontext)); - var status = HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); - return status; + return HandleOperationStatus(sessionFunctions.Ctx, ref pcontext, internalStatus); } /// @@ -759,9 +796,9 @@ public async Task GrowIndexAsync() if (epoch.ThisInstanceProtected()) throw new TsavoriteException("Cannot use GrowIndex when using non-async sessions"); - var indexResizeTask = new IndexResizeSMTask(this); + var indexResizeTask = new IndexResizeSMTask(this); var indexResizeSM = new IndexResizeSM(indexResizeTask); - return await stateMachineDriver.RunAsync(indexResizeSM); + return await stateMachineDriver.RunAsync(indexResizeSM).ConfigureAwait(false); } /// @@ -771,9 +808,8 @@ public void Dispose() { Free(); hlogBase.Dispose(); - readCacheBase?.Dispose(); + readcacheBase?.Dispose(); LockTable.Dispose(); - _lastSnapshotCheckpoint.Dispose(); if (disposeCheckpointManager) checkpointManager?.Dispose(); RevivificationManager.Dispose(); @@ -792,16 +828,16 @@ private unsafe long GetEntryCount() long total_entry_count = 0; long beginAddress = hlogBase.BeginAddress; - for (long bucket = 0; bucket < table_size_; ++bucket) + for (long bucket = 0; bucket < table_size_; bucket++) { HashBucket b = *(ptable_ + bucket); while (true) { - for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; ++bucket_entry) + for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; bucket_entry++) if (b.bucket_entries[bucket_entry] >= beginAddress) ++total_entry_count; - if ((b.bucket_entries[Constants.kOverflowBucketIndex] & Constants.kAddressMask) == 0) break; - b = *(HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(b.bucket_entries[Constants.kOverflowBucketIndex] & Constants.kAddressMask); + if ((b.bucket_entries[Constants.kOverflowBucketIndex] & kAddressBitMask) == 0) break; + b = *(HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(b.bucket_entries[Constants.kOverflowBucketIndex] & kAddressBitMask); } } return total_entry_count; @@ -824,7 +860,7 @@ private unsafe string DumpDistributionInternal(int version) Dictionary slots_unused_by_nonofb_buckets_histogram = new(); Dictionary slots_unused_by_ofb_buckets_histogram = new(); - for (long bucket = 0; bucket < table_size_; ++bucket) + for (long bucket = 0; bucket < table_size_; bucket++) { bool is_bucket_in_ofb_table = false; List tags = new(); @@ -835,7 +871,7 @@ private unsafe string DumpDistributionInternal(int version) { // per bucket calculate the number of zero'd out slots int zeroed_out_slots = 0; - for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; ++bucket_entry) + for (int bucket_entry = 0; bucket_entry < Constants.kOverflowBucketIndex; bucket_entry++) { var x = default(HashBucketEntry); x.word = b.bucket_entries[bucket_entry]; @@ -843,7 +879,7 @@ private unsafe string DumpDistributionInternal(int version) if (x.Tentative) ++total_entries_with_tentative_bit_set; - if (((!x.ReadCache) && (x.Address >= beginAddress)) || (x.ReadCache && (x.AbsoluteAddress >= readCacheBase.HeadAddress))) + if (((!x.IsReadCache) && (x.Address >= beginAddress)) || (x.IsReadCache && (x.Address >= readcacheBase.HeadAddress))) { if (tags.Contains(x.Tag) && !x.Tentative) throw new TsavoriteException("Duplicate tag found in index"); @@ -882,10 +918,10 @@ private unsafe string DumpDistributionInternal(int version) slots_unused_by_nonofb_buckets_histogram[zeroed_out_slots]++; } - if ((b.bucket_entries[Constants.kOverflowBucketIndex] & Constants.kAddressMask) == 0) + if ((b.bucket_entries[Constants.kOverflowBucketIndex] & kAddressBitMask) == 0) break; - b = *(HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(b.bucket_entries[Constants.kOverflowBucketIndex] & Constants.kAddressMask); + b = *(HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(b.bucket_entries[Constants.kOverflowBucketIndex] & kAddressBitMask); is_bucket_in_ofb_table = true; } diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteBase.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteBase.cs index 2aef03a0e44..7655c75745e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteBase.cs @@ -8,6 +8,8 @@ namespace Tsavorite.core { + using static LogAddress; + internal unsafe struct InternalHashTable { public long size; @@ -144,7 +146,7 @@ internal bool FindTag(ref HashEntryInfo hei) do { // Search through the bucket looking for our key. Last entry is reserved for the overflow pointer. - for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) + for (int index = 0; index < Constants.kOverflowBucketIndex; index++) { target_entry_word = *(((long*)hei.bucket) + index); if (0 == target_entry_word) @@ -159,7 +161,7 @@ internal bool FindTag(ref HashEntryInfo hei) } // Go to next bucket in the chain (if it is a nonzero overflow allocation) - target_entry_word = *(((long*)hei.bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask; + target_entry_word = *(((long*)hei.bucket) + Constants.kOverflowBucketIndex) & kAddressBitMask; if (target_entry_word == 0) { // We lock the firstBucket, so it can't be cleared. @@ -189,7 +191,7 @@ internal void FindOrCreateTag(ref HashEntryInfo hei, long BeginAddress) // Install tentative tag in free slot hei.entry = default; hei.entry.Tag = hei.tag; - hei.entry.Address = Constants.kTempInvalidAddress; + hei.entry.Address = kTempInvalidAddress; hei.entry.Tentative = true; // Insert the tag into this slot. Failure means another session inserted a key into that slot, so continue the loop to find another free slot. @@ -228,7 +230,7 @@ private bool FindTagOrFreeInternal(ref HashEntryInfo hei, long BeginAddress = 0) do { // Search through the bucket looking for our key. Last entry is reserved for the overflow pointer. - for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) + for (int index = 0; index < Constants.kOverflowBucketIndex; index++) { target_entry_word = *(((long*)hei.bucket) + index); if (0 == target_entry_word) @@ -244,9 +246,9 @@ private bool FindTagOrFreeInternal(ref HashEntryInfo hei, long BeginAddress = 0) // If the entry points to an address that has been truncated, it's free; try to reclaim it by setting its word to 0. hei.entry.word = target_entry_word; - if (hei.entry.Address < BeginAddress && hei.entry.Address != Constants.kTempInvalidAddress) + if (hei.entry.Address < BeginAddress && hei.entry.Address != kTempInvalidAddress) { - if (hei.entry.word == Interlocked.CompareExchange(ref hei.bucket->bucket_entries[index], Constants.kInvalidAddress, target_entry_word)) + if (hei.entry.word == Interlocked.CompareExchange(ref hei.bucket->bucket_entries[index], kInvalidAddress, target_entry_word)) { if (hei.slot == Constants.kInvalidEntrySlot) { @@ -266,7 +268,7 @@ private bool FindTagOrFreeInternal(ref HashEntryInfo hei, long BeginAddress = 0) // Go to next bucket in the chain (if it is a nonzero overflow allocation). Don't mask off the non-address bits here; they're needed for CAS. target_entry_word = *(((long*)hei.bucket) + Constants.kOverflowBucketIndex); - while ((target_entry_word & Constants.kAddressMask) == 0) + while ((target_entry_word & kAddressBitMask) == 0) { // There is no next bucket. If slot is Constants.kInvalidEntrySlot then we did not find an empty slot, so must allocate a new bucket. if (hei.slot == Constants.kInvalidEntrySlot) @@ -276,7 +278,7 @@ private bool FindTagOrFreeInternal(ref HashEntryInfo hei, long BeginAddress = 0) var physicalBucketAddress = (HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(logicalBucketAddress); long compare_word = target_entry_word; target_entry_word = logicalBucketAddress; - target_entry_word |= compare_word & ~Constants.kAddressMask; + target_entry_word |= compare_word & ~kAddressBitMask; long result_word = Interlocked.CompareExchange( ref hei.bucket->bucket_entries[Constants.kOverflowBucketIndex], @@ -305,7 +307,7 @@ private bool FindTagOrFreeInternal(ref HashEntryInfo hei, long BeginAddress = 0) } // The next bucket was there or was allocated. Move to it. - hei.bucket = (HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word & Constants.kAddressMask); + hei.bucket = (HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word & kAddressBitMask); } while (true); } @@ -323,7 +325,7 @@ private bool FindOtherSlotForThisTagMaybeTentativeInternal(ushort tag, ref HashB do { // Search through the bucket looking for our key. Last entry is reserved for the overflow pointer. - for (int index = 0; index < Constants.kOverflowBucketIndex; ++index) + for (int index = 0; index < Constants.kOverflowBucketIndex; index++) { target_entry_word = *(((long*)bucket) + index); if (0 == target_entry_word) @@ -341,7 +343,7 @@ private bool FindOtherSlotForThisTagMaybeTentativeInternal(ushort tag, ref HashB } // Go to next bucket in the chain (if it is a nonzero overflow allocation). - target_entry_word = *(((long*)bucket) + Constants.kOverflowBucketIndex) & Constants.kAddressMask; + target_entry_word = *(((long*)bucket) + Constants.kOverflowBucketIndex) & kAddressBitMask; if (target_entry_word == 0) return false; bucket = (HashBucket*)overflowBucketsAllocator.GetPhysicalAddress(target_entry_word); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteIterator.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteIterator.cs index 640575d0957..796f8a60357 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteIterator.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteIterator.cs @@ -7,9 +7,9 @@ namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { /// /// Pull iterator for all (distinct) live key-values stored in Tsavorite @@ -17,12 +17,12 @@ public partial class TsavoriteKV : Ts /// Functions used to manage key-values during iteration /// Report records until this address (tail by default) /// Tsavorite iterator - public ITsavoriteScanIterator Iterate(TFunctions functions, long untilAddress = -1) - where TFunctions : ISessionFunctions + public ITsavoriteScanIterator Iterate(TFunctions functions, long untilAddress = -1) + where TFunctions : ISessionFunctions { if (untilAddress == -1) untilAddress = Log.TailAddress; - return new TsavoriteKVIterator(this, functions, untilAddress, loggerFactory: loggerFactory); + return new TsavoriteKVIterator(this, functions, untilAddress, loggerFactory: loggerFactory); } /// @@ -33,19 +33,19 @@ public ITsavoriteScanIterator IterateReport records until this address (tail by default) /// Tsavorite iterator public bool Iterate(TFunctions functions, ref TScanFunctions scanFunctions, long untilAddress = -1) - where TFunctions : ISessionFunctions - where TScanFunctions : IScanIteratorFunctions + where TFunctions : ISessionFunctions + where TScanFunctions : IScanIteratorFunctions { if (untilAddress == -1) untilAddress = Log.TailAddress; - using TsavoriteKVIterator iter = new(this, functions, untilAddress, loggerFactory: loggerFactory); + using TsavoriteKVIterator iter = new(this, functions, untilAddress, loggerFactory: loggerFactory); if (!scanFunctions.OnStart(iter.BeginAddress, iter.EndAddress)) return false; long numRecords = 1; - bool stop = false; - for (; !stop && iter.PushNext(ref scanFunctions, numRecords, out stop); ++numRecords) + var stop = false; + for (; !stop && iter.PushNext(ref scanFunctions, numRecords, out stop); numRecords++) ; scanFunctions.OnStop(!stop, numRecords); @@ -53,18 +53,17 @@ public bool Iterate(TFunc } } - internal sealed class TsavoriteKVIterator : ITsavoriteScanIterator - where TFunctions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal sealed class TsavoriteKVIterator : ITsavoriteScanIterator + where TFunctions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - private readonly TsavoriteKV store; - private readonly TsavoriteKV tempKv; - private readonly ClientSession tempKvSession; - private readonly BasicContext tempbContext; - private readonly ITsavoriteScanIterator mainKvIter; - private readonly IPushScanIterator pushScanIterator; - private ITsavoriteScanIterator tempKvIter; + private readonly TsavoriteKV store; + private readonly TsavoriteKV tempKv; + private readonly ClientSession tempKvSession; + private readonly BasicContext tempbContext; + private ITsavoriteScanIterator mainKvIter; + private ITsavoriteScanIterator tempKvIter; enum IterationPhase { @@ -74,34 +73,35 @@ enum IterationPhase }; private IterationPhase iterationPhase; - public TsavoriteKVIterator(TsavoriteKV store, TFunctions functions, long untilAddress, ILoggerFactory loggerFactory = null) + public TsavoriteKVIterator(TsavoriteKV store, TFunctions functions, long untilAddress, ILoggerFactory loggerFactory = null) { this.store = store; iterationPhase = IterationPhase.MainKv; - var tempKVSettings = new KVSettings(baseDir: null, loggerFactory: loggerFactory) + var tempKVSettings = new KVSettings(baseDir: null, loggerFactory: loggerFactory) { - IndexSize = KVSettings.SetIndexSizeFromCacheLines(store.IndexSize), + IndexSize = KVSettings.SetIndexSizeFromCacheLines(store.IndexSize), LogDevice = new NullDevice(), ObjectLogDevice = new NullDevice(), MutableFraction = 1, loggerFactory = loggerFactory }; - tempKv = new TsavoriteKV(tempKVSettings, store.storeFunctions, store.allocatorFactory); - tempKvSession = tempKv.NewSession(functions); + tempKv = new TsavoriteKV(tempKVSettings, store.storeFunctions, store.allocatorFactory); + tempKvSession = tempKv.NewSession(functions); tempbContext = tempKvSession.BasicContext; mainKvIter = store.Log.Scan(store.Log.BeginAddress, untilAddress); - pushScanIterator = mainKvIter as IPushScanIterator; } - public long CurrentAddress => iterationPhase == IterationPhase.MainKv ? mainKvIter.CurrentAddress : tempKvIter.CurrentAddress; + ITsavoriteScanIterator CurrentIter => iterationPhase == IterationPhase.MainKv ? mainKvIter : tempKvIter; - public long NextAddress => iterationPhase == IterationPhase.MainKv ? mainKvIter.NextAddress : tempKvIter.NextAddress; + public long CurrentAddress => CurrentIter.CurrentAddress; - public long BeginAddress => iterationPhase == IterationPhase.MainKv ? mainKvIter.BeginAddress : tempKvIter.BeginAddress; + public long NextAddress => CurrentIter.NextAddress; - public long EndAddress => iterationPhase == IterationPhase.MainKv ? mainKvIter.EndAddress : tempKvIter.EndAddress; + public long BeginAddress => CurrentIter.BeginAddress; + + public long EndAddress => CurrentIter.EndAddress; public void Dispose() { @@ -111,24 +111,19 @@ public void Dispose() tempKv?.Dispose(); } - public ref TKey GetKey() => ref iterationPhase == IterationPhase.MainKv ? ref mainKvIter.GetKey() : ref tempKvIter.GetKey(); - - public ref TValue GetValue() => ref iterationPhase == IterationPhase.MainKv ? ref mainKvIter.GetValue() : ref tempKvIter.GetValue(); - - public bool GetNext(out RecordInfo recordInfo) + public bool GetNext() { while (true) { if (iterationPhase == IterationPhase.MainKv) { - if (mainKvIter.GetNext(out recordInfo)) + if (mainKvIter.GetNext()) { - ref var key = ref mainKvIter.GetKey(); - OperationStackContext stackCtx = default; - if (IsTailmostMainKvRecord(ref key, recordInfo, ref stackCtx)) + OperationStackContext stackCtx = default; + if (IsTailmostMainKvRecord(mainKvIter, mainKvIter.Info, ref stackCtx)) return true; - ProcessNonTailmostMainKvRecord(recordInfo, key); + ProcessNonTailmostMainKvRecord(mainKvIter.Info, mainKvIter); continue; } @@ -140,9 +135,9 @@ public bool GetNext(out RecordInfo recordInfo) if (iterationPhase == IterationPhase.TempKv) { - if (tempKvIter.GetNext(out recordInfo)) + if (tempKvIter.GetNext()) { - if (!recordInfo.Tombstone) + if (!tempKvIter.Info.Tombstone) return true; continue; } @@ -153,35 +148,30 @@ public bool GetNext(out RecordInfo recordInfo) } // We're done. This handles both the call that exhausted tempKvIter, and any subsequent calls on this outer iterator. - recordInfo = default; return false; } } internal bool PushNext(ref TScanFunctions scanFunctions, long numRecords, out bool stop) - where TScanFunctions : IScanIteratorFunctions + where TScanFunctions : IScanIteratorFunctions { while (true) { if (iterationPhase == IterationPhase.MainKv) { - OperationStackContext stackCtx = default; - if (mainKvIter.GetNext(out var recordInfo)) + OperationStackContext stackCtx = default; + if (mainKvIter.GetNext()) { try { - ref var key = ref mainKvIter.GetKey(); - if (IsTailmostMainKvRecord(ref key, recordInfo, ref stackCtx)) + if (IsTailmostMainKvRecord(mainKvIter, mainKvIter.Info, ref stackCtx)) { - // Push Iter records are in temp storage so do not need locks, but we'll call ConcurrentReader because, for example, GenericAllocator - // may need to know the object is in that region. - stop = mainKvIter.CurrentAddress >= store.hlogBase.ReadOnlyAddress - ? !scanFunctions.ConcurrentReader(ref key, ref mainKvIter.GetValue(), new RecordMetadata(recordInfo, mainKvIter.CurrentAddress), numRecords, out _) - : !scanFunctions.SingleReader(ref key, ref mainKvIter.GetValue(), new RecordMetadata(recordInfo, mainKvIter.CurrentAddress), numRecords, out _); + // Push Iter records are in temp storage so do not need locks. + stop = !scanFunctions.Reader(in mainKvIter, new RecordMetadata(mainKvIter.CurrentAddress), numRecords, out _); return !stop; } - ProcessNonTailmostMainKvRecord(recordInfo, key); + ProcessNonTailmostMainKvRecord(mainKvIter.Info, mainKvIter); continue; } catch (Exception ex) @@ -204,11 +194,11 @@ internal bool PushNext(ref TScanFunctions scanFunctions, long nu if (iterationPhase == IterationPhase.TempKv) { - if (tempKvIter.GetNext(out var recordInfo)) + if (tempKvIter.GetNext()) { - if (!recordInfo.Tombstone) + if (!tempKvIter.Info.Tombstone) { - stop = !scanFunctions.SingleReader(ref tempKvIter.GetKey(), ref tempKvIter.GetValue(), new RecordMetadata(recordInfo, tempKvIter.CurrentAddress), numRecords, out _); + stop = !scanFunctions.Reader(in tempKvIter, new RecordMetadata(tempKvIter.CurrentAddress), numRecords, out _); return !stop; } continue; @@ -225,23 +215,26 @@ internal bool PushNext(ref TScanFunctions scanFunctions, long nu } } - private void ProcessNonTailmostMainKvRecord(RecordInfo recordInfo, TKey key) + private void ProcessNonTailmostMainKvRecord(RecordInfo recordInfo, ITsavoriteScanIterator key) { // Not the tailmost record in the tag chain so add it to or remove it from tempKV (we want to return only the latest version). if (recordInfo.Tombstone) { // Check if it's in-memory first so we don't spuriously create a tombstone record. - if (tempbContext.ContainsKeyInMemory(ref key, out _).Found) - _ = tempbContext.Delete(ref key); + if (tempbContext.ContainsKeyInMemory(key, out _).Found) + _ = tempbContext.Delete(key); } else - _ = tempbContext.Upsert(ref key, ref mainKvIter.GetValue()); + { + var iterLogRecord = mainKvIter as ISourceLogRecord; // Can't use 'ref' on a 'using' variable + _ = tempbContext.Upsert(in iterLogRecord); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] - bool IsTailmostMainKvRecord(ref TKey key, RecordInfo mainKvRecordInfo, ref OperationStackContext stackCtx) + bool IsTailmostMainKvRecord(ITsavoriteScanIterator key, RecordInfo mainKvRecordInfo, ref OperationStackContext stackCtx) { - stackCtx = new(store.storeFunctions.GetKeyHashCode64(ref key)); + stackCtx = new(store.storeFunctions.GetKeyHashCode64(key)); if (store.FindTag(ref stackCtx.hei)) { stackCtx.SetRecordSourceToHashEntry(store.hlogBase); @@ -253,8 +246,8 @@ bool IsTailmostMainKvRecord(ref TKey key, RecordInfo mainKvRecordInfo, ref Opera if (mainKvRecordInfo.PreviousAddress >= store.Log.BeginAddress) { // Check if it's in-memory first so we don't spuriously create a tombstone record. - if (tempbContext.ContainsKeyInMemory(ref key, out _).Found) - tempbContext.Delete(ref key); + if (tempbContext.ContainsKeyInMemory(key, out _).Found) + _ = tempbContext.Delete(key); } // If the record is not deleted, we can let the caller process it directly within mainKvIter. @@ -264,26 +257,110 @@ bool IsTailmostMainKvRecord(ref TKey key, RecordInfo mainKvRecordInfo, ref Opera return false; } - public bool GetNext(out RecordInfo recordInfo, out TKey key, out TValue value) + #region ISourceLogRecord + /// + public ref RecordInfo InfoRef => ref CurrentIter.InfoRef; + /// + public RecordInfo Info => CurrentIter.Info; + + /// + public byte RecordType => CurrentIter.RecordType; + + /// + public ReadOnlySpan Namespace => CurrentIter.Namespace; + + /// + public ObjectIdMap ObjectIdMap => CurrentIter.ObjectIdMap; + + /// + public bool IsSet => !CurrentIter.IsSet; + + /// + public ReadOnlySpan Key => CurrentIter.Key; + + /// + public bool IsPinnedKey => CurrentIter.IsPinnedKey; + + /// + public unsafe byte* PinnedKeyPointer => CurrentIter.PinnedKeyPointer; + + /// + public OverflowByteArray KeyOverflow { - if (GetNext(out recordInfo)) - { - if (iterationPhase == IterationPhase.MainKv) - { - key = mainKvIter.GetKey(); - value = mainKvIter.GetValue(); - } - else - { - key = tempKvIter.GetKey(); - value = tempKvIter.GetValue(); - } - return true; - } + get => CurrentIter.KeyOverflow; + set => CurrentIter.KeyOverflow = value; + } - key = default; - value = default; - return false; + /// + public unsafe Span ValueSpan => CurrentIter.ValueSpan; + + /// + public IHeapObject ValueObject => CurrentIter.ValueObject; + + /// + public bool IsPinnedValue => CurrentIter.IsPinnedValue; + + /// + public unsafe byte* PinnedValuePointer => CurrentIter.PinnedValuePointer; + + /// + public OverflowByteArray ValueOverflow + { + get => CurrentIter.ValueOverflow; + set => CurrentIter.ValueOverflow = value; } + + /// + public SpanByteAndMemory ValueSpanByteAndMemory => CurrentIter.ValueSpanByteAndMemory; + + /// + public long ETag => CurrentIter.ETag; + + /// + public long Expiration => CurrentIter.Expiration; + + /// + public void ClearValueIfHeap() { } // Not relevant for "iterator as logrecord" + + /// + public bool IsMemoryLogRecord => CurrentIter.IsMemoryLogRecord; + + /// + public unsafe ref LogRecord AsMemoryLogRecordRef() => throw new InvalidOperationException("Cannot cast a TsavoriteKVIterator to a memory LogRecord."); + + /// + public bool IsDiskLogRecord => CurrentIter.IsDiskLogRecord; + + /// + public unsafe ref DiskLogRecord AsDiskLogRecordRef() => ref CurrentIter.AsDiskLogRecordRef(); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public RecordFieldInfo GetRecordFieldInfo() => CurrentIter.GetRecordFieldInfo(); + + /// + public int AllocatedSize => CurrentIter.AllocatedSize; + + /// + public int ActualSize => CurrentIter.ActualSize; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long CalculateHeapMemorySize() => CurrentIter.CalculateHeapMemorySize(); + #endregion // ISourceLogRecord + + #region IKey + /// + public bool IsPinned => IsPinnedKey; + + /// + public ReadOnlySpan KeyBytes => Key; + + /// + public bool HasNamespace => CurrentIter.HasNamespace; + + /// + public ReadOnlySpan NamespaceBytes => CurrentIter.NamespaceBytes; + #endregion } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs index e0d7e6a54f2..a7d41aaa20e 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/TsavoriteThread.cs @@ -7,13 +7,13 @@ namespace Tsavorite.core { - public partial class TsavoriteKV : TsavoriteBase - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + public partial class TsavoriteKV : TsavoriteBase + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { [MethodImpl(MethodImplOptions.AggressiveInlining)] internal void InternalRefresh(TSessionFunctionsWrapper sessionFunctions) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { epoch.ProtectAndDrain(); @@ -40,7 +40,7 @@ internal void InternalRefresh(TSessionFunctionsWrapper sessionFunctions, bool wait = false, - CompletedOutputIterator completedOutputs = null) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + CompletedOutputIterator completedOutputs = null) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { while (true) { - InternalCompletePendingRequests(sessionFunctions, completedOutputs); - if (wait) sessionFunctions.Ctx.WaitPending(epoch); + InternalCompletePendingRequests(sessionFunctions, completedOutputs); + if (wait) + sessionFunctions.Ctx.WaitPending(epoch); - if (sessionFunctions.Ctx.HasNoPendingRequests) return true; + if (sessionFunctions.Ctx.HasNoPendingRequests) + return true; InternalRefresh(sessionFunctions); - if (!wait) return false; - Thread.Yield(); + if (!wait) + return false; + _ = Thread.Yield(); } } @@ -74,20 +77,21 @@ internal bool InternalCompletePending(TSessionFunctionsWrapper sessionFunctions, - CompletedOutputIterator completedOutputs) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + CompletedOutputIterator completedOutputs) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { _ = hlogBase.TryComplete(); - if (sessionFunctions.Ctx.readyResponses.Count == 0) return; + if (sessionFunctions.Ctx.readyResponses.Count == 0) + return; - while (sessionFunctions.Ctx.readyResponses.TryDequeue(out AsyncIOContext request)) + while (sessionFunctions.Ctx.readyResponses.TryDequeue(out AsyncIOContext request)) InternalCompletePendingRequest(sessionFunctions, request, completedOutputs); } - internal void InternalCompletePendingRequest(TSessionFunctionsWrapper sessionFunctions, AsyncIOContext request, - CompletedOutputIterator completedOutputs) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal void InternalCompletePendingRequest(TSessionFunctionsWrapper sessionFunctions, AsyncIOContext request, + CompletedOutputIterator completedOutputs) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { // Get and Remove this request.id pending dictionary if it is there. if (sessionFunctions.Ctx.ioPendingRequests.Remove(request.id, out var pendingContext)) @@ -99,27 +103,29 @@ internal void InternalCompletePendingRequest /// Caller is expected to dispose pendingContext after this method completes /// - internal Status InternalCompletePendingRequestFromContext(TSessionFunctionsWrapper sessionFunctions, AsyncIOContext request, - ref PendingContext pendingContext, out AsyncIOContext newRequest) - where TSessionFunctionsWrapper : ISessionFunctionsWrapper + internal unsafe Status InternalCompletePendingRequestFromContext(TSessionFunctionsWrapper sessionFunctions, AsyncIOContext request, + ref PendingContext pendingContext, out AsyncIOContext newRequest) + where TSessionFunctionsWrapper : ISessionFunctionsWrapper { - Debug.Assert(epoch.ThisInstanceProtected(), "InternalCompletePendingRequestFromContext requires epoch acquision"); + Debug.Assert(epoch.ThisInstanceProtected(), "InternalCompletePendingRequestFromContext requires epoch acquisition"); newRequest = default; - // If NoKey, we do not have the key in the initial call and must use the key from the satisfied request. - // With the new overload of CompletePending that returns CompletedOutputs, pendingContext must have the key. - if (pendingContext.NoKey && pendingContext.key == default) - pendingContext.key = hlog.GetKeyContainer(ref hlog.GetContextRecordKey(ref request)); - ref TKey key = ref pendingContext.key.Get(); + // If this was an operation that was trying to retrieve a target record, copy it into the pendingContext. + // CONDITIONAL_* operations do not care about the retrieved data; they only care whether a record was found. + if (request.diskLogRecord.IsSet && !pendingContext.IsConditionalOp) + pendingContext.TransferFrom(ref request.diskLogRecord, hlogBase.bufferPool); - OperationStatus internalStatus = pendingContext.type switch + var internalStatus = pendingContext.type switch { OperationType.READ => ContinuePendingRead(request, ref pendingContext, sessionFunctions), OperationType.RMW => ContinuePendingRMW(request, ref pendingContext, sessionFunctions), @@ -135,30 +141,26 @@ internal Status InternalCompletePendingRequestFromContext - /// The reason a SingleWriter was performed - /// - public enum WriteReason : byte - { - /// A new record appended by Upsert - Upsert, - - /// Copying a read from disk to the tail of the log - CopyToTail, - - /// Copying a read from disk to the read cache - CopyToReadCache, - - /// The user called Compact() - Compaction - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Tsavorite.core.csproj b/libs/storage/Tsavorite/cs/src/core/Tsavorite.core.csproj index b91f3e8132d..6e63c51fdb9 100644 --- a/libs/storage/Tsavorite/cs/src/core/Tsavorite.core.csproj +++ b/libs/storage/Tsavorite/cs/src/core/Tsavorite.core.csproj @@ -11,6 +11,22 @@ ;NU1605 + + 1701;1702;IDE0130 + + + + 1701;1702;IDE0130 + + + + 1701;1702;IDE0130 + + + + 1701;1702;IDE0130 + + @@ -32,6 +48,12 @@ + + + + + + diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/ILogCommitManager.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/ILogCommitManager.cs index 7d561d11eb7..e48ed668e04 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/ILogCommitManager.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/ILogCommitManager.cs @@ -7,7 +7,7 @@ namespace Tsavorite.core { /// - /// Log commit manager + /// TsavoriteLog commit manager /// public interface ILogCommitManager : IDisposable { @@ -41,7 +41,7 @@ public interface ILogCommitManager : IDisposable public void RemoveCommit(long commitNum); /// - /// Remove all log commits from this manager + /// Remove all TsavoriteLog commits from this manager /// public void RemoveAllCommits(); diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/LogCommitPolicy.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/LogCommitPolicy.cs index ba9c0a5774a..933021d10fe 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/LogCommitPolicy.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/LogCommitPolicy.cs @@ -50,7 +50,7 @@ public abstract class LogCommitPolicy /// return false. /// /// policy object - public static LogCommitPolicy Default() => new DefaultLogCommitPolicy(); + public static LogCommitPolicy Default() => new DefaulLogCommitPolicy(); /// /// MaxParallel log commit policy allows k (non-strong) commit requests to be in progress at any giving time. The k commits are guaranteed @@ -76,7 +76,7 @@ public abstract class LogCommitPolicy public static LogCommitPolicy RateLimit(long thresholdMilli, long thresholdBytes) => new RateLimitLogCommitPolicy(thresholdMilli, thresholdBytes); } - internal sealed class DefaultLogCommitPolicy : LogCommitPolicy + internal sealed class DefaulLogCommitPolicy : LogCommitPolicy { /// public override void OnAttached(TsavoriteLog log) { } @@ -175,7 +175,7 @@ public override bool AdmitCommit(long currentTail, bool commitRequired) { Task.Run(async () => { - await Task.Delay(TimeSpan.FromMilliseconds(thresholdMilli)); + await Task.Delay(TimeSpan.FromMilliseconds(thresholdMilli)).ConfigureAwait(false); shouldRetry = 0; log.Commit(); }); diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs index fbaa4852054..b24fe0663dd 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs @@ -13,16 +13,16 @@ namespace Tsavorite.core { - using EmptyStoreFunctions = StoreFunctions>; + using static LogAddress; /// - /// Tsavorite log + /// Tsavorite Log /// public sealed class TsavoriteLog : IDisposable { private Exception cannedException = null; - readonly BlittableAllocatorImpl allocator; + readonly TsavoriteLogAllocatorImpl allocator; readonly LightEpoch epoch; readonly bool isEpochOwned; readonly ILogCommitManager logCommitManager; @@ -43,7 +43,7 @@ public sealed class TsavoriteLog : IDisposable // Offsets for all currently unprocessed commit records readonly Queue<(long, TsavoriteLogRecoveryInfo)> ongoingCommitRequests; - readonly List coveredCommits = new(); + readonly List coveredCommits = []; long commitNum, commitCoveredAddress; readonly LogCommitPolicy commitPolicy; @@ -73,9 +73,21 @@ public sealed class TsavoriteLog : IDisposable public long FlushedUntilAddress => allocator.FlushedUntilAddress; /// - /// Log safe read-only address + /// Log safe read-only address. + /// This is the largest address below which every byte has been fully written and is safe to + /// read by uncommitted iterators / replication streams. Computed lazily via a min fold + /// over per-thread in-flight slot publishes in the user-word column, + /// clamped by above and by commit/recovery/reset floors below. + /// Reads are O(1) (return the monotonically-advanced cache); call + /// to force recomputation from the current in-flight state. /// - public long SafeTailAddress; + public long SafeTailAddress => Volatile.Read(ref cachedSafeTailAddress); + + /// + /// Monotonically-advanced cache of the safe tail address. Advanced by + /// and by commit/recovery/reset paths. + /// + long cachedSafeTailAddress; /// /// Log committed until address @@ -93,7 +105,7 @@ public sealed class TsavoriteLog : IDisposable public byte[] RecoveredCookie; /// - /// Header size used by TsavoriteLog + /// Header size used by TsavoriteLog, for entryLength and possibly checkSum /// public int HeaderSize => headerSize; @@ -105,7 +117,7 @@ public sealed class TsavoriteLog : IDisposable /// /// Task notifying log flush completions /// - internal CompletionEvent FlushEvent => allocator.FlushEvent; + internal CompletionEvent FlushEvent => allocator.flushEvent; /// /// Committed view of commitMetadataVersion @@ -121,44 +133,42 @@ public sealed class TsavoriteLog : IDisposable readonly ILogger logger; /// - /// SafeTailAddress refresh frequency in milliseconds. -1 => disabled; 0 => immediate refresh after every enqueue, >1 => refresh period in milliseconds. - /// - readonly int SafeTailRefreshFrequencyMs; - - /// - /// CTS to allow cancellation of the safe tail refresh background task, called during Dispose + /// Index of the user-word slot used by this log to track in-flight enqueue + /// slot start addresses (or when the thread is not currently enqueueing). + /// The minimum value across the column, clamped above by , yields + /// — the largest address below which every byte has been fully written. + /// See the "SafeTail via per-thread in-flight publish" region below for the protocol. /// - readonly CancellationTokenSource safeTailRefreshTaskCts; + readonly int inflightWord; /// - /// Last captured safe tail address before epoch bump + /// Sentinel written to the in-flight slot when the thread has no enqueue in progress. Chosen as + /// so that idle threads contribute neutrally to the min fold + /// that computes SafeTailAddress. /// - long safeTailRefreshLastTailAddress = 0; + const long InflightInactive = long.MaxValue; /// - /// Events to control callback execution + /// Callback fired when the safe tail crosses a page boundary. Arguments are the old and new + /// . Fires at most once per page — byte-level SafeTail advances that + /// do not cross a page boundary are coalesced. Iterators that need byte-level notification should + /// use instead of this callback. /// - readonly SingleWaiterAutoResetEvent safeTailRefreshCallbackCompleted, safeTailRefreshEntryEnqueued; + public Action SafeTailPageShiftCallback; - /// - /// Task corresponding to safe tail refresh - /// - readonly Task safeTailRefreshTask; - - /// - /// Action for bump epoch to refresh safe tail - /// - readonly Action periodicRefreshSafeTailAddressBumpCallbackAction; + /// Last published page for . Written only inside + /// the callback-dispatch path; read without locks under the monotonic-update invariant. + long lastPublishedSafeTailPage; - /// - /// Callback when safe tail shifts - /// - public Action SafeTailShiftCallback; + /// Highest page any producer has observed the tail reaching. Producers CAS this when they + /// cross into a new page and the CAS winner drives a . Ensures + /// the page-shift callback fires even with no active iterators driving scans. + long lastProducerObservedPage; /// /// Whether we automatically commit as records are inserted /// - readonly bool AutoCommit; + readonly bool autoCommit; /// /// Maximum memory size in bytes @@ -166,17 +176,17 @@ public sealed class TsavoriteLog : IDisposable public long MaxMemorySizeBytes => allocator.MaxMemorySizeBytes; /// - /// Actual memory used by log + /// Actual memory used by log. Does not include overflow free pages. /// - public long MemorySizeBytes => ((long)(allocator.AllocatedPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; + public long MemorySizeBytes => allocator.GetLogicalAddressOfStartOfPage(allocator.AllocatedPageCount); /// /// Create new log instance /// /// Log settings - /// Log settings + /// User provided logger instance public TsavoriteLog(TsavoriteLogSettings logSettings, ILogger logger = null) - : this(logSettings, logSettings.TryRecoverLatest, logger) + : this(logSettings, logSettings.TryRecoverLatest, logger: logger) { } /// @@ -188,19 +198,17 @@ public TsavoriteLog(TsavoriteLogSettings logSettings, ILogger logger = null) private TsavoriteLog(TsavoriteLogSettings logSettings, bool syncRecover, ILogger logger = null) { this.logger = logger; - AutoCommit = logSettings.AutoCommit; + autoCommit = logSettings.AutoCommit; logCommitManager = logSettings.LogCommitManager ?? new DeviceLogCommitCheckpointManager - (new LocalStorageNamedDeviceFactoryCreator(), - new DefaultCheckpointNamingScheme( - logSettings.LogCommitDir ?? - new FileInfo(logSettings.LogDevice.FileName).Directory.FullName), + (new LocalStorageNamedDeviceFactoryCreator(), + new DefaultCheckpointNamingScheme(logSettings.LogCommitDir ?? new FileInfo(logSettings.LogDevice.FileName).Directory.FullName), !logSettings.ReadOnlyMode && logSettings.RemoveOutdatedCommits); if (logSettings.LogCommitManager == null) disposeLogCommitManager = true; - // Reserve 8 byte checksum in header if requested + // Reserve 8 byte checksum in header if requested, in addition to the entry length logChecksum = logSettings.LogChecksum; headerSize = logChecksum == LogChecksumType.PerEntry ? 12 : 4; getMemory = logSettings.GetMemory; @@ -211,14 +219,12 @@ private TsavoriteLog(TsavoriteLogSettings logSettings, bool syncRecover, ILogger } else epoch = logSettings.Epoch; - CommittedUntilAddress = Constants.kFirstValidAddress; - CommittedBeginAddress = Constants.kFirstValidAddress; - SafeTailAddress = Constants.kFirstValidAddress; + + CommittedUntilAddress = FirstValidAddress; + CommittedBeginAddress = FirstValidAddress; + cachedSafeTailAddress = FirstValidAddress; commitQueue = new WorkQueueLIFO(SerialCommitCallbackWorker); - allocator = new( - new AllocatorSettings(logSettings.GetLogSettings(), epoch, logger) { flushCallback = CommitCallback }, - StoreFunctions.Create(EmptyKeyComparer.Instance), - @this => new BlittableAllocator(@this)); + allocator = new(new AllocatorSettings(logSettings.GetLogSettings(), epoch, logger) { flushCallback = CommitCallback }); allocator.Initialize(); beginAddress = allocator.BeginAddress; @@ -246,127 +252,217 @@ private TsavoriteLog(TsavoriteLogSettings logSettings, bool syncRecover, ILogger catch { } } - // Set up safe tail refresh - SafeTailRefreshFrequencyMs = logSettings.SafeTailRefreshFrequencyMs; - if (SafeTailRefreshFrequencyMs >= 0) - { - safeTailRefreshCallbackCompleted = new() - { - RunContinuationsAsynchronously = true - }; - if (SafeTailRefreshFrequencyMs == 0) - { - safeTailRefreshEntryEnqueued = new() - { - RunContinuationsAsynchronously = true - }; - } - safeTailRefreshTaskCts = new(); - periodicRefreshSafeTailAddressBumpCallbackAction = PeriodicRefreshSafeTailAddressBumpCallback; - safeTailRefreshTask = Task.Run(SafeTailRefreshWorker); - } + // Claim a LightEpoch user-word slot for our in-flight enqueue publish protocol. + // Idle threads carry the InflightInactive sentinel (long.MaxValue) so they contribute + // neutrally to the min fold that produces SafeTailAddress. + inflightWord = epoch.AllocateUserWord(InflightInactive); + } + + #region SafeTail via per-thread in-flight publish + // + // Each enqueue publishes its in-flight slot start address into a per-thread LightEpoch user-word, + // cleared when the payload write completes. SafeTailAddress = min(TailAddress, min over threads + // of inflightStart). This replaces the background-worker + epoch-bump design that previously + // maintained SafeTailAddress, eliminating the refresh-frequency tuning knob entirely. + // + // Producer protocol (must run inside epoch.Resume / before epoch.Suspend): + // 1. BeginInflightEnqueue() — publish a lower bound ≤ eventual slot start + // 2. TryAllocateRetryNow(...) — FAA advances TailAddress and returns our slot start + // 3. (payload write) + // 4. EndInflightEnqueue() — publish InflightInactive, wake parked iterators + // On allocation failure, call EndInflightEnqueue() to clear the lower bound before Suspend. + // + // The Begin pre-publish is required because otherwise a reader could observe TailAddress + // advanced past our slot while our in-flight slot still reads InflightInactive, erroneously + // concluding that region is safe. By publishing a lower bound before the FAA, any reader that + // sees TailAddress ≥ X is guaranteed (via release/acquire ordering) to also observe our slot at + // some value ≤ X. + // + // The published lower bound may be slightly below our actual slot start (by the amount other + // threads allocated between our GetTailAddress read and our FAA). This makes SafeTailAddress + // lag by at most O(N_threads × entry_size) bytes — negligible compared to page-level + // granularity of downstream consumers. We tolerate this in exchange for removing one + // Volatile.Write per enqueue from the hot path. + + /// + /// Publish a conservative lower bound into this thread's in-flight slot before the allocator's + /// FAA. The value is ≤ our eventual slot start because is + /// monotonic and the FAA can only increase it. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void BeginInflightEnqueue() + { + Volatile.Write(ref epoch.ThisThreadUserWord(inflightWord), allocator.GetTailAddress()); } - async Task SafeTailRefreshWorker() + /// + /// Clear this thread's in-flight publish (mark not-in-flight) and wake any parked iterators / + /// awaiters. Safe to call unconditionally at the end of an + /// enqueue (success or failure). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void EndInflightEnqueue() { - try - { - var token = safeTailRefreshTaskCts.Token; + Volatile.Write(ref epoch.ThisThreadUserWord(inflightWord), InflightInactive); + NotifyParkedWaiters(); + MaybeProducerDriveSafeTail(); + } - // Outer loop makes the worker wake up every so often (either delay or enqueue-signal) - // and try to move SafeTailAddress towards TailAddress - while (!token.IsCancellationRequested) - { - // Inner loop keeps moving SafeTailAddress towards TailAddress until we have - // caught up and there is no more movement necessary. - while (!token.IsCancellationRequested) - { - try - { - // Resume epoch protection - epoch.Resume(); + /// + /// If a is registered and this enqueue crossed into a + /// new page, drive a from the producer side. This keeps + /// the callback progressing even with no active iterators. Cost on the hot path is a cheap + /// (unsynchronized) long read + branch; the scan only runs once per page transition, driven by + /// exactly one producer (the CAS winner). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void MaybeProducerDriveSafeTail() + { + if (SafeTailPageShiftCallback == null) return; + long tail = allocator.GetTailAddress(); + long newPage = tail >> allocator.LogPageSizeBits; + // Non-volatile read — stale values only cause a redundant CAS attempt, never missed progress + // (some subsequent producer will observe the shift and take the slow path). + if (newPage <= lastProducerObservedPage) return; + ProducerDriveSafeTailSlow(newPage); + } - // Capture the tail address before epoch refresh, so that the bump action - // knows what the new SafeTailAddress should be set to. - safeTailRefreshLastTailAddress = TailAddress; + [MethodImpl(MethodImplOptions.NoInlining)] + void ProducerDriveSafeTailSlow(long newPage) + { + long prev = Volatile.Read(ref lastProducerObservedPage); + if (newPage <= prev) return; + if (Interlocked.CompareExchange(ref lastProducerObservedPage, newPage, prev) != prev) return; + _ = RefreshSafeTailAddress(); + } - // Break out of inner loop if there is no more work to do - if (safeTailRefreshLastTailAddress <= SafeTailAddress) - break; + /// + /// Wake any iterators parked in or as single-iterators. + /// Fast path (no waiters): two null-check loads. When waiters exist, defers to the slow path. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void NotifyParkedWaiters() + { + // Fast path: both references are null when no iterators/waiters exist (NoCons scenario). + // The JIT can inline these two loads and the branch without hitting IL size limits. + if (refreshUncommittedTcs != null || activeSingleIterators != null) + NotifyParkedWaitersSlow(); + } - // Bump epoch with an action to update SafeTailAddress to the captured safeTailRefreshLastTailAddress - epoch.BumpCurrentEpoch(periodicRefreshSafeTailAddressBumpCallbackAction); - } - finally - { - // Suspend epoch protection - epoch.Suspend(); - } - // Wait for the bump epoch action to finish executing, so we can re-check - await safeTailRefreshCallbackCompleted.WaitAsync().ConfigureAwait(false); - } - // Work is done, wait for the next iteration of the worker loop - if (SafeTailRefreshFrequencyMs > 0) - { - await Task.Delay(SafeTailRefreshFrequencyMs, token).ConfigureAwait(false); - } - else - { - await safeTailRefreshEntryEnqueued.WaitAsync().ConfigureAwait(false); - } - } - } - catch (TaskCanceledException) when (safeTailRefreshTaskCts.Token.IsCancellationRequested) + [MethodImpl(MethodImplOptions.NoInlining)] + void NotifyParkedWaitersSlow() + { + var tcs = refreshUncommittedTcs; + var asi = activeSingleIterators; + + // When multiple iterators exist, refresh before signaling so they all see + // the fresh cache and skip their own RefreshSafeTailAddress scan. + // The count is a stale-tolerant hint: if we read an old value of 1 when it's actually 2, + // the extra iterator simply does its own scan (correct, just redundant). If we read 2 when + // it's actually 1, we do one extra scan (harmless). + if (asi != null && Volatile.Read(ref activeSingleIteratorCount) > 1) + _ = RefreshSafeTailAddress(); + + if (tcs != null && Interlocked.CompareExchange(ref refreshUncommittedTcs, null, tcs) == tcs) + tcs.TrySetResult(Empty.Default); + if (asi != null) { - // Suppress the exception if the task was cancelled due to TsavoriteLog disposal or refresh task cancellation + foreach (var iter in asi) + iter.Signal(); } - catch (Exception e) + } + + /// + /// Recompute from the current in-flight state, advance the + /// monotonic cache, and invoke if the new SafeTail + /// crossed a page boundary. Also notifies any parked iterators of the new value. Consumers + /// needing up-to-the-moment progress call this; iterator hot loops can read + /// directly (O(1) cached read). + /// + public long RefreshSafeTailAddress() + { + // Fast path: if TailAddress hasn't moved beyond the cached SafeTailAddress, no new + // records have been allocated and scanning the inflight column cannot yield a higher + // value. Skip the expensive epoch-table scan entirely. + long tail = allocator.GetTailAddress(); + long cached = Volatile.Read(ref cachedSafeTailAddress); + if (tail <= cached) + return cached; + + // Ordering is critical: read the tail *before* the inflight column, with a full fence in + // between. Producers publish their inflight slot via a release store and then advance the + // tail via an interlocked FAA. If we read inflight first and tail second, a reader could + // observe a fresh tail value (post-FAA) while still seeing the producer's slot as + // InflightInactive (pre-BeginInflightEnqueue), incorrectly concluding that the entire + // range up to the new tail is safe even though the producer has not written its payload. + // Reading tail first + memory barrier guarantees that if we observed the FAA we will + // also observe the preceding BeginInflightEnqueue store. + Interlocked.MemoryBarrier(); + long minInflight = epoch.GetMinUserWord(inflightWord); + long computed = minInflight < tail ? minInflight : tail; + + long oldSafe; + if (Utility.MonotonicUpdate(ref cachedSafeTailAddress, computed, out oldSafe)) { - logger?.LogError(e, "Exception encountered during PeriodicSafeTailRefreshRunner"); + NotifyParkedWaiters(); + MaybeInvokePageShiftCallback(oldSafe, computed); + return computed; } + return oldSafe; } - void PeriodicRefreshSafeTailAddressBumpCallback() + /// + /// Monotonically advance the cached to at least + /// . Used by commit/recovery/reset paths to publish a known-safe address + /// (e.g., committed-until, recovered-until) without scanning in-flight slots. + /// + void AdvanceSafeTailFloor(long floor) { + if (Utility.MonotonicUpdate(ref cachedSafeTailAddress, floor, out var oldSafe)) + { + NotifyParkedWaiters(); + MaybeInvokePageShiftCallback(oldSafe, floor); + } + } + + /// + /// Fires only when the new SafeTail is on a different + /// page than the last call. Uses as a monotonic filter so + /// that concurrent callers cannot double-fire for the same page transition. The callback is + /// always invoked outside epoch protection so it can safely re-enter Tsavorite APIs (matching + /// the contract of the previous background-worker design). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + void MaybeInvokePageShiftCallback(long oldSafe, long newSafe) + { + var cb = SafeTailPageShiftCallback; + if (cb == null) return; + long newPage = newSafe >> allocator.LogPageSizeBits; + long prev = Volatile.Read(ref lastPublishedSafeTailPage); + if (newPage <= prev) return; + if (Interlocked.CompareExchange(ref lastPublishedSafeTailPage, newPage, prev) != prev) return; + + // Invoke callback outside epoch protection. Producer drive and direct RefreshSafeTailAddress + // callers may hold the epoch; suspend it so the callback can re-enter log APIs without + // tripping nested-epoch asserts or corrupting epoch bookkeeping. + // Exceptions are caught and logged — the callback is best-effort (e.g., AOF truncation) + // and must not propagate into EndInflightEnqueue / producer cleanup paths. + var isProtected = epoch.ThisInstanceProtected(); + if (isProtected) epoch.Suspend(); try { - if (Utility.MonotonicUpdate(ref SafeTailAddress, safeTailRefreshLastTailAddress, out long oldSafeTailAddress)) - { - var tcs = refreshUncommittedTcs; - if (tcs != null && Interlocked.CompareExchange(ref refreshUncommittedTcs, null, tcs) == tcs) - tcs.SetResult(Empty.Default); - var _callback = SafeTailShiftCallback; - if (_callback != null || activeSingleIterators != null) - { - // We invoke callback outside epoch protection - bool isProtected = epoch.ThisInstanceProtected(); - if (isProtected) epoch.Suspend(); - try - { - // Notify waiting single iterators, if any - var _asi = activeSingleIterators; - if (_asi != null) - { - foreach (var iter in _asi) - { - iter.Signal(); - } - } - // Invoke callback, if any - _callback?.Invoke(oldSafeTailAddress, safeTailRefreshLastTailAddress); - } - finally - { - if (isProtected) epoch.Resume(); - } - } - } + cb(oldSafe, newSafe); + } + catch (Exception e) + { + logger?.LogError(e, "SafeTailPageShiftCallback failed"); } finally { - safeTailRefreshCallbackCompleted.Signal(); + if (isProtected) epoch.Resume(); } } + #endregion /// /// Reset TsavoriteLog to empty state @@ -374,11 +470,17 @@ void PeriodicRefreshSafeTailAddressBumpCallback() /// public void Reset() { - var beginAddress = allocator.GetFirstValidLogicalAddress(0); + var beginAddress = allocator.GetFirstValidLogicalAddressOnPage(0); allocator.Reset(); CommittedUntilAddress = beginAddress; CommittedBeginAddress = beginAddress; - SafeTailAddress = beginAddress; + cachedSafeTailAddress = beginAddress; + + // Reset monotonic page trackers to the new (lower) address so that the first post-reset + // enqueue that crosses into a new page re-arms both producer-drive and callback dispatch. + var resetPage = beginAddress >> allocator.LogPageSizeBits; + Volatile.Write(ref lastPublishedSafeTailPage, resetPage); + Volatile.Write(ref lastProducerObservedPage, resetPage); commitNum = 0; this.beginAddress = beginAddress; @@ -406,7 +508,7 @@ public void SafeInitialize(long beginAddress, long committedUntilAddress, long l // Wait for initialization to complete while (Initializing) - Thread.Yield(); + _ = Thread.Yield(); } /// @@ -422,7 +524,7 @@ public void Initialize(long beginAddress, long committedUntilAddress, long lastC try { if (beginAddress == 0) - beginAddress = allocator.GetFirstValidLogicalAddress(0); + beginAddress = allocator.GetFirstValidLogicalAddressOnPage(0); if (committedUntilAddress == 0) committedUntilAddress = beginAddress; @@ -439,12 +541,21 @@ public void Initialize(long beginAddress, long committedUntilAddress, long lastC CommittedUntilAddress = committedUntilAddress; CommittedBeginAddress = beginAddress; - SafeTailAddress = committedUntilAddress; + + // Align monotonic page trackers to the restored address so that post-recovery producer + // drive and page-shift callbacks re-arm correctly (they only advance beyond the + // initial floor). + var resetPage = committedUntilAddress >> allocator.LogPageSizeBits; + Volatile.Write(ref lastPublishedSafeTailPage, resetPage); + Volatile.Write(ref lastProducerObservedPage, resetPage); + + AdvanceSafeTailFloor(committedUntilAddress); commitNum = lastCommitNum; this.beginAddress = beginAddress; - if (lastCommitNum > 0) logCommitManager.OnRecovery(lastCommitNum); + if (lastCommitNum > 0) + logCommitManager.OnRecovery(lastCommitNum); } finally { @@ -493,14 +604,14 @@ public void Dispose() } /// - /// Mark the log as complete. A completed log will no longer allow enqueues, and all currently enqueued items will + /// Mark the log as complete. A completed log log will no longer allow enqueues, and all currently enqueued items will /// be immediately committed. /// /// whether to spin until log completion becomes committed public void CompleteLog(bool spinWait = false) { // Ensure progress even if there is no thread in epoch table. Also, BumpCurrentEpoch must be done on a protected thread. - bool isProtected = epoch.ThisInstanceProtected(); + var isProtected = epoch.ThisInstanceProtected(); if (!isProtected) epoch.Resume(); try @@ -508,7 +619,7 @@ public void CompleteLog(bool spinWait = false) // Ensure all currently started entries will enqueue before we declare log closed epoch.BumpCurrentEpoch(() => { - CommitInternal(out _, out _, false, [], long.MaxValue, null); + _ = CommitInternal(out _, out _, false, [], long.MaxValue, null); }); } finally @@ -522,18 +633,18 @@ public void CompleteLog(bool spinWait = false) } /// - /// Check if the log is complete. A completed log will no longer allow enqueues, and all currently enqueued items will + /// Check if the log is complete. A completed log log will no longer allow enqueues, and all currently enqueued items will /// be immediately committed. /// public bool LogCompleted => commitNum == long.MaxValue; internal void TrueDispose() { - safeTailRefreshTaskCts?.Cancel(); - safeTailRefreshCallbackCompleted?.Signal(); - safeTailRefreshEntryEnqueued?.Signal(); + // Release our in-flight user-word slot back to the epoch. Iterators no longer parked; the + // slot column is no longer referenced. + epoch.ReleaseUserWord(inflightWord); commitQueue.Dispose(); - commitTcs.TrySetException(new ObjectDisposedException("Log has been disposed")); + _ = commitTcs.TrySetException(new ObjectDisposedException("TsavoriteLog has been disposed")); allocator.Dispose(); if (isEpochOwned) epoch.Dispose(); @@ -551,7 +662,7 @@ public long Enqueue(byte[] entry) { long logicalAddress; while (!TryEnqueue(entry, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } @@ -564,7 +675,7 @@ public long Enqueue(ReadOnlySpan entry) { long logicalAddress; while (!TryEnqueue(entry, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } @@ -578,7 +689,7 @@ public long UnsafeEnqueueRaw(ReadOnlySpan entryBytes, bool noCommit = fals { long logicalAddress; while (!UnsafeTryEnqueueRaw(entryBytes, noCommit, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } @@ -596,13 +707,15 @@ public void UnsafeCommitMetadataOnly(TsavoriteLogRecoveryInfo info, bool isProte } try { - if (!isProtected) epoch.Resume(); + if (!isProtected) + epoch.Resume(); if (!allocator.ShiftReadOnlyToTail(out _, out _)) CommitMetadataOnly(ref info); } finally { - if (!isProtected) epoch.Suspend(); + if (!isProtected) + epoch.Suspend(); } } @@ -610,14 +723,18 @@ public void UnsafeCommitMetadataOnly(TsavoriteLogRecoveryInfo info, bool isProte /// Get page size in bits /// /// - public int UnsafeGetLogPageSizeBits() - => allocator.LogPageSizeBits; + public int UnsafeGetLogPageSizeBits() => allocator.LogPageSizeBits; /// /// Get read only lag address /// - public long UnsafeGetReadOnlyAddressLagOffset() - => allocator.GetReadOnlyAddressLagOffset(); + public long UnsafeGetReadOnlyAddressAbove(long newTailAddress, int numPagesAbove) + { + var readOnlyAddress = allocator.CalculateReadOnlyAddress(newTailAddress, allocator.HeadAddress) + numPagesAbove * allocator.PageSize; + if (readOnlyAddress > newTailAddress) + readOnlyAddress = newTailAddress; + return readOnlyAddress; + } /// /// Enqueue batch of entries to log (in memory) - no guarantee of flush/commit @@ -628,7 +745,7 @@ public long Enqueue(IReadOnlySpanBatch readOnlySpanBatch) { long logicalAddress; while (!TryEnqueue(readOnlySpanBatch, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } @@ -642,7 +759,7 @@ public long Enqueue(T entry) where T : ILogEnqueueEntry { long logicalAddress; while (!TryEnqueue(entry, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } @@ -656,7 +773,7 @@ public long Enqueue(IEnumerable entries) where T : ILogEnqueueEntry { long logicalAddress; while (!TryEnqueue(entries, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); return logicalAddress; } #endregion @@ -674,28 +791,32 @@ public unsafe bool TryEnqueue(T entry, out long logicalAddress) where T : ILo { logicalAddress = 0; var length = entry.SerializedLength; - int allocatedLength = headerSize + Align(length); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) - if (logicalAddress == 0) + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) { - epoch.Suspend(); - if (cannedException != null) throw cannedException; + if (cannedException != null) + throw cannedException; return false; } - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - entry.SerializeTo(new Span((void*)(headerSize + physicalAddress), length)); - SetHeader(length, (byte*)physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + entry.SerializeTo(new Span((void*)(headerSize + physicalAddress), length)); + SetHeader(length, (byte*)physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); return true; } @@ -720,28 +841,33 @@ public unsafe bool TryEnqueue(IEnumerable entries, out long logicalAddress ValidateAllocatedLength(allocatedLength); epoch.Resume(); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - if (logicalAddress == 0) - { - epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + foreach (var entry in entries) + { + var length = entry.SerializedLength; + entry.SerializeTo(new Span((void*)(headerSize + physicalAddress), length)); + SetHeader(length, (byte*)physicalAddress); + physicalAddress += Align(length) + headerSize; + } } - - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - foreach (var entry in entries) + finally { - var length = entry.SerializedLength; - entry.SerializeTo(new Span((void*)(headerSize + physicalAddress), length)); - SetHeader(length, (byte*)physicalAddress); - physicalAddress += Align(length) + headerSize; + EndInflightEnqueue(); + epoch.Suspend(); } - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } @@ -756,29 +882,34 @@ public unsafe bool TryEnqueue(byte[] entry, out long logicalAddress) { logicalAddress = 0; var length = entry.Length; - int allocatedLength = headerSize + Align(length); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) + throw new TsavoriteException("Attempting to enqueue into a completed log"); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) - if (logicalAddress == 0) + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) { - epoch.Suspend(); - if (cannedException != null) throw cannedException; + if (cannedException != null) + throw cannedException; return false; } - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - fixed (byte* bp = entry) - Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), length, length); - SetHeader(length, (byte*)physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + fixed (byte* bp = entry) + Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), length, length); + SetHeader(length, (byte*)physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); return true; } @@ -792,31 +923,36 @@ public unsafe bool TryEnqueue(byte[] entry, out long logicalAddress) /// Whether the append succeeded public unsafe bool UnsafeTryEnqueueRaw(ReadOnlySpan entryBytes, bool noCommit, out long logicalAddress) { - int length = entryBytes.Length; + var length = entryBytes.Length; // Length should be pre-aligned Debug.Assert(length == Align(length)); logicalAddress = 0; - int allocatedLength = length; + var allocatedLength = length; ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + entryBytes.CopyTo(new Span((byte*)physicalAddress, length)); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; } - - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - entryBytes.CopyTo(new Span((byte*)physicalAddress, length)); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit && !noCommit) Commit(); + if (autoCommit && !noCommit) Commit(); return true; } @@ -831,28 +967,33 @@ public unsafe bool TryEnqueue(ReadOnlySpan entry, out long logicalAddress) { logicalAddress = 0; var length = entry.Length; - int allocatedLength = headerSize + Align(length); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + fixed (byte* bp = &entry.GetPinnableReference()) + Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), length, length); + SetHeader(length, (byte*)physicalAddress); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; } - - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - fixed (byte* bp = &entry.GetPinnableReference()) - Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), length, length); - SetHeader(length, (byte*)physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } @@ -866,142 +1007,170 @@ public unsafe void Enqueue(THeader userHeader, out long logicalAddress) { logicalAddress = 0; var length = sizeof(THeader); - int allocatedLength = headerSize + Align(length); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); - - logicalAddress = AllocateBlock(allocatedLength); - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); } /// - /// Append a user-defined blittable struct header and one entry atomically to the log. + /// Append a user-defined blittable struct header and one entry atomically to the log. /// /// /// /// Logical address of added entry - public unsafe void Enqueue(THeader userHeader, ref SpanByte item, out long logicalAddress) + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item, out long logicalAddress) where THeader : unmanaged { logicalAddress = 0; - var length = sizeof(THeader) + item.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(THeader) + item.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); - - logicalAddress = AllocateBlock(allocatedLength); - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - item.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); } /// - /// Append a user-defined blittable struct header and two entries entries atomically to the log. + /// Append a user-defined blittable struct header and two entries entries atomically to the log. /// /// /// /// /// Logical address of added entry - public unsafe void Enqueue(THeader userHeader, ref SpanByte item1, ref SpanByte item2, TEpochAccessor epochAccessor, out long logicalAddress) + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item1, ReadOnlySpan item2, TEpochAccessor epochAccessor, out long logicalAddress) where THeader : unmanaged where TEpochAccessor : IEpochAccessor { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + item2.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); try { logicalAddress = AllocateBlock(allocatedLength, epochAccessor); - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - item2.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize); + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); } finally { + EndInflightEnqueue(); epoch.Suspend(); } - if (AutoCommit) Commit(); + if (autoCommit) + Commit(); } /// - /// Append two entries entries atomically to the log. + /// Append two entries entries atomically to the log. /// /// /// /// Logical address of added entry - public unsafe void Enqueue(ref SpanByte item1, ref SpanByte item2, out long logicalAddress) + public unsafe void Enqueue(ReadOnlySpan item1, ReadOnlySpan item2, out long logicalAddress) { - logicalAddress = 0; - var length = item1.TotalSize + item2.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = item1.TotalSize() + item2.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); - - logicalAddress = AllocateBlock(allocatedLength); - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - item1.CopyTo(physicalAddress + headerSize); - item2.CopyTo(physicalAddress + headerSize + item1.TotalSize); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + var offset = headerSize; + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); } /// - /// Append a user-defined blittable struct header and three entries entries atomically to the log. + /// Append a user-defined blittable struct header and three entries entries atomically to the log. /// /// /// /// /// /// Logical address of added entry - public unsafe void Enqueue(THeader userHeader, ref SpanByte item1, ref SpanByte item2, ref SpanByte item3, out long logicalAddress) + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item1, ReadOnlySpan item2, ReadOnlySpan item3, out long logicalAddress) where THeader : unmanaged { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + item2.TotalSize + item3.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize() + item3.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); - - logicalAddress = AllocateBlock(allocatedLength); - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - item2.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize); - item3.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize + item2.TotalSize); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item2.TotalSize(); + item3.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); } /// - /// Append a user-defined blittable struct header and three entries entries atomically to the log. + /// Append a user-defined blittable struct header and three entries entries atomically to the log. /// /// /// @@ -1015,112 +1184,161 @@ public unsafe void Enqueue(THeader userHeader, ref TInput input ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + _ = input.CopyTo(physicalAddress + headerSize + sizeof(THeader), input.SerializedLength); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); + } - logicalAddress = AllocateBlock(allocatedLength); - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - input.CopyTo(physicalAddress + headerSize + sizeof(THeader), input.SerializedLength); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + /// + /// Append a user-defined blittable struct header, one entry, and one atomically to the log. + /// + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item1, ref TInput input, out long logicalAddress) + where THeader : unmanaged where TInput : IStoreInput + { + logicalAddress = 0; + var length = sizeof(THeader) + item1.TotalSize() + input.SerializedLength; + var allocatedLength = headerSize + Align(length); + ValidateAllocatedLength(allocatedLength); + + epoch.Resume(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + _ = input.CopyTo(physicalAddress + offset, input.SerializedLength); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) + Commit(); } /// - /// Append a user-defined blittable struct header and three entries entries atomically to the log. + /// Append a user-defined blittable struct header, one entry, and one atomically to the log. /// - /// - /// - /// - /// Logical address of added entry - public unsafe void Enqueue(THeader userHeader, ref SpanByte item1, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item1, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) where THeader : unmanaged where TInput : IStoreInput where TEpochAccessor : IEpochAccessor { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + input.SerializedLength; + var length = sizeof(THeader) + item1.TotalSize() + input.SerializedLength; var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); try { logicalAddress = AllocateBlock(allocatedLength, epochAccessor); var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - input.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize, input.SerializedLength); + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + _ = input.CopyTo(physicalAddress + offset, input.SerializedLength); SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); } finally { + EndInflightEnqueue(); epoch.Suspend(); } - if (AutoCommit) Commit(); + if (autoCommit) + Commit(); } /// - /// Append a user-defined blittable struct header and three entries entries atomically to the log. + /// Append a user-defined blittable struct header and three entries entries atomically to the log. /// /// /// /// /// /// Logical address of added entry - public unsafe void Enqueue(THeader userHeader, ref SpanByte item1, ref SpanByte item2, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) + public unsafe void Enqueue(THeader userHeader, ReadOnlySpan item1, ReadOnlySpan item2, ref TInput input, TEpochAccessor epochAccessor, out long logicalAddress) where THeader : unmanaged where TInput : IStoreInput where TEpochAccessor : IEpochAccessor { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + item2.TotalSize + input.SerializedLength; + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize() + input.SerializedLength; var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); try { logicalAddress = AllocateBlock(allocatedLength, epochAccessor); var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - item2.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize); - input.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize + item2.TotalSize, - input.SerializedLength); + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item2.TotalSize(); + _ = input.CopyTo(physicalAddress + offset, input.SerializedLength); SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); } finally { + EndInflightEnqueue(); epoch.Suspend(); } - if (AutoCommit) Commit(); + if (autoCommit) + Commit(); } /// - /// Append a user-defined header byte and a entry atomically to the log. + /// Append a user-defined header byte and a entry atomically to the log. /// /// /// /// Logical address of added entry - public unsafe void Enqueue(byte userHeader, ref SpanByte item, out long logicalAddress) + public unsafe void Enqueue(byte userHeader, ReadOnlySpan item, out long logicalAddress) { logicalAddress = 0; - var length = sizeof(byte) + item.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(byte) + item.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); - - logicalAddress = AllocateBlock(allocatedLength); - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *physicalAddress = userHeader; - item.CopyTo(physicalAddress + sizeof(byte)); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + BeginInflightEnqueue(); + try + { + logicalAddress = AllocateBlock(allocatedLength); + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *physicalAddress = userHeader; + var offset = sizeof(byte); + item.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally + { + EndInflightEnqueue(); + epoch.Suspend(); + } + if (autoCommit) Commit(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1128,23 +1346,26 @@ private long AllocateBlock(int recordSize) { while (true) { - var flushEvent = allocator.FlushEvent; - var logicalAddress = allocator.TryAllocateRetryNow(recordSize); - if (logicalAddress > 0) + var flushEvent = allocator.flushEvent; + if (allocator.TryAllocateRetryNow(recordSize, out var logicalAddress)) return logicalAddress; - // logicalAddress less than 0 (RETRY_NOW) should already have been handled + // logicalAddress less than 0 (RETRY_NOW) should already have been handled. We expect flushEvent to be signaled. Debug.Assert(logicalAddress == 0); + // Clear in-flight slot before suspending, re-publish after resuming + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; try { + if (cannedException != null) + throw cannedException; flushEvent.Wait(); } finally { epoch.Resume(); + BeginInflightEnqueue(); } } } @@ -1155,25 +1376,30 @@ private long AllocateBlock(int recordSize, TEpochAccessor epochA { while (true) { - var flushEvent = allocator.FlushEvent; - var logicalAddress = allocator.TryAllocateRetryNow(recordSize); + var flushEvent = allocator.flushEvent; + allocator.TryAllocateRetryNow(recordSize, out var logicalAddress); if (logicalAddress > 0) return logicalAddress; // logicalAddress less than 0 (RETRY_NOW) should already have been handled Debug.Assert(logicalAddress == 0); + // Clear in-flight slot before suspending, re-publish after resuming + EndInflightEnqueue(); epoch.Suspend(); var suspended = epochAccessor.TrySuspend(); try { - if (cannedException != null) ThrowException(cannedException); + if (cannedException != null) + ThrowException(cannedException); flushEvent.Wait(); } finally { - if (suspended) epochAccessor.Resume(); + if (suspended) + epochAccessor.Resume(); epoch.Resume(); + BeginInflightEnqueue(); } } } @@ -1182,7 +1408,7 @@ private long AllocateBlock(int recordSize, TEpochAccessor epochA static void ThrowException(Exception e) => throw e; /// - /// Try to append a user-defined blittable struct header and two entries entries atomically to the log. + /// Try to append a user-defined blittable struct header and two entries entries atomically to the log. /// If it returns true, we are done. If it returns false, we need to retry. /// /// @@ -1190,37 +1416,44 @@ private long AllocateBlock(int recordSize, TEpochAccessor epochA /// /// Logical address of added entry /// Whether the append succeeded - public unsafe bool TryEnqueue(THeader userHeader, ref SpanByte item1, ref SpanByte item2, out long logicalAddress) + public unsafe bool TryEnqueue(THeader userHeader, ReadOnlySpan item1, ReadOnlySpan item2, out long logicalAddress) where THeader : unmanaged { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + item2.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; } - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - item2.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } /// - /// Try to append a user-defined blittable struct header and three entries entries atomically to the log. + /// Try to append a user-defined blittable struct header and three entries entries atomically to the log. /// If it returns true, we are done. If it returns false, we need to retry. /// /// @@ -1229,68 +1462,82 @@ public unsafe bool TryEnqueue(THeader userHeader, ref SpanByte item1, r /// /// Logical address of added entry /// Whether the append succeeded - public unsafe bool TryEnqueue(THeader userHeader, ref SpanByte item1, ref SpanByte item2, ref SpanByte item3, out long logicalAddress) + public unsafe bool TryEnqueue(THeader userHeader, ReadOnlySpan item1, ReadOnlySpan item2, ReadOnlySpan item3, out long logicalAddress) where THeader : unmanaged { logicalAddress = 0; - var length = sizeof(THeader) + item1.TotalSize + item2.TotalSize + item3.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(THeader) + item1.TotalSize() + item2.TotalSize() + item3.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *(THeader*)(physicalAddress + headerSize) = userHeader; + var offset = headerSize + sizeof(THeader); + item1.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item1.TotalSize(); + item2.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + offset += item2.TotalSize(); + item3.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; } - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *(THeader*)(physicalAddress + headerSize) = userHeader; - item1.CopyTo(physicalAddress + headerSize + sizeof(THeader)); - item2.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize); - item3.CopyTo(physicalAddress + headerSize + sizeof(THeader) + item1.TotalSize + item2.TotalSize); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } /// - /// Try to append a user-defined header byte and a entry atomically to the log. If it returns true, we are + /// Try to append a user-defined header byte and a entry atomically to the log. If it returns true, we are /// done. If it returns false, we need to retry. /// /// /// /// Logical address of added entry /// Whether the append succeeded - public unsafe bool TryEnqueue(byte userHeader, ref SpanByte item, out long logicalAddress) + public unsafe bool TryEnqueue(byte userHeader, ReadOnlySpan item, out long logicalAddress) { logicalAddress = 0; - var length = sizeof(byte) + item.TotalSize; - int allocatedLength = headerSize + Align(length); + var length = sizeof(byte) + item.TotalSize(); + var allocatedLength = headerSize + Align(length); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); + *physicalAddress = userHeader; + var offset = sizeof(byte); + item.SerializeTo(new Span(physicalAddress + offset, allocatedLength - offset)); + SetHeader(length, physicalAddress); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; } - - var physicalAddress = (byte*)allocator.GetPhysicalAddress(logicalAddress); - *physicalAddress = userHeader; - item.CopyTo(physicalAddress + sizeof(byte)); - SetHeader(length, physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } @@ -1318,7 +1565,7 @@ public bool TryEnqueue(IReadOnlySpanBatch readOnlySpanBatch, out long logicalAdd public ValueTask EnqueueAsync(byte[] entry, CancellationToken token = default) { token.ThrowIfCancellationRequested(); - if (TryEnqueue(entry, out long logicalAddress)) + if (TryEnqueue(entry, out var logicalAddress)) return new ValueTask(logicalAddress); return SlowEnqueueAsync(this, entry, token); @@ -1353,7 +1600,7 @@ private static async ValueTask SlowEnqueueAsync(TsavoriteLog @this, byte[] public ValueTask EnqueueAsync(ReadOnlyMemory entry, CancellationToken token = default) { token.ThrowIfCancellationRequested(); - if (TryEnqueue(entry.Span, out long logicalAddress)) + if (TryEnqueue(entry.Span, out var logicalAddress)) return new ValueTask(logicalAddress); return SlowEnqueueAsync(this, entry, token); @@ -1388,7 +1635,7 @@ private static async ValueTask SlowEnqueueAsync(TsavoriteLog @this, ReadOn public ValueTask EnqueueAsync(IReadOnlySpanBatch readOnlySpanBatch, CancellationToken token = default) { token.ThrowIfCancellationRequested(); - if (TryEnqueue(readOnlySpanBatch, out long address)) + if (TryEnqueue(readOnlySpanBatch, out var address)) return new ValueTask(address); return SlowEnqueueAsync(this, readOnlySpanBatch, token); @@ -1424,7 +1671,7 @@ private static async ValueTask SlowEnqueueAsync(TsavoriteLog @this, IReadO public ValueTask EnqueueAsync(T entry, CancellationToken token = default) where T : ILogEnqueueEntry { token.ThrowIfCancellationRequested(); - if (TryEnqueue(entry, out long logicalAddress)) + if (TryEnqueue(entry, out var logicalAddress)) return new ValueTask(logicalAddress); return SlowEnqueueAsync(this, entry, token); @@ -1461,7 +1708,7 @@ private static async ValueTask SlowEnqueueAsync(TsavoriteLog @this, T e public ValueTask EnqueueAsync(IEnumerable entries, CancellationToken token = default) where T : ILogEnqueueEntry { token.ThrowIfCancellationRequested(); - if (TryEnqueue(entries, out long logicalAddress)) + if (TryEnqueue(entries, out var logicalAddress)) return new ValueTask(logicalAddress); return SlowEnqueueAsync(this, entries, token); @@ -1506,7 +1753,7 @@ public void WaitForCommit(long untilAddress = 0, long commitNum = -1) while (commitNum > persistedCommitNum || untilAddress > CommittedUntilAddress) { if (cannedException != null) throw cannedException; - Thread.Yield(); + _ = Thread.Yield(); } } @@ -1540,10 +1787,15 @@ public async ValueTask WaitForCommitAsync(long untilAddress = 0, long commitNum /// true if there's more data available to be read; false if there will never be more data (log has been shutdown) public async ValueTask WaitUncommittedAsync(long nextAddress, CancellationToken token = default) { - Debug.Assert(SafeTailRefreshFrequencyMs >= 0); + // Fast path — cache already past nextAddress. if (nextAddress < SafeTailAddress) return true; + // Refresh once in case in-flight enqueues have already completed but haven't triggered a + // recompute yet (e.g., single-producer, no other reader has forced RefreshSafeTailAddress). + if (nextAddress < RefreshSafeTailAddress()) + return true; + while (true) { token.ThrowIfCancellationRequested(); @@ -1558,13 +1810,13 @@ public async ValueTask WaitUncommittedAsync(long nextAddress, Cancellation tcs ??= newTcs; // successful CAS so update the local var } - if (nextAddress < SafeTailAddress) + if (nextAddress < SafeTailAddress || nextAddress < RefreshSafeTailAddress()) return true; // Ignore refresh-uncommitted exceptions, except when the token is signaled try { - await tcs.Task.WithCancellationAsync(token).ConfigureAwait(false); + _ = await tcs.Task.WithCancellationAsync(token).ConfigureAwait(false); } catch (ObjectDisposedException) { return false; } catch when (!token.IsCancellationRequested) { } @@ -1578,8 +1830,8 @@ public async ValueTask WaitUncommittedAsync(long nextAddress, Cancellation /// Issue commit request for log (until tail) /// /// If true, spin-wait until commit completes. Otherwise, issue commit and return immediately. + /// /// whether there is anything to commit. - public void Commit(bool spinWait = false, byte[] cookie = null) { // Take a lower-bound of the content of this commit in case our request is filtered but we need to spin @@ -1629,6 +1881,8 @@ public bool CommitStrongly(out long commitTail, out long actualCommitNum, bool s /// complete the commit. Throws exception if this or any /// ongoing commit fails. /// + /// + /// /// public async ValueTask CommitAsync(byte[] cookie = null, CancellationToken token = default) { @@ -1672,12 +1926,9 @@ public async ValueTask> CommitAsync(Task entry) { long logicalAddress; while (!TryEnqueue(entry, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); WaitForCommit(logicalAddress + 1); return logicalAddress; } @@ -1777,7 +2028,7 @@ public long EnqueueAndWaitForCommit(IReadOnlySpanBatch readOnlySpanBatch) { long logicalAddress; while (!TryEnqueue(readOnlySpanBatch, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); WaitForCommit(logicalAddress + 1); return logicalAddress; } @@ -1793,7 +2044,7 @@ public long EnqueueAndWaitForCommit(T entry) where T : ILogEnqueueEntry { long logicalAddress; while (!TryEnqueue(entry, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); WaitForCommit(logicalAddress + 1); return logicalAddress; } @@ -1809,7 +2060,7 @@ public long EnqueueAndWaitForCommit(IEnumerable entries) where T : ILogEnq { long logicalAddress; while (!TryEnqueue(entries, out logicalAddress)) - Thread.Yield(); + _ = Thread.Yield(); WaitForCommit(logicalAddress + 1); return logicalAddress; } @@ -2090,7 +2341,7 @@ public async ValueTask EnqueueAndWaitForCommitAsync(IEnumerable entr /// Until address public void TruncateUntil(long untilAddress) { - Utility.MonotonicUpdate(ref beginAddress, untilAddress, out _); + _ = Utility.MonotonicUpdate(ref beginAddress, untilAddress, out _); } /// @@ -2105,9 +2356,9 @@ public void UnsafeShiftBeginAddress(long untilAddress, bool snapToPageStart = fa if (Utility.MonotonicUpdate(ref beginAddress, untilAddress, out _)) { if (snapToPageStart) - untilAddress &= ~allocator.PageSizeMask; + untilAddress = allocator.GetAddressOfStartOfPageOfAddress(untilAddress); - bool epochProtected = epoch.ThisInstanceProtected(); + var epochProtected = epoch.ThisInstanceProtected(); try { if (!epochProtected) @@ -2130,7 +2381,7 @@ public void UnsafeShiftBeginAddress(long untilAddress, bool snapToPageStart = fa /// Until address public void TruncateUntilPageStart(long untilAddress) { - Utility.MonotonicUpdate(ref beginAddress, untilAddress & ~allocator.PageSizeMask, out _); + _ = Utility.MonotonicUpdate(ref beginAddress, allocator.GetAddressOfStartOfPageOfAddress(untilAddress), out _); } /// @@ -2143,19 +2394,15 @@ public void TruncateUntilPageStart(long untilAddress) /// Whether we scan uncommitted data /// /// - public TsavoriteLogScanIterator Scan(long beginAddress, long endAddress, bool recover = true, ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) + public TsavoriteLogScanIterator Scan(long beginAddress, long endAddress, bool recover = true, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) { if (readOnlyMode) { - scanBufferingMode = ScanBufferingMode.SinglePageBuffering; - + scanBufferingMode = DiskScanBufferingMode.SinglePageBuffering; if (scanUncommitted) throw new TsavoriteException("Cannot use scanUncommitted with read-only TsavoriteLog"); } - if (scanUncommitted && SafeTailRefreshFrequencyMs < 0) - throw new TsavoriteException("Cannot use scanUncommitted without setting SafeTailRefreshFrequencyMs to a non-negative value in TsavoriteLog settings"); - var iter = new TsavoriteLogScanIterator(this, allocator, beginAddress, endAddress, getMemory, scanBufferingMode, epoch, headerSize, scanUncommitted, logger: logger); if (Interlocked.Increment(ref logRefCount) == 1) @@ -2163,8 +2410,24 @@ public TsavoriteLogScanIterator Scan(long beginAddress, long endAddress, bool re return iter; } + /// + /// Registered single-waiter iterators (one per replica / pub-sub subscriber). Non-null when at + /// least one is active. Replaced atomically under + /// lock(this) on iterator add/remove — read without lock on the hot path + /// (). + /// List activeSingleIterators; + /// + /// Count of registered single iterators, mirroring activeSingleIterators.Count. + /// Read via Volatile.Read by to + /// decide whether to pre-refresh before signaling iterators. + /// When > 1, a single scan at the producer side prevents N redundant scans in the + /// N woken iterators. Stale reads are benign — see comments. + /// Written under lock(this) alongside . + /// + int activeSingleIteratorCount; + public void RemoveIterator(TsavoriteLogScanSingleIterator iterator) { lock (this) @@ -2181,29 +2444,30 @@ public void RemoveIterator(TsavoriteLogScanSingleIterator iterator) } } activeSingleIterators = newList; + // Keep the count in sync; read without lock by NotifyParkedWaiters as a hint. + activeSingleIteratorCount = newList?.Count ?? 0; } } } - public TsavoriteLogScanSingleIterator ScanSingle(long beginAddress, long endAddress, bool recover = true, ScanBufferingMode scanBufferingMode = ScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) + public TsavoriteLogScanSingleIterator ScanSingle(long beginAddress, long endAddress, bool recover = true, DiskScanBufferingMode scanBufferingMode = DiskScanBufferingMode.DoublePageBuffering, bool scanUncommitted = false, ILogger logger = null) { if (readOnlyMode) { - scanBufferingMode = ScanBufferingMode.SinglePageBuffering; + scanBufferingMode = DiskScanBufferingMode.SinglePageBuffering; if (scanUncommitted) throw new TsavoriteException("Cannot use scanUncommitted with read-only TsavoriteLog"); } - if (scanUncommitted && SafeTailRefreshFrequencyMs < 0) - throw new TsavoriteException("Cannot use scanUncommitted without setting SafeTailRefreshFrequencyMs to a non-negative value in TsavoriteLog settings"); - var iter = new TsavoriteLogScanSingleIterator(this, allocator, beginAddress, endAddress, getMemory, scanBufferingMode, epoch, headerSize, scanUncommitted, logger: logger); lock (this) { List newList = activeSingleIterators == null ? new() { iter } : new(activeSingleIterators) { iter }; activeSingleIterators = newList; + // Keep the count in sync; read without lock by NotifyParkedWaiters as a hint. + activeSingleIteratorCount = newList.Count; } if (Interlocked.Increment(ref logRefCount) == 1) @@ -2234,7 +2498,7 @@ public TsavoriteLogScanSingleIterator ScanSingle(long beginAddress, long endAddr }; unsafe { - allocator.AsyncReadRecordToMemory(address, headerSize + estimatedLength, AsyncGetFromDiskCallback, ref ctx); + allocator.AsyncReadBlittableRecordToMemory(address, headerSize + estimatedLength, AsyncGetFromDiskCallback, ref ctx); } epoch.Suspend(); await ctx.completedRead.WaitAsync(token).ConfigureAwait(false); @@ -2265,7 +2529,7 @@ public TsavoriteLogScanSingleIterator ScanSingle(long beginAddress, long endAddr }; unsafe { - allocator.AsyncReadRecordToMemory(address, headerSize + estimatedLength, AsyncGetFromDiskCallback, ref ctx); + allocator.AsyncReadBlittableRecordToMemory(address, headerSize + estimatedLength, AsyncGetFromDiskCallback, ref ctx); } epoch.Suspend(); await ctx.completedRead.WaitAsync(token).ConfigureAwait(false); @@ -2294,21 +2558,13 @@ public async ValueTask ReadRecordLengthAsync(long address, CancellationToke }; unsafe { - allocator.AsyncReadRecordToMemory(address, headerSize, AsyncGetHeaderOnlyFromDiskCallback, ref ctx); + allocator.AsyncReadBlittableRecordToMemory(address, headerSize, AsyncGetHeaderOnlyFromDiskCallback, ref ctx); } epoch.Suspend(); await ctx.completedRead.WaitAsync(token).ConfigureAwait(false); return GetRecordLengthAndFree(ctx.record); } - /// - /// Trigger refresh of safe tail address - /// - private void DoAutoRefreshSafeTailAddress() - { - safeTailRefreshEntryEnqueued?.Signal(); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] private static int Align(int length) { @@ -2329,29 +2585,34 @@ private unsafe bool TryEnqueueCommitRecord(ref TsavoriteLogRecoveryInfo info) { var entryBodySize = info.SerializedSize(); - int allocatedLength = headerSize + Align(entryBodySize); + var allocatedLength = headerSize + Align(entryBodySize); ValidateAllocatedLength(allocatedLength); epoch.Resume(); + BeginInflightEnqueue(); + try + { + if (!allocator.TryAllocateRetryNow(allocatedLength, out var logicalAddress)) + { + return false; + } + + // Finish filling in all fields + info.BeginAddress = BeginAddress; + info.UntilAddress = logicalAddress + allocatedLength; - var logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); - if (logicalAddress == 0) + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + + var entryBody = info.ToByteArray(); + fixed (byte* bp = entryBody) + Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), entryBody.Length, entryBody.Length); + SetCommitRecordHeader(entryBody.Length, (byte*)physicalAddress); + } + finally { + EndInflightEnqueue(); epoch.Suspend(); - return false; } - // Finish filling in all fields - info.BeginAddress = BeginAddress; - info.UntilAddress = logicalAddress + allocatedLength; - - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - - var entryBody = info.ToByteArray(); - fixed (byte* bp = entryBody) - Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), entryBody.Length, entryBody.Length); - SetCommitRecordHeader(entryBody.Length, (byte*)physicalAddress); - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); // Return the commit tail return true; } @@ -2378,7 +2639,7 @@ private void UpdateCommittedState(TsavoriteLogRecoveryInfo recoveryInfo) { CommittedBeginAddress = recoveryInfo.BeginAddress; CommittedUntilAddress = recoveryInfo.UntilAddress; - Utility.MonotonicUpdate(ref persistedCommitNum, recoveryInfo.CommitNum, out _); + _ = Utility.MonotonicUpdate(ref persistedCommitNum, recoveryInfo.CommitNum, out _); } private void WriteCommitMetadata(TsavoriteLogRecoveryInfo recoveryInfo) @@ -2387,7 +2648,7 @@ private void WriteCommitMetadata(TsavoriteLogRecoveryInfo recoveryInfo) // If we are in fast-commit, we may not write every metadata to disk. However, when we are deleting files // on disk, we have to write metadata for the new start location on disk so we know where to scan forward from. - bool forceWriteMetadata = fastCommitMode && (allocator.BeginAddress < recoveryInfo.BeginAddress); + var forceWriteMetadata = fastCommitMode && (allocator.BeginAddress < recoveryInfo.BeginAddress); logCommitManager.Commit(recoveryInfo.BeginAddress, recoveryInfo.UntilAddress, recoveryInfo.ToByteArray(), recoveryInfo.CommitNum, forceWriteMetadata); @@ -2419,16 +2680,16 @@ private void SerialCommitCallbackWorker(CommitInfo commitInfo) { var oldCommitTcs = commitTcs; commitTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - oldCommitTcs.TrySetException(exception); + _ = oldCommitTcs.TrySetException(exception); // Silently set flushed until past this range - Utility.MonotonicUpdate(ref allocator.FlushedUntilAddress, commitInfo.UntilAddress, out _); + _ = Utility.MonotonicUpdate(ref allocator.FlushedUntilAddress, commitInfo.UntilAddress, out _); allocator.UnsafeSkipError(commitInfo); } else { cannedException = exception; // Make sure future waiters do not get a fresh tcs - commitTcs.TrySetException(cannedException); + _ = commitTcs.TrySetException(cannedException); } return; } @@ -2441,7 +2702,7 @@ private void SerialCommitCallbackWorker(CommitInfo commitInfo) var (addr, recoveryInfo) = ongoingCommitRequests.Peek(); if (addr > commitInfo.UntilAddress) break; coveredCommits.Add(recoveryInfo); - ongoingCommitRequests.Dequeue(); + _ = ongoingCommitRequests.Dequeue(); } } @@ -2483,7 +2744,7 @@ private void SerialCommitCallbackWorker(CommitInfo commitInfo) CommitInfo = commitInfo, NextTask = commitTcs.Task }; - _commitTcs?.TrySetResult(lci); + _ = (_commitTcs?.TrySetResult(lci)); } /// @@ -2506,20 +2767,18 @@ public async ValueTask RecoverReadOnlyAsync(CancellationToken cancellationToken if (!readOnlyMode) throw new TsavoriteException("This method can only be used with a read-only TsavoriteLog instance used for iteration. Set TsavoriteLogSettings.ReadOnlyMode to true during creation to indicate this."); - await RestoreLatestAsync(cancellationToken).ConfigureAwait(false); + _ = await RestoreLatestAsync(cancellationToken).ConfigureAwait(false); SignalWaitingROIterators(); } private void SignalWaitingROIterators() { - // One RecoverReadOnly use case is to allow a TsavoriteLogIterator to continuously read a mirror TsavoriteLog (over the same log storage) of a primary TsavoriteLog. + // One RecoverReadOnly use case is to allow a TsavoriteLogScanIterator to continuously read a mirror TsavoriteLog (over the same log storage) of a primary TsavoriteLog. // In this scenario, when the iterator arrives at the tail after a previous call to RestoreReadOnly, it will wait asynchronously until more data // is committed and read by a subsequent call to RecoverReadOnly. Here, we signal iterators that we have completed recovery. var _commitTcs = commitTcs; - if (commitTcs.Task.Status != TaskStatus.Faulted || commitTcs.Task.Exception.InnerException as CommitFailureException != null) - { + if (commitTcs.Task.Status != TaskStatus.Faulted || commitTcs.Task.Exception.InnerException is CommitFailureException) commitTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - } // Update commit to release pending iterators. var lci = new LinkedCommitInfo @@ -2527,7 +2786,7 @@ private void SignalWaitingROIterators() CommitInfo = new CommitInfo { FromAddress = BeginAddress, UntilAddress = FlushedUntilAddress }, NextTask = commitTcs.Task }; - _commitTcs?.TrySetResult(lci); + _ = (_commitTcs?.TrySetResult(lci)); } private bool LoadCommitMetadata(long commitNum, out TsavoriteLogRecoveryInfo info) @@ -2577,7 +2836,7 @@ private void RestoreLatest(out byte[] cookie) try { using var scanIterator = Scan(info.UntilAddress, long.MaxValue, recover: false); - scanIterator.ScanForwardForCommit(ref info); + _ = scanIterator.ScanForwardForCommit(ref info); } catch { } } @@ -2590,7 +2849,7 @@ private void RestoreLatest(out byte[] cookie) // Reset variables to normal allocator.Initialize(); - CommittedUntilAddress = Constants.kFirstValidAddress; + CommittedUntilAddress = FirstValidAddress; beginAddress = allocator.BeginAddress; if (readOnlyMode) allocator.HeadAddress = long.MaxValue; @@ -2599,12 +2858,12 @@ private void RestoreLatest(out byte[] cookie) if (!readOnlyMode) { - var headAddress = info.UntilAddress - allocator.GetOffsetInPage(info.UntilAddress); + var headAddress = info.UntilAddress - allocator.GetOffsetOnPage(info.UntilAddress); if (info.BeginAddress > headAddress) headAddress = info.BeginAddress; if (headAddress == 0) - headAddress = Constants.kFirstValidAddress; + headAddress = FirstValidAddress; try { @@ -2625,7 +2884,8 @@ private void RestoreLatest(out byte[] cookie) if (readOnlyMode) allocator.HeadAddress = long.MaxValue; - if (scanStart > 0) logCommitManager.OnRecovery(scanStart); + if (scanStart > 0) + logCommitManager.OnRecovery(scanStart); } private void RestoreSpecificCommit(long requestedCommitNum, out byte[] cookie) @@ -2637,7 +2897,8 @@ private void RestoreSpecificCommit(long requestedCommitNum, out byte[] cookie) long scanStart = 0; foreach (var metadataCommit in logCommitManager.ListCommits()) { - if (metadataCommit > requestedCommitNum) continue; + if (metadataCommit > requestedCommitNum) + continue; try { if (LoadCommitMetadata(metadataCommit, out info)) @@ -2672,15 +2933,15 @@ private void RestoreSpecificCommit(long requestedCommitNum, out byte[] cookie) } // At this point, we should have found the exact commit num requested - Debug.Assert(info.CommitNum == requestedCommitNum); + Debug.Assert(info.CommitNum == requestedCommitNum, $"info.CommitNum {info.CommitNum} must equal requestedCommitNum {requestedCommitNum}"); if (!readOnlyMode) { - var headAddress = info.UntilAddress - allocator.GetOffsetInPage(info.UntilAddress); + var headAddress = info.UntilAddress - allocator.GetOffsetOnPage(info.UntilAddress); if (info.BeginAddress > headAddress) headAddress = info.BeginAddress; if (headAddress == 0) - headAddress = Constants.kFirstValidAddress; + headAddress = FirstValidAddress; try { allocator.RestoreHybridLog(info.BeginAddress, headAddress, info.UntilAddress, info.UntilAddress); @@ -2698,7 +2959,8 @@ private void RestoreSpecificCommit(long requestedCommitNum, out byte[] cookie) if (readOnlyMode) allocator.HeadAddress = long.MaxValue; - if (scanStart > 0) logCommitManager.OnRecovery(scanStart); + if (scanStart > 0) + logCommitManager.OnRecovery(scanStart); } /// @@ -2732,7 +2994,7 @@ private async ValueTask RestoreLatestAsync(CancellationToken cancellatio try { using var scanIterator = Scan(info.UntilAddress, long.MaxValue, recover: false); - scanIterator.ScanForwardForCommit(ref info); + _ = scanIterator.ScanForwardForCommit(ref info); } catch { } } @@ -2744,7 +3006,7 @@ private async ValueTask RestoreLatestAsync(CancellationToken cancellatio logger?.LogDebug("Unable to recover using any available commit"); // Reset things to be something normal lol allocator.Initialize(); - CommittedUntilAddress = Constants.kFirstValidAddress; + CommittedUntilAddress = FirstValidAddress; beginAddress = allocator.BeginAddress; if (readOnlyMode) allocator.HeadAddress = long.MaxValue; @@ -2753,12 +3015,12 @@ private async ValueTask RestoreLatestAsync(CancellationToken cancellatio if (!readOnlyMode) { - var headAddress = info.UntilAddress - allocator.GetOffsetInPage(info.UntilAddress); + var headAddress = info.UntilAddress - allocator.GetOffsetOnPage(info.UntilAddress); if (info.BeginAddress > headAddress) headAddress = info.BeginAddress; if (headAddress == 0) - headAddress = Constants.kFirstValidAddress; + headAddress = FirstValidAddress; await allocator.RestoreHybridLogAsync(info.BeginAddress, headAddress, info.UntilAddress, info.UntilAddress, cancellationToken: cancellationToken).ConfigureAwait(false); } @@ -2769,7 +3031,8 @@ private async ValueTask RestoreLatestAsync(CancellationToken cancellatio if (readOnlyMode) allocator.HeadAddress = long.MaxValue; - if (scanStart > 0) logCommitManager.OnRecovery(scanStart); + if (scanStart > 0) + logCommitManager.OnRecovery(scanStart); return cookie; } @@ -2778,7 +3041,7 @@ private void CompleteRestoreFromCommit(TsavoriteLogRecoveryInfo info) { CommittedUntilAddress = info.UntilAddress; CommittedBeginAddress = info.BeginAddress; - SafeTailAddress = info.UntilAddress; + AdvanceSafeTailFloor(info.UntilAddress); } /// @@ -2793,40 +3056,43 @@ private unsafe bool TryAppend(IReadOnlySpanBatch readOnlySpanBatch, out long log { logicalAddress = 0; - int totalEntries = readOnlySpanBatch.TotalEntries(); + var totalEntries = readOnlySpanBatch.TotalEntries(); allocatedLength = 0; - for (int i = 0; i < totalEntries; i++) - { + for (var i = 0; i < totalEntries; i++) allocatedLength += Align(readOnlySpanBatch.Get(i).Length) + headerSize; - } ValidateAllocatedLength(allocatedLength); epoch.Resume(); - if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); + BeginInflightEnqueue(); + try + { + if (commitNum == long.MaxValue) throw new TsavoriteException("Attempting to enqueue into a completed log"); - logicalAddress = allocator.TryAllocateRetryNow(allocatedLength); + if (!allocator.TryAllocateRetryNow(allocatedLength, out logicalAddress)) + { + if (cannedException != null) + throw cannedException; + return false; + } - if (logicalAddress == 0) - { - epoch.Suspend(); - if (cannedException != null) throw cannedException; - return false; + var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); + for (var i = 0; i < totalEntries; i++) + { + var span = readOnlySpanBatch.Get(i); + var entryLength = span.Length; + fixed (byte* bp = &span.GetPinnableReference()) + Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), entryLength, entryLength); + SetHeader(entryLength, (byte*)physicalAddress); + physicalAddress += Align(entryLength) + headerSize; + } } - - var physicalAddress = allocator.GetPhysicalAddress(logicalAddress); - for (int i = 0; i < totalEntries; i++) + finally { - var span = readOnlySpanBatch.Get(i); - var entryLength = span.Length; - fixed (byte* bp = &span.GetPinnableReference()) - Buffer.MemoryCopy(bp, (void*)(headerSize + physicalAddress), entryLength, entryLength); - SetHeader(entryLength, (byte*)physicalAddress); - physicalAddress += Align(entryLength) + headerSize; + EndInflightEnqueue(); + epoch.Suspend(); } - safeTailRefreshEntryEnqueued?.Signal(); - epoch.Suspend(); - if (AutoCommit) Commit(); + if (autoCommit) Commit(); return true; } @@ -2839,7 +3105,7 @@ private unsafe void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, obje logger?.LogError($"{nameof(AsyncGetFromDiskCallback)} error: {{errorCode}}", errorCode); ctx.record.Return(); ctx.record = null; - ctx.completedRead.Release(); + _ = ctx.completedRead.Release(); } else { @@ -2851,19 +3117,19 @@ private unsafe void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, obje logger?.LogDebug("Invalid record length found: {length}", length); ctx.record.Return(); ctx.record = null; - ctx.completedRead.Release(); + _ = ctx.completedRead.Release(); } else { - int requiredBytes = headerSize + length; + var requiredBytes = headerSize + length; if (ctx.record.available_bytes >= requiredBytes) { - ctx.completedRead.Release(); + _ = ctx.completedRead.Release(); } else { ctx.record.Return(); - allocator.AsyncReadRecordToMemory(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ref ctx); + allocator.AsyncReadBlittableRecordToMemory(ctx.logicalAddress, requiredBytes, AsyncGetFromDiskCallback, ref ctx); } } } @@ -2878,7 +3144,7 @@ private void AsyncGetHeaderOnlyFromDiskCallback(uint errorCode, uint numBytes, o logger?.LogError($"{nameof(AsyncGetHeaderOnlyFromDiskCallback)} error: {{errorCode}}", errorCode); ctx.record.Return(); ctx.record = null; - ctx.completedRead.Release(); + _ = ctx.completedRead.Release(); } else { @@ -2888,7 +3154,7 @@ private void AsyncGetHeaderOnlyFromDiskCallback(uint errorCode, uint numBytes, o ctx.record.Return(); ctx.record = null; } - ctx.completedRead.Release(); + _ = ctx.completedRead.Release(); } } @@ -2962,7 +3228,6 @@ private int GetRecordLengthAndFree(SectorAlignedMemory record) return length; } - private bool CommitInternal(out long commitTail, out long actualCommitNum, bool fastForwardAllowed, byte[] cookie, long proposedCommitNum, Action callback) { if (cannedException != null) @@ -3018,7 +3283,7 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool { // Ok to retry in critical section, any concurrently invoked commit would block, but cannot progress // anyways if no record can be enqueued - while (!TryEnqueueCommitRecord(ref info)) Thread.Yield(); + while (!TryEnqueueCommitRecord(ref info)) _ = Thread.Yield(); commitTail = info.UntilAddress; } else @@ -3030,7 +3295,7 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool info.UntilAddress = commitTail = TailAddress; } - Utility.MonotonicUpdate(ref commitCoveredAddress, commitTail, out _); + _ = Utility.MonotonicUpdate(ref commitCoveredAddress, commitTail, out _); commitPolicy.OnCommitCreated(info); // Enqueue the commit record's content and offset into the queue so it can be picked up by the next flush @@ -3038,7 +3303,6 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool ongoingCommitRequests.Enqueue((commitTail, info)); } - // As an optimization, if a concurrent flush has already advanced FlushedUntilAddress // past this commit, we can manually trigger a commit callback for safety, and return. if (commitTail <= FlushedUntilAddress) @@ -3048,7 +3312,7 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool } // Otherwise, move to set read-only tail and flush - bool isProtected = epoch.ThisInstanceProtected(); + var isProtected = epoch.ThisInstanceProtected(); if (!isProtected) epoch.Resume(); try diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogRecoveryInfo.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogRecoveryInfo.cs index 244d36f0855..fe3e4c760d0 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogRecoveryInfo.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogRecoveryInfo.cs @@ -5,12 +5,11 @@ using System.Buffers.Binary; using System.Diagnostics; using System.IO; -using System.Text; namespace Tsavorite.core { /// - /// Recovery info for Tsavorite Log + /// Recovery info for TsavoriteLog /// public struct TsavoriteLogRecoveryInfo { @@ -45,7 +44,7 @@ public struct TsavoriteLogRecoveryInfo public bool FastForwardAllowed; /// - /// callback to invoke when commit is presistent + /// callback to invoke when commit is persisted /// public Action Callback; @@ -78,62 +77,33 @@ public void Initialize(ReadOnlySpan input) UntilAddress = BinaryPrimitives.ReadInt64LittleEndian(input); input = input.Slice(sizeof(long)); - if (version > 0) - { - CommitNum = BinaryPrimitives.ReadInt64LittleEndian(input); - input = input.Slice(sizeof(long)); - } - else - { - CommitNum = -1; - } + CommitNum = BinaryPrimitives.ReadInt64LittleEndian(input); + input = input.Slice(sizeof(long)); if (version < 0 || version > TsavoriteLogRecoveryVersion) throw new TsavoriteException("Invalid version found during commit recovery"); - if (BinaryPrimitives.TryReadInt32LittleEndian(input, out var iteratorCount)) - input = input.Slice(sizeof(int)); - - if (iteratorCount > 0) - { - for (var i = 0; i < iteratorCount; i++) - { - var keyLength = BinaryPrimitives.ReadInt32LittleEndian(input); - input = input.Slice(sizeof(int)); - - var iteratorKey = Encoding.UTF8.GetString(input.Slice(0, keyLength)); - input = input.Slice(keyLength); - - var iteratorValue = BinaryPrimitives.ReadInt64LittleEndian(input); - input = input.Slice(sizeof(long)); - } - } - int cookieLength = -1; long cookieChecksum = 0; - if (version >= TsavoriteLogRecoveryVersion) - { - if (BinaryPrimitives.TryReadInt32LittleEndian(input, out cookieLength)) - input = input.Slice(sizeof(int)); - if (cookieLength >= 0) + if (BinaryPrimitives.TryReadInt32LittleEndian(input, out cookieLength)) + input = input.Slice(sizeof(int)); + + if (cookieLength >= 0) + { + Cookie = input.Slice(0, cookieLength).ToArray(); + unsafe { - Cookie = input.Slice(0, cookieLength).ToArray(); - unsafe - { - fixed (byte* ptr = Cookie) - cookieChecksum = (long)Utility.XorBytes(ptr, cookieLength); - } + fixed (byte* ptr = Cookie) + cookieChecksum = (long)Utility.XorBytes(ptr, cookieLength); } } - long computedChecksum = BeginAddress ^ UntilAddress; - if (version >= TsavoriteLogRecoveryVersion) - computedChecksum ^= CommitNum ^ iteratorCount ^ cookieLength ^ cookieChecksum; + long computedChecksum = BeginAddress ^ UntilAddress ^ CommitNum ^ cookieLength ^ cookieChecksum; // Handle case where all fields are zero - if (version == 0 && BeginAddress == 0 && UntilAddress == 0 && iteratorCount == 0) - throw new TsavoriteException("Invalid checksum found during commit recovery"); + if (version == 0 && BeginAddress == 0 && UntilAddress == 0) + throw new TsavoriteException("Invalid all-fields-zero found during commit recovery"); if (checkSum != computedChecksum) throw new TsavoriteException("Invalid checksum found during commit recovery"); @@ -157,7 +127,6 @@ public readonly byte[] ToByteArray() { writer.Write(TsavoriteLogRecoveryVersion); // version - int iteratorCount = 0; int cookieLength = -1; long cookieChecksum = 0; if (Cookie != null) @@ -171,11 +140,10 @@ public readonly byte[] ToByteArray() } } - writer.Write(BeginAddress ^ UntilAddress ^ CommitNum ^ iteratorCount ^ cookieLength ^ cookieChecksum); // checksum + writer.Write(BeginAddress ^ UntilAddress ^ CommitNum ^ cookieLength ^ cookieChecksum); // checksum writer.Write(BeginAddress); writer.Write(UntilAddress); writer.Write(CommitNum); - writer.Write(iteratorCount); // leaving this field for backwards compatibility writer.Write(cookieLength); if (cookieLength > 0) writer.Write(Cookie); @@ -188,7 +156,7 @@ public readonly byte[] ToByteArray() /// size of this recovery info serialized public int SerializedSize() { - return sizeof(int) + 4 * sizeof(long) + sizeof(int) + sizeof(int) + (Cookie?.Length ?? 0); + return sizeof(int) + 4 * sizeof(long) + sizeof(int) + (Cookie?.Length ?? 0); } /// diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogIterator.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanIterator.cs similarity index 86% rename from libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogIterator.cs rename to libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanIterator.cs index 0785aeb9f82..f491002303e 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogIterator.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanIterator.cs @@ -12,15 +12,13 @@ namespace Tsavorite.core { - using EmptyStoreFunctions = StoreFunctions>; - /// - /// Scan iterator for hybrid log + /// Scan iterator for TsavoriteLog /// public class TsavoriteLogScanIterator : ScanIteratorBase, IDisposable { protected readonly TsavoriteLog tsavoriteLog; - private readonly BlittableAllocatorImpl allocator; + private readonly TsavoriteLogAllocatorImpl allocator; private readonly BlittableFrame frame; private readonly GetMemory getMemory; private readonly int headerSize; @@ -33,22 +31,11 @@ public class TsavoriteLogScanIterator : ScanIteratorBase, IDisposable /// public bool Ended => (nextAddress >= endAddress) || (tsavoriteLog.LogCompleted && nextAddress == tsavoriteLog.TailAddress); - /// - /// Constructor - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// - internal unsafe TsavoriteLogScanIterator(TsavoriteLog tsavoriteLog, BlittableAllocatorImpl hlog, long beginAddress, long endAddress, - GetMemory getMemory, ScanBufferingMode scanBufferingMode, LightEpoch epoch, int headerSize, bool scanUncommitted = false, ILogger logger = null) - : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddress(0) : beginAddress, endAddress, scanBufferingMode, false, epoch, hlog.LogPageSizeBits, logger: logger) + /// Constructor + internal unsafe TsavoriteLogScanIterator(TsavoriteLog tsavoriteLog, TsavoriteLogAllocatorImpl hlog, long beginAddress, long endAddress, + GetMemory getMemory, DiskScanBufferingMode diskScanBufferingMode, LightEpoch epoch, int headerSize, bool scanUncommitted = false, ILogger logger = null) + : base(beginAddress == 0 ? hlog.GetFirstValidLogicalAddressOnPage(0) : beginAddress, endAddress, + diskScanBufferingMode, InMemoryScanBufferingMode.NoBuffering, includeClosedRecords: false, epoch, hlog.LogPageSizeBits, logger: logger) { this.tsavoriteLog = tsavoriteLog; allocator = hlog; @@ -76,7 +63,6 @@ internal unsafe TsavoriteLogScanIterator(TsavoriteLog tsavoriteLog, BlittableAll if (!await WaitAsync(token).ConfigureAwait(false)) yield break; } - yield return (result, length, currentAddress, nextAddress); } } @@ -158,7 +144,7 @@ public ValueTask WaitAsync(CancellationToken token = default) return SlowWaitAsync(this, token); } - if (NextAddress < tsavoriteLog.SafeTailAddress) + if (NextAddress < tsavoriteLog.SafeTailAddress || NextAddress < tsavoriteLog.RefreshSafeTailAddress()) return new ValueTask(true); return SlowWaitUncommittedAsync(token); } @@ -176,7 +162,7 @@ private static async ValueTask SlowWaitAsync(TsavoriteLogScanIterator @thi // Ignore commit exceptions, except when the token is signaled try { - await commitTask.WithCancellationAsync(token).ConfigureAwait(false); + _ = await commitTask.WithCancellationAsync(token).ConfigureAwait(false); } catch (ObjectDisposedException) { return false; } catch when (!token.IsCancellationRequested) { } @@ -187,25 +173,25 @@ protected virtual async ValueTask SlowWaitUncommittedAsync(CancellationTok { while (true) { - if (this.disposed) + if (disposed) return false; - if (this.Ended) return false; + if (Ended) return false; - var tcs = this.tsavoriteLog.refreshUncommittedTcs; + var tcs = tsavoriteLog.refreshUncommittedTcs; if (tcs == null) { var newTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - tcs = Interlocked.CompareExchange(ref this.tsavoriteLog.refreshUncommittedTcs, newTcs, null); + tcs = Interlocked.CompareExchange(ref tsavoriteLog.refreshUncommittedTcs, newTcs, null); tcs ??= newTcs; // successful CAS so update the local var } - if (this.NextAddress < this.tsavoriteLog.SafeTailAddress) + if (NextAddress < tsavoriteLog.SafeTailAddress || NextAddress < tsavoriteLog.RefreshSafeTailAddress()) return true; // Ignore refresh-uncommitted exceptions, except when the token is signaled try { - await tcs.Task.WithCancellationAsync(token).ConfigureAwait(false); + _ = await tcs.Task.WithCancellationAsync(token).ConfigureAwait(false); } catch (ObjectDisposedException) { return false; } catch when (!token.IsCancellationRequested) { } @@ -386,7 +372,6 @@ public unsafe bool GetNext(MemoryPool pool, out IMemoryOwner entry, } } - /// /// Consume the next entry in the log with the given consumer /// @@ -411,7 +396,7 @@ public unsafe bool TryConsumeNext(T consumer) where T : ILogEntryConsumer long physicalAddress; bool isCommitRecord; int entryLength; - bool onFrame = false; + var onFrame = false; try { var hasNext = GetNextInternal(out physicalAddress, out entryLength, out currentAddress, @@ -499,11 +484,11 @@ public unsafe bool TryBulkConsumeNext(T consumer, int maxChunkSize = 0) where // If initializing wait for completion while (tsavoriteLog.Initializing) { - Thread.Yield(); + _ = Thread.Yield(); epoch.ProtectAndDrain(); } - var hasNext = GetNextInternal(out long startPhysicalAddress, out var newEntryLength, out var startLogicalAddress, out var endLogicalAddress, out bool isCommitRecord, out bool onFrame); + var hasNext = GetNextInternal(out long startPhysicalAddress, out int newEntryLength, out long startLogicalAddress, out long endLogicalAddress, out bool isCommitRecord, out bool onFrame); if (!hasNext) { @@ -650,35 +635,32 @@ public override void Dispose() } } - internal override void AsyncReadPagesFromDeviceToFrame(long readPageStart, int numPages, long untilAddress, TContext context, out CountdownEvent completed, long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) - => allocator.AsyncReadPagesFromDeviceToFrame(readPageStart, numPages, untilAddress, AsyncReadPagesCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice, cts); + internal override void AsyncReadPageFromDeviceToFrame(CircularDiskReadBuffer _ /*readBuffers*/, long readPage, long untilAddress, TContext context, out CountdownEvent completed, + long devicePageOffset = 0, IDevice device = null, IDevice objectLogDevice = null, CancellationTokenSource cts = null) + => allocator.AsyncReadPageFromDeviceToFrame(readBuffers: null, readPage, untilAddress, AsyncReadPagesToFrameCallback, context, frame, out completed, devicePageOffset, device, objectLogDevice, cts); - private unsafe void AsyncReadPagesCallback(uint errorCode, uint numBytes, object context) + private unsafe void AsyncReadPagesToFrameCallback(uint errorCode, uint numBytes, object context) { try { var result = (PageAsyncReadResult)context; - if (errorCode != 0) + if (errorCode == 0) + _ = result.handle?.Signal(); + else { - logger?.LogError($"{nameof(AsyncReadPagesCallback)} error: {{errorCode}}", errorCode); + logger?.LogError($"{nameof(AsyncReadPagesToFrameCallback)} error: {{errorCode}}", errorCode); result.cts?.Cancel(); } - - if (result.freeBuffer1 != null) - { - if (errorCode == 0) - allocator._wrapper.PopulatePage(result.freeBuffer1.GetValidPointer(), result.freeBuffer1.required_bytes, result.page); - result.freeBuffer1.Return(); - result.freeBuffer1 = null; - } - - if (errorCode == 0) - result.handle?.Signal(); - Interlocked.MemoryBarrier(); } - catch when (disposed) { } + catch when (disposed) + { + } + finally + { + _ = Interlocked.Decrement(ref pendingDrainCallbacks); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -720,7 +702,7 @@ internal unsafe bool ScanForwardForCommit(ref TsavoriteLogRecoveryInfo info, lon if (info.CommitNum == commitNum) return true; - // User wants any commie + // User wants any commit if (commitNum == -1) return foundCommit; // requested commit not found @@ -728,16 +710,8 @@ internal unsafe bool ScanForwardForCommit(ref TsavoriteLogRecoveryInfo info, lon } /// - /// Retrieve physical address of next iterator value - /// (under epoch protection if it is from main page buffer) + /// Retrieve physical address of next iterator value (under epoch protection if it is from main page buffer) /// - /// - /// - /// - /// - /// - /// - /// private unsafe bool GetNextInternal(out long physicalAddress, out int entryLength, out long currentAddress, out long outNextAddress, out bool commitRecord, out bool onFrame) { while (true) @@ -767,15 +741,35 @@ private unsafe bool GetNextInternal(out long physicalAddress, out int entryLengt outNextAddress = currentAddress; } - var _currentPage = currentAddress >> allocator.LogPageSizeBits; + var _currentPage = allocator.GetPage(currentAddress); var _currentFrame = _currentPage % frameSize; - var _currentOffset = currentAddress & allocator.PageSizeMask; + var _currentOffset = allocator.GetOffsetOnPage(currentAddress); if (disposed) return false; - if ((currentAddress >= endAddress) || (currentAddress >= (scanUncommitted ? tsavoriteLog.SafeTailAddress : tsavoriteLog.CommittedUntilAddress))) + if (currentAddress >= endAddress) + return false; + + if (scanUncommitted) + { + // Check cached SafeTailAddress first (O(1)). If caught up, spin briefly (~2-4μs / + // 100 PAUSE instructions) to let producers complete in-flight writes, then re-check + // cache (may have been advanced by multi-iterator pre-refresh or page-drive) before + // falling back to the full epoch-table scan. At 15 Mops this batches ~30-60 records + // per scan, amortizing the O(kTableSize) scan cost to ~1-3ns per record. + if (currentAddress >= tsavoriteLog.SafeTailAddress) + { + Thread.SpinWait(100); + if (currentAddress >= tsavoriteLog.SafeTailAddress + && currentAddress >= tsavoriteLog.RefreshSafeTailAddress()) + return false; + } + } + else if (currentAddress >= tsavoriteLog.CommittedUntilAddress) + { return false; + } if (currentAddress < _headAddress) { @@ -805,14 +799,14 @@ private unsafe bool GetNextInternal(out long physicalAddress, out int entryLengt if (entryLength == 0) { // Zero-ed out bytes could be padding at the end of page, first jump to the start of next page. - var nextStart = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits; + var nextStart = allocator.GetLogicalAddressOfStartOfPage(1 + allocator.GetPage(currentAddress)); if (Utility.MonotonicUpdate(ref nextAddress, nextStart, out _)) { - var pageOffset = currentAddress & ((1 << allocator.LogPageSizeBits) - 1); + var pageOffset = allocator.GetOffsetOnPage(currentAddress); // If zeroed out field is at page start, we encountered an uninitialized page and should signal up if (pageOffset == 0) - throw new TsavoriteException("Uninitialized page found during scan at page " + (currentAddress >> allocator.LogPageSizeBits)); + throw new TsavoriteException("Uninitialized page found during scan at page " + allocator.GetPage(currentAddress)); } continue; } @@ -845,8 +839,8 @@ private unsafe bool GetNextInternal(out long physicalAddress, out int entryLengt } } - if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize) - currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits; + if ((allocator.GetOffsetOnPage(currentAddress) + recordSize) == allocator.PageSize) + currentAddress = allocator.GetLogicalAddressOfStartOfPage(1 + allocator.GetPage(currentAddress)); else currentAddress += recordSize; @@ -885,16 +879,32 @@ private unsafe bool ExpandGetNextInternal(long startPhysicalAddress, ref int tot return false; } - var _currentPage = currentAddress >> allocator.LogPageSizeBits; + var _currentPage = allocator.GetPage(currentAddress); var _currentFrame = _currentPage % frameSize; - var _currentOffset = currentAddress & allocator.PageSizeMask; + var _currentOffset = allocator.GetOffsetOnPage(currentAddress); if (disposed) return false; - if ((currentAddress >= endAddress) || (currentAddress >= (scanUncommitted ? tsavoriteLog.SafeTailAddress : tsavoriteLog.CommittedUntilAddress))) + if (currentAddress >= endAddress) return false; + if (scanUncommitted) + { + // Same spin-wait + scan amortization as the primary GetNext path above. + if (currentAddress >= tsavoriteLog.SafeTailAddress) + { + Thread.SpinWait(100); + if (currentAddress >= tsavoriteLog.SafeTailAddress + && currentAddress >= tsavoriteLog.RefreshSafeTailAddress()) + return false; + } + } + else if (currentAddress >= tsavoriteLog.CommittedUntilAddress) + { + return false; + } + if (currentAddress < _headAddress) { var _endAddress = endAddress; @@ -950,8 +960,8 @@ private unsafe bool ExpandGetNextInternal(long startPhysicalAddress, ref int tot } } - if ((currentAddress & allocator.PageSizeMask) + recordSize == allocator.PageSize) - currentAddress = (1 + (currentAddress >> allocator.LogPageSizeBits)) << allocator.LogPageSizeBits; + if ((allocator.GetOffsetOnPage(currentAddress) + recordSize) == allocator.PageSize) + currentAddress = allocator.GetLogicalAddressOfStartOfPage(1 + allocator.GetPage(currentAddress)); else currentAddress += recordSize; diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanSingleIterator.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanSingleIterator.cs index 7ffb879bff8..89dd3de2970 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanSingleIterator.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogScanSingleIterator.cs @@ -7,10 +7,8 @@ namespace Tsavorite.core { - using EmptyStoreFunctions = StoreFunctions>; - /// - /// Scan iterator for hybrid log - only a single scan is supported per instance + /// Scan iterator for TsavoriteLog's hybrid log - only a single scan is supported per instance /// This modification allows us to use a SingleWaiterAutoResetEvent per iterator /// so we can avoid TCS allocations per tail bump. /// @@ -18,9 +16,9 @@ public sealed class TsavoriteLogScanSingleIterator : TsavoriteLogScanIterator { readonly SingleWaiterAutoResetEvent onEnqueue; - internal TsavoriteLogScanSingleIterator(TsavoriteLog tsavoriteLog, BlittableAllocatorImpl hlog, long beginAddress, long endAddress, - GetMemory getMemory, ScanBufferingMode scanBufferingMode, LightEpoch epoch, int headerSize, bool scanUncommitted = false, ILogger logger = null) - : base(tsavoriteLog, hlog, beginAddress, endAddress, getMemory, scanBufferingMode, epoch, headerSize, scanUncommitted, logger) + internal TsavoriteLogScanSingleIterator(TsavoriteLog TsavoriteLog, TsavoriteLogAllocatorImpl hlog, long beginAddress, long endAddress, + GetMemory getMemory, DiskScanBufferingMode scanBufferingMode, LightEpoch epoch, int headerSize, bool scanUncommitted = false, ILogger logger = null) + : base(TsavoriteLog, hlog, beginAddress, endAddress, getMemory, scanBufferingMode, epoch, headerSize, scanUncommitted, logger) { onEnqueue = new() { @@ -47,7 +45,8 @@ protected override async ValueTask SlowWaitUncommittedAsync(CancellationTo return false; if (this.Ended) return false; - if (this.NextAddress < this.tsavoriteLog.SafeTailAddress) + if (this.NextAddress < this.tsavoriteLog.SafeTailAddress + || this.NextAddress < this.tsavoriteLog.RefreshSafeTailAddress()) return true; // Ignore refresh-uncommitted exceptions, except when the token is signaled diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogSettings.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogSettings.cs index 974d0e47234..0c9f1678059 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogSettings.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLogSettings.cs @@ -14,7 +14,7 @@ namespace Tsavorite.core public delegate byte[] GetMemory(int minLength); /// - /// Type of checksum to add to log + /// Type of checksum to add to TsavoriteLog /// public enum LogChecksumType { @@ -29,7 +29,7 @@ public enum LogChecksumType } /// - /// Tsavorite Log Settings + /// Tsavorite Log LogSettings /// public class TsavoriteLogSettings : IDisposable { @@ -60,7 +60,7 @@ public class TsavoriteLogSettings : IDisposable public long MemorySize = 1L << 23; /// - /// Support bit-based setting of memory size for backward compatibility, use MemorySize directly for simplicity. + /// Support bit-based setting of memory size for backward compatibility, use LogMemorySize directly for simplicity. /// public int MemorySizeBits { set { MemorySize = 1L << value; } } @@ -76,7 +76,7 @@ public class TsavoriteLogSettings : IDisposable public int SegmentSizeBits { set { SegmentSize = 1L << value; } } /// - /// Log commit manager - if you want to override the default implementation of commit. + /// TsavoriteLog commit manager - if you want to override the default implementation of commit. /// public ILogCommitManager LogCommitManager = null; @@ -93,7 +93,7 @@ public class TsavoriteLogSettings : IDisposable public GetMemory GetMemory = null; /// - /// Type of checksum to add to log + /// Type of checksum to add to TsavoriteLog /// public LogChecksumType LogChecksum = LogChecksumType.None; @@ -121,7 +121,7 @@ public class TsavoriteLogSettings : IDisposable public bool RemoveOutdatedCommits = true; /// - /// Log commit policy that influences the behavior of Commit() calls. + /// TsavoriteLog commit policy that influences the behavior of Commit() calls. /// public LogCommitPolicy LogCommitPolicy = LogCommitPolicy.Default(); @@ -130,11 +130,6 @@ public class TsavoriteLogSettings : IDisposable /// public bool TryRecoverLatest = true; - /// - /// SafeTailAddress refresh frequency in milliseconds. -1 => disabled; 0 => immediate refresh after every enqueue, >1 => refresh period in milliseconds. - /// - public int SafeTailRefreshFrequencyMs = -1; - /// /// Whether we automatically commit the log as records are inserted /// @@ -204,12 +199,14 @@ public override string ToString() internal LogSettings GetLogSettings() { + var pageSizeBits = Utility.NumBitsPreviousPowerOf2(PageSize); return new LogSettings { LogDevice = LogDevice, - PageSizeBits = Utility.NumBitsPreviousPowerOf2(PageSize), + PageSizeBits = pageSizeBits, SegmentSizeBits = Utility.NumBitsPreviousPowerOf2(SegmentSize), - MemorySizeBits = ReadOnlyMode ? 0 : Utility.NumBitsPreviousPowerOf2(MemorySize), + MemorySize = ReadOnlyMode ? 0 : MemorySize, + PageCount = (int)(MemorySize >> pageSizeBits), ReadCopyOptions = ReadCopyOptions.None, MutableFraction = MutableFraction, ObjectLogDevice = null, diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/AsyncResultTypes.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/AsyncResultTypes.cs index 7d374beb3cc..389ccb8d3b0 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/AsyncResultTypes.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/AsyncResultTypes.cs @@ -10,13 +10,13 @@ internal struct AsyncGetFromDiskResult public TContext context; } - internal unsafe struct HashIndexPageAsyncFlushResult + internal struct HashIndexPageAsyncFlushResult { public int chunkIndex; public SectorAlignedMemory mem; } - internal unsafe struct HashIndexPageAsyncReadResult + internal struct HashIndexPageAsyncReadResult { public int chunkIndex; } diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.cs index 8461949ea3b..f65bf5923f1 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/BufferPool.cs @@ -16,6 +16,9 @@ namespace Tsavorite.core { +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + /// /// Sector aligned memory allocator /// @@ -35,27 +38,33 @@ public sealed unsafe class SectorAlignedMemory internal GCHandle handle; /// - /// Offset + /// Offset for initial allocation alignment of the block; this is the offset from the first element of to form . + /// This alignment is internal to , and ensures that callers see an aligned starting address. /// - public int offset; + public int aligned_offset; /// - /// Aligned pointer + /// Aligned pointer; initial allocation (the first element of ) plus + /// This alignment is internal to , and ensures that callers see an aligned starting address. /// public byte* aligned_pointer; /// - /// Valid offset + /// Valid offset for operations above , to get their own desired alignment relative to our aligned starting address. + /// This is set by the caller for operations such as file reading, which rounds down to the nearest sector size; this is the amount of that rounding down. + /// Used by , which is + . /// public int valid_offset; /// - /// Required bytes + /// Required (requested) bytes for the current operation: the unaligned number of bytes to read. There will always be at least this much usable space in the allocation. + /// Use this when the original request size is needed. /// public int required_bytes; /// - /// Available bytes + /// Available bytes after the operation is complete: the number of bytes actually read, e.g. aligned number of bytes requested. See . + /// Use this to see if there are additional bytes over the original request (see . /// public int available_bytes; @@ -106,16 +115,23 @@ public SectorAlignedMemory(int level = default) /// public SectorAlignedMemory(int numRecords, int sectorSize) { - int recordSize = 1; - int requiredSize = sectorSize + (((numRecords) * recordSize + (sectorSize - 1)) & ~(sectorSize - 1)); + const int recordSize = 1; + required_bytes = numRecords * recordSize; + int requiredSize = sectorSize + RoundUp(required_bytes, sectorSize); // An additional sector size for the aligned_offset buffer = GC.AllocateArray(requiredSize, true); long bufferAddr = (long)Unsafe.AsPointer(ref buffer[0]); aligned_pointer = (byte*)((bufferAddr + (sectorSize - 1)) & ~((long)sectorSize - 1)); - offset = (int)((long)aligned_pointer - bufferAddr); + aligned_offset = (int)((long)aligned_pointer - bufferAddr); // Assume ctor is called for allocation and leave Free unset } + public unsafe (byte[] array, long offset) GetArrayAndUnalignedOffset(long alignedOffset) + { + long ptr = (long)Unsafe.AsPointer(ref buffer[0]); + return (buffer, alignedOffset + ptr - (long)aligned_pointer); + } + /// /// Dispose /// @@ -139,17 +155,44 @@ public void Return() /// /// Get the total aligned memory capacity of the buffer /// - public int AlignedTotalCapacity => buffer.Length - offset; + public int AlignedTotalCapacity => buffer.Length - aligned_offset; + + /// + /// Get the total valid memory capacity of the buffer + /// + public int ValidTotalCapacity => AlignedTotalCapacity - valid_offset; + + /// + /// Get the total valid required (requested) capacity of the buffer + /// + public int RequiredCapacity => required_bytes - valid_offset; /// - /// Get valid pointer + /// Get valid pointer (accounts for aligned padding plus any offset specified for the valid start of data) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public byte* GetValidPointer() - { - return aligned_pointer + valid_offset; - } + public byte* GetValidPointer() => aligned_pointer + valid_offset; + + /// + /// Get Span of entire allocated space after the valid pointer + /// + public Span TotalValidSpan => new(GetValidPointer(), ValidTotalCapacity); + + /// + /// Get Span of entire allocated space after the aligned pointer (see ). + /// + public Span AvailableSpan => new(aligned_pointer, available_bytes); + + /// + /// Get Span of entire allocated space after the valid pointer (see ). + /// + public Span AvailableValidSpan => new(GetValidPointer(), available_bytes - valid_offset); + + /// + /// Returns the Span of requested space (see ). + /// + public Span RequiredValidSpan => new(GetValidPointer(), RequiredCapacity); /// /// ToString @@ -157,9 +200,11 @@ public void Return() /// public override string ToString() { - return string.Format($"{(long)aligned_pointer} {offset} {valid_offset} {required_bytes} {available_bytes}" + return string.Format($"aligned: [offset {aligned_offset}, ptr {(long)aligned_pointer} = 0x{(long)aligned_pointer:X}];" + + $" valid: [offset {valid_offset} ptr {(long)GetValidPointer()} = 0x{(long)GetValidPointer():X}];" + + $" reqBytes {required_bytes}; availBytes {available_bytes}; cap {AlignedTotalCapacity}" #if CHECK_FREE - + $" {this.Free}" + + $"; free {Free}" #endif ); } @@ -206,6 +251,24 @@ public SectorAlignedBufferPool(int recordSize, int sectorSize) this.sectorSize = sectorSize; } + public void EnsureSize(ref SectorAlignedMemory page, int size) + { + if (page is null) + { + page = Get(size); + return; + } + if (page.AlignedTotalCapacity < size) + { + page.Return(); + page = Get(size); + return; + } + + // Reusing the page, so ensure this is set correctly. + page.required_bytes = size; + } + /// /// Return /// @@ -262,7 +325,8 @@ public unsafe SectorAlignedMemory Get(int numRecords) Interlocked.Increment(ref totalGets); #endif - int requiredSize = sectorSize + (((numRecords) * recordSize + (sectorSize - 1)) & ~(sectorSize - 1)); + int required_bytes = numRecords * recordSize; + int requiredSize = RoundUp(required_bytes, sectorSize); int index = Position(requiredSize / sectorSize); if (queue[index] == null) { @@ -278,21 +342,24 @@ public unsafe SectorAlignedMemory Get(int numRecords) if (UnpinOnReturn) { page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned); - page.aligned_pointer = (byte*)(((long)page.handle.AddrOfPinnedObject() + (sectorSize - 1)) & ~((long)sectorSize - 1)); - page.offset = (int)((long)page.aligned_pointer - (long)page.handle.AddrOfPinnedObject()); + page.aligned_pointer = (byte*)RoundUp(page.handle.AddrOfPinnedObject(), sectorSize); + page.aligned_offset = (int)((long)page.aligned_pointer - page.handle.AddrOfPinnedObject()); } + page.required_bytes = required_bytes; return page; } page = new SectorAlignedMemory(level: index) { - buffer = GC.AllocateArray(sectorSize * (1 << index), !UnpinOnReturn) + // Add an additional sector for the leading RoundUp of pageAddr to sectorSize. + buffer = GC.AllocateArray(sectorSize * ((1 << index) + 1), !UnpinOnReturn) }; if (UnpinOnReturn) page.handle = GCHandle.Alloc(page.buffer, GCHandleType.Pinned); long pageAddr = (long)Unsafe.AsPointer(ref page.buffer[0]); - page.aligned_pointer = (byte*)((pageAddr + (sectorSize - 1)) & ~((long)sectorSize - 1)); - page.offset = (int)((long)page.aligned_pointer - pageAddr); + page.aligned_pointer = (byte*)RoundUp(pageAddr, sectorSize); + page.aligned_offset = (int)((long)page.aligned_pointer - pageAddr); + page.required_bytes = required_bytes; page.pool = this; return page; } diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/CompletionEvent.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/CompletionEvent.cs index 9149c1447f8..6ce89aed336 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/CompletionEvent.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/CompletionEvent.cs @@ -30,6 +30,7 @@ internal void Set() { // Release all waiting threads tempSemaphore.Release(int.MaxValue); + // tempSemaphore.Dispose(); TODO: We cannot Dispose() here because there may still be waiters that have not yet been released. break; } } @@ -41,11 +42,14 @@ internal void Set() internal Task WaitAsync(CancellationToken token = default) => semaphore.WaitAsync(token); + internal Task WaitAsync(TimeSpan timeSpan, CancellationToken cancellationToken = default) => semaphore.WaitAsync(timeSpan, cancellationToken); + /// public void Dispose() { - semaphore?.Dispose(); - semaphore = null; + var tempSemaphore = semaphore; + if (tempSemaphore != null && Interlocked.CompareExchange(ref semaphore, null, tempSemaphore) == tempSemaphore) + tempSemaphore.Dispose(); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/EmptyKey.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/EmptyKey.cs new file mode 100644 index 00000000000..fd50fde1a4f --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/EmptyKey.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; + +namespace Tsavorite.core +{ + /// + /// implementation representing a completely empty key. + /// + /// Not a key with no bytes, and key that isn't set. + /// + internal readonly struct EmptyKey : IKey + { + /// + public bool IsPinned => true; + + /// + bool IKey.IsEmpty => true; + + /// + public ReadOnlySpan KeyBytes => []; + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/FlushCompletionTracker.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/FlushCompletionTracker.cs new file mode 100644 index 00000000000..2e844c24fb3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/FlushCompletionTracker.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Tsavorite.core +{ + /// + /// Tracks the completion of page flush operations during snapshot checkpoints. + /// Signals a when all pages have been flushed, + /// or faults it if an exception occurs. Optionally supports per-page throttle waiting. + /// + internal sealed class FlushCompletionTracker + { + /// + /// Task completion source to signal when all page flushes are done, or to fault on error. + /// + readonly TaskCompletionSource completionTcs; + + /// + /// Semaphore for per-page flush completion, used only when throttling is enabled. + /// + readonly SemaphoreSlim flushSemaphore; + + /// + /// Number of pages being flushed + /// + int count; + + public override string ToString() + { + var flushSemCount = flushSemaphore?.CurrentCount.ToString() ?? "null"; + return $"count {count}, flushSemCount {flushSemCount}"; + } + + /// + /// Create a flush completion tracker + /// + /// TaskCompletionSource to signal when all flushes complete or to fault on error + /// If true, creates a semaphore for per-page throttle waiting + /// Number of pages to flush + public FlushCompletionTracker(TaskCompletionSource completionTcs, bool enableThrottling, int count) + { + this.completionTcs = completionTcs; + this.flushSemaphore = enableThrottling ? new SemaphoreSlim(0) : null; + this.count = count; + + if (count == 0) + _ = completionTcs.TrySetResult(true); + } + + /// + /// Complete flush of one page + /// + public void CompleteFlush() + { + _ = (flushSemaphore?.Release()); + if (Interlocked.Decrement(ref count) == 0) + _ = completionTcs.TrySetResult(true); + } + + /// + /// Signal that the flush failed with an exception. + /// + public void SetException(Exception ex) + => _ = completionTcs.TrySetException(ex); + + /// + /// Wait for one page flush to complete. Only valid when throttling is enabled. + /// + public void WaitOneFlush() => flushSemaphore?.Wait(); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/LockType.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/LockType.cs index 042658c07ef..73ec1ba63db 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/LockType.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/LockType.cs @@ -27,10 +27,10 @@ public enum LockType : byte /// /// Interface that must be implemented to participate in keyHash-based locking. /// - public interface ILockableKey + public interface ITransactionalKey { /// - /// The hash code for a specific key, obtained from + /// The hash code for a specific key, obtained from /// public long KeyHash { get; } @@ -40,70 +40,6 @@ public interface ILockableKey public LockType LockType { get; } } - /// - /// A utility class to carry a fixed-length key (blittable or object type) and its assciated info for Locking - /// - /// - public struct FixedLengthLockableKeyStruct : ILockableKey - { - /// - /// The key that is acquiring or releasing a lock - /// - public TKey Key; - - #region ILockableKey - /// - public long KeyHash { get; set; } - - /// - public LockType LockType { get; set; } - #endregion ILockableKey - - /// - /// Constructor - /// - public FixedLengthLockableKeyStruct(TKey key, LockType lockType, ITsavoriteContext context) : this(ref key, lockType, context) { } - - /// - /// Constructor - /// - public FixedLengthLockableKeyStruct(ref TKey key, LockType lockType, ITsavoriteContext context) - { - Key = key; - LockType = lockType; - KeyHash = context.GetKeyHash(ref key); - } - /// - /// Constructor - /// - public FixedLengthLockableKeyStruct(TKey key, long keyHash, LockType lockType, ILockableContext context) : this(ref key, keyHash, lockType, context) { } - - /// - /// Constructor - /// - public FixedLengthLockableKeyStruct(ref TKey key, long keyHash, LockType lockType, ILockableContext context) - { - Key = key; - KeyHash = keyHash; - LockType = lockType; - } - - /// - /// Sort the passed key array for use in - /// and - /// - /// - /// - public static void Sort(FixedLengthLockableKeyStruct[] keys, ILockableContext context) => context.SortKeyHashes>(keys); - - /// - public override string ToString() - { - var hashStr = Utility.GetHashString(KeyHash); - return $"key {Key}, hash {hashStr}, {LockType}"; - } - } - /// /// Lock state of a record /// diff --git a/libs/server/Storage/Session/Common/MemoryUtils.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/MemoryUtils.cs similarity index 76% rename from libs/server/Storage/Session/Common/MemoryUtils.cs rename to libs/storage/Tsavorite/cs/src/core/Utilities/MemoryUtils.cs index 7d76ad3a8ed..045a27d4fbc 100644 --- a/libs/server/Storage/Session/Common/MemoryUtils.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/MemoryUtils.cs @@ -2,9 +2,8 @@ // Licensed under the MIT license. using System; -using Tsavorite.core; -namespace Garnet.server +namespace Tsavorite.core { /// /// Utility class for memory related operations. @@ -44,12 +43,12 @@ public static class MemoryUtils /// .Net object avg. overhead for holding a priority queue entry public const int PriorityQueueEntryOverhead = 48; - internal static long CalculateKeyValueSize(byte[] key, IGarnetObject value) - { - // Round up key size to account for alignment during allocation - // and add up overhead for allocating a byte array - return Utility.RoundUp(key.Length, IntPtr.Size) + ByteArrayOverhead + - value.Size; - } + /// This is but that is a static expression, not a constant + public const int ArrayMaxLength = 0x7FFFFFC7; + + /// Calculate the heap memory size of this + public static long CalculateHeapMemorySize(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => logRecord.CalculateHeapMemorySize(); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/MultiLevelPageArray.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/MultiLevelPageArray.cs new file mode 100644 index 00000000000..d2ebbc152b3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/MultiLevelPageArray.cs @@ -0,0 +1,249 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; + +namespace Tsavorite.core +{ + struct MultiLevelPageArray + { + // TODO: Make MLPA config numbers internally configurable (e.g. smaller log pages need less overhead). Should be able to do this internally + // and not expose another set of public config options. + internal const int InitialBookSizeBits = 2; + internal const int PrimaryClearRetainedChapterSizeBits = InitialBookSizeBits << 1; + internal const int FreeListClearRetainedChapterSizeBits = InitialBookSizeBits; + internal const int ChapterSizeBits = 10; + + internal const int InitialBookSize = 1 << InitialBookSizeBits; + internal const int ChapterSize = 1 << ChapterSizeBits; + internal const int PageIndexMask = (1 << ChapterSizeBits) - 1; + } + + /// + /// This creates a 3-d array of page vectors. This can be envisioned as a book, where the first two dimensions are infrastructure, and the third is where + /// the user-visible allocations are created. + /// + /// The first dimension is the "book", which is a collection of "chapters". + /// The second dimension is the "chapters", which is a collection of pages. + /// The third dimension is the actual pages of data which are returned to the user + /// + /// This structure is chosen so that only the "book" is grown; individual chapters are allocated as a fixed size. This means that + /// getting and clearing items in the chapter does not have to take a latch to prevent a lost update as the array is grown, as + /// would be necessary if there was only a single level of infrastructure (i.e. a growable chapter). + /// + internal class MultiLevelPageArray + { + internal TElement[][] book; + + /// Value of the tail before initialization; start at -1 so Allocate() sets it to 0 + private const int InitialTail = -1; + + /// The next index to be returned; if we are not yet initialized. + internal int tail = InitialTail; + + public bool IsInitialized => book is not null; + + public int Count => tail < 0 ? 0 : tail; // InitialTail is -1, and tail is the next id to return + + public int Allocate() + { + // The first loop ensures the book is initialized. It may be repeated up to three times; if either of the CompareExchanges fails, + // it will be because the desired condition was already set by another thread. + while (tail == InitialTail) + { + // The book may be non-null due to Clear() (e.g. when we wrap around in the log to page 0) or to the newBook allocation below. + // If the book is not null but the tail is InitialTail, we need to set tail to 0 *once* to start allocations. + if (book is not null) + { + _ = Interlocked.CompareExchange(ref tail, 0, InitialTail); + continue; + } + + // Another thread may have allocated and set the book since the last test; if not, try to do so now + if (book is null) + { + // Allocate the book as a two-step process so we don't overwrite another thread's book allocation. Because we can't set both the + // book and tail in a single atomic operation, we set only the book, and loop back up to detect the non-null book and set tail + // instead of setting it here; otherwise, there is a race where multiple threads could set tail to 0 at the two locations. + var newBook = new TElement[MultiLevelPageArray.InitialBookSize][]; + if (Interlocked.CompareExchange(ref book, newBook, null) == null) + continue; + } + _ = Thread.Yield(); + } + + while (true) + { + var originalTail = tail; + var originalChapter = originalTail >> MultiLevelPageArray.ChapterSizeBits; + if ((originalChapter >= book.Length || book[originalChapter] is null) && ((originalTail & MultiLevelPageArray.PageIndexMask) > 0)) + { + // Only the first-page return in a chapter can allocate the chapter and one or more other threads has already incremented tail + // into a new, not-yet-allocated chapter, which means the first one that did so owns the "latch" to allocate this chapter. + // Other threads should exit without incrementing tail; just wait for that owning thread to allocate the new chapter. + _ = Thread.Yield(); // TODO consider SpinWait.SpinOnce() with backoff + continue; + } + + // If we are here, our first test indicated we did not need to allocate a new chapter (but that may have changed), or we are a candidate to + // be the first to allocate it and thus own the new-chapter "latch". Increment tail and we'll return the prior-to-increment value once we have + // done any needed allocation. Because there is a gap between incrementing tail and checking for a null chapter, we need to track whether + // the next chapter is allocated; if not, we should not return the first index in the next chapter (that is done by the chapter-allocating thread). + var nextChapterWasNotAllocated = originalChapter + 1 >= book.Length || book[originalChapter + 1] is null; + var returnTail = Interlocked.Increment(ref tail) - 1; + var returnChapter = returnTail >> MultiLevelPageArray.ChapterSizeBits; + Debug.Assert(returnChapter >= originalChapter, $"newChapter {returnChapter} must not be < originalChapter {originalChapter}"); + + // If returnChapter is allocated and we are not returning trying to return the first page in a newly-allocated chapter (that should be only be + // done by the allocating thread, and we're not it), we can return returnTail. + if (returnChapter < book.Length && book[returnChapter] is not null && (returnChapter == originalChapter || !nextChapterWasNotAllocated || returnTail > 0)) + return returnTail; + + // Multiple threads might have seen the initial "first page in chapter" condition and incremented tail. We only want to stay if we're the + // first; that is, if we are returning the first page in the new chapter. Others threads should exit and wait and the first thread will + // "release the increment" by resetting tail after it's allocated the chapter. + var newPage = returnTail & MultiLevelPageArray.PageIndexMask; + if (newPage > 0) + { + _ = Thread.Yield(); + continue; + } + + // We are allocating the first page on a new chapter so we "own the latch" on this newChapter, but it's (barely) possible that tail was incremented + // so many times it went to a second page. Therefore only try to allocate newChapter if it is the first in the book or the previous chapter is allocated. + if (returnChapter > 0 && book[returnChapter - 1] is null) + { + _ = Thread.Yield(); + continue; + } + + // We still own the latch and are allocating the new chapter, and possibly need to grow the book. If this returns false, tail is reset to the first page + // in the chapter; otherwise it is set to the second and we return the first. These are per tail's "next item to return" definition. + AddChapter(returnChapter, out returnTail); + Debug.Assert(returnTail >= originalTail, $"returnTail {returnTail} must be >= originalTail after AddChapter"); + return returnTail; + } + } + + /// + /// Add a chapter. has been incremented to be the next chapter after the last non-null chapter. + /// + private void AddChapter(int newChapterIndex, out int returnTail) + { + // We should only be here after we have verified that we need to grow the book or allocate the newChapterIndex, and only one thread should + // be here due to the "latch" described above. If we are reusing a book, this should already have passed the "chapter is not null" test + // and we wouldn't be here. + Debug.Assert(newChapterIndex >= book.Length || book[newChapterIndex] is null, $"Trying to allocate an existing chapter {newChapterIndex}"); + var firstPageInChapter = newChapterIndex << MultiLevelPageArray.ChapterSizeBits; + + try + { + // First see if we need to grow the book. + if (newChapterIndex == book.Length) + { + var newBook = new TElement[book.Length * 2][]; + Array.Copy(book, newBook, book.Length); + book = newBook; + } + + // Now allocate the new chapter. + var newChapter = new TElement[MultiLevelPageArray.ChapterSize]; + + // Before setting the new chapter into the book, set tail to the second page in the new chapter as "next to return", and we will return + // the first page after setting the chapter. This ensures that other threads entering Allocate() will see the second page as tail and + // the chapter as null, so will not try to increment tail until the state is consistent between book[newChapterIndex] and tail. + tail = firstPageInChapter + 1; + if (Interlocked.CompareExchange(ref book[newChapterIndex], newChapter, null) != null) + throw new TsavoriteException("Unexpected multiple threads in AddChapter"); + returnTail = firstPageInChapter; + } + catch + { + // Restore tail to the first index in the newChapter. This keeps book[newChapterIndex[0]] as the next page to be returned, and we will + // retry from the beginning on the next Allocate() iteration (on a different thread, as we're re-throwing (probably OOM) on this one). + tail = firstPageInChapter; + throw; + } + } + + public TElement this[int index] + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => Get(index); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set => Set(index, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public TElement Get(int index) + { + Debug.Assert(index < tail, $"Get(): index {index} must be less than tail {tail}"); + var localBook = book; // Temp copy as 'book' may be reallocated while we do this (but the chapter indexing remains unchanged and the chapter remains valid). + + var chapterIndex = index >> MultiLevelPageArray.ChapterSizeBits; + var pageIndex = index & MultiLevelPageArray.PageIndexMask; + Debug.Assert(localBook[chapterIndex] is not null, $"index {index} out of range of chapters {chapterIndex}"); + return localBook[chapterIndex][pageIndex]; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(int index, TElement element) + { + Debug.Assert(index < tail, $"Set(): index {index} must be less than tail {tail}"); + var localBook = book; // Temp copy as 'book' may be reallocated while we do this (but the chapter indexing remains unchanged and the chapter remains valid). + + var chapterIndex = index >> MultiLevelPageArray.ChapterSizeBits; + var pageIndex = index & MultiLevelPageArray.PageIndexMask; + Debug.Assert(localBook[chapterIndex] is not null, $"index {index} out of range of chapters {chapterIndex}"); + localBook[chapterIndex][pageIndex] = element; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear(int retainedChapterCount = 1 << MultiLevelPageArray.PrimaryClearRetainedChapterSizeBits) + { + if (!IsInitialized || tail == 0) + return; + + // Tail is the next item to return, so may be the first item in a chapter that may still be null--or may be past end of book. + var lastChapterIndex = (tail - 1) >> MultiLevelPageArray.ChapterSizeBits; + for (var chapter = 0; chapter <= lastChapterIndex; chapter++) + { + Array.Clear(book[chapter], 0, MultiLevelPageArray.ChapterSize); + if (chapter > retainedChapterCount) + book[chapter] = null; + } + + tail = InitialTail; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear(Action action, int retainedChapterCount = 1 << MultiLevelPageArray.PrimaryClearRetainedChapterSizeBits) + { + if (!IsInitialized || tail == 0) + return; + + // Tail is the next item to return, so may be the first item in a chapter that may still be null--or may be past end of book. + var lastChapterIndex = (tail - 1) >> MultiLevelPageArray.ChapterSizeBits; + var lastPageIndex = (tail - 1) & MultiLevelPageArray.PageIndexMask; + for (var chapter = 0; chapter <= lastChapterIndex; chapter++) + { + var maxPage = chapter < lastChapterIndex ? MultiLevelPageArray.ChapterSize : lastPageIndex; + for (var page = 0; page < maxPage; page++) + { + // Note: 'action' must check for null/default. + action(book[chapter][page]); + book[chapter][page] = default; + } + if (chapter > retainedChapterCount) + book[chapter] = null; + } + tail = InitialTail; + } + + /// + public override string ToString() => $"Tail: {tail}, IsInit: {IsInitialized}, book.Len: {(book is not null ? book.Length : "null")}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/PageAsyncResultTypes.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/PageAsyncResultTypes.cs index 75098b5458c..9bff4c26769 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/PageAsyncResultTypes.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/PageAsyncResultTypes.cs @@ -3,6 +3,9 @@ #define CALLOC +using System; +using System.Diagnostics; +using System.Runtime.InteropServices; using System.Threading; namespace Tsavorite.core @@ -13,86 +16,78 @@ namespace Tsavorite.core /// public sealed class PageAsyncReadResult { + /// Index of the main-log page being read internal long page; - internal long offset; + + /// Recovery device page offset + internal long devicePageOffset; + + /// Context state to be passed through the read operation internal TContext context; + + /// Event to be signaled when the main-log page read is complete internal CountdownEvent handle; - internal SectorAlignedMemory freeBuffer1; - internal SectorAlignedMemory freeBuffer2; + + /// Callback to be called when the main-log page has completed processing; for + /// this means after all Overflow or Objects on the page have been read as well. internal DeviceIOCompletionCallback callback; - internal IDevice objlogDevice; - internal object frame; + + /// The destination pointer being read into. + internal IntPtr destinationPtr; + + /// The cancellation token source, if any, for the Read operation internal CancellationTokenSource cts; - /* Used for iteration */ - internal long resumePtr; - internal long untilPtr; - internal long maxPtr; + /// Read buffers if Reading ObjectAllocator. + public CircularDiskReadBuffer readBuffers; - /// - /// Free - /// + /// Number of bytes read. + public uint numBytesRead; + + /// The max offset on the main log page to iterate records when determining how many bytes in the ObjectLog to read. + internal long maxAddressOffsetOnPage; + + /// If true, we are called from recovery, and should use the non-transient . + internal bool isForRecovery; + + /// + public override string ToString() + => $"page {page}, isRecov {isForRecovery}, devPgOffset {devicePageOffset}, ctx {context}, countdown {handle?.CurrentCount}, destPtr {destinationPtr} (0x{destinationPtr:X}), maxPtr {maxAddressOffsetOnPage}"; + + /// Currently nothing to free. public void Free() { - if (freeBuffer1 != null) - { - freeBuffer1.Return(); - freeBuffer1 = null; - } + } - if (freeBuffer2 != null) - { - freeBuffer2.Return(); - freeBuffer2 = null; - } + /// + public void DisposeHandle() + { + handle?.Dispose(); + handle = null; + readBuffers?.Dispose(); + readBuffers = null; } } /// /// Shared flush completion tracker, when bulk-flushing many pages /// - internal sealed class FlushCompletionTracker + internal enum FlushRequestState : byte { - /// - /// Semaphore to set on flush completion - /// - readonly SemaphoreSlim completedSemaphore; + /// The default; we are here for flush. This + /// includes FoldOver checkpoints + ReadOnly = 0, - /// - /// Semaphore to wait on for flush completion - /// - readonly SemaphoreSlim flushSemaphore; + /// The flush is for , so the object log files have + /// already been written; we must reuse the deserialized object lengths to update the LogRecord's ObjectLogPosition rather than serialize again. + Recovery, - /// - /// Number of pages being flushed - /// - int count; - - /// - /// Create a flush completion tracker - /// - /// Semaphpore to release when all flushes completed - /// Semaphpore to release when each flush completes - /// Number of pages to flush - public FlushCompletionTracker(SemaphoreSlim completedSemaphore, SemaphoreSlim flushSemaphore, int count) - { - this.completedSemaphore = completedSemaphore; - this.flushSemaphore = flushSemaphore; - this.count = count; - } - - /// - /// Complete flush of one page - /// - public void CompleteFlush() - { - flushSemaphore?.Release(); - if (Interlocked.Decrement(ref count) == 0) - completedSemaphore.Release(); - } + /// The flush is for , so we do not hold the epoch + /// initially and thus must check to handle the case where HeadAddress increases out of the range of the flush + Snapshot, - public void WaitOneFlush() - => flushSemaphore?.Wait(); + /// The flush operation did not issue a write, likely because is true and HeadAddress advanced beyond the page + WriteNotIssued } /// @@ -101,44 +96,282 @@ public void WaitOneFlush() /// public sealed class PageAsyncFlushResult { - /// - /// Page - /// + /// The index of the log Page being written public long page; - /// - /// Context - /// + + /// Context object for the callback public TContext context; - /// - /// Count - /// + + /// Flush buffers if flushing ObjectAllocator. + public CircularDiskWriteBuffer flushBuffers; + + /// Count of active pending flush operations; the callback decrements this and when it hits 0, the overall flush operation is complete. public int count; + /// If true, this is a flush of a partial page. internal bool partial; + internal long fromAddress; internal long untilAddress; + + /// Identifes the operation that triggered the flush. + internal FlushRequestState flushRequestState; + + /// The record buffer, passed through the IO process to retain a reference to it so it will not be GC'd before the Flush write completes. internal SectorAlignedMemory freeBuffer1; - internal SectorAlignedMemory freeBuffer2; + + /// The event that is signaled by the callback so any waiting thread knows the IO has completed. internal AutoResetEvent done; + internal FlushCompletionTracker flushCompletionTracker; + /// If this is set then we are using a different objectLog device from that in the allocator, and do not use the allocator's . + internal ObjectLogFilePositionInfo objectLogFilePositionInfo; + + /// + public override string ToString() + { + static string bstr(bool value) => value ? "T" : "F"; + return $"page {page}, isFor {flushRequestState}, ctx {context}, count {count}, partial {bstr(partial)}," + + $" fromAddr {fromAddress} (0x{fromAddress:X}), untilAddr {untilAddress} (0x{untilAddress:X})," + + $" flushCompTrack [{flushCompletionTracker}], circFlushBufs [{flushBuffers}]"; + } + /// - /// Free + /// Release our and if it is zero, clear buffers. /// - public void Free() + /// The decremented count + /// There is currently no AddRef(); count is assigned 1 at creation + internal int Release() { - if (freeBuffer1 != null) + var result = Interlocked.Decrement(ref count); + if (result == 0) { - freeBuffer1.Return(); + freeBuffer1?.Return(); freeBuffer1 = null; + flushCompletionTracker?.CompleteFlush(); + flushCompletionTracker = null; } - if (freeBuffer2 != null) + return result; + } + } + + /// + /// A class to carry callback and context through operations that may chain callbacks. + /// + internal sealed class DiskWriteCallbackContext + { + /// If we had separate Writes for multiple spans of a single array, this is a refcounted wrapper for the ; + /// it is released after the write and if it is the final release, all spans have been written and the GCHandle is freed (and the object unpinned). + public RefCountedPinnedGCHandle refCountedGCHandle { get; private set; } + + /// Separate public Set() call so we ensure it is AddRef'd + /// + public void SetRefCountedHandle(RefCountedPinnedGCHandle refGcHandle) + { + Debug.Assert(!gcHandle.IsAllocated, "Cannot have both GCHandle and RefCountedPinnedGCHandle"); + refCountedGCHandle = refGcHandle; + refCountedGCHandle.AddRef(); + } + + /// If this Write is from a , this keeps its byte[] pinned during the Write. + /// It is freed (and the array unpinned) after the Write. Used instead of for only a single span of the array to avoid a heap allocation. + private readonly GCHandle gcHandle; + + /// The countdown callback for the entire partial flush, including s, external writes, and final sector-aligning write. + private readonly CountdownCallbackAndContext countdownCallbackAndContext; + + /// The countdown event if this write is associated with a . + private CountdownEvent bufferCountdownEvent; + + public override string ToString() + { + static string bstr(bool value) => value ? "T" : "F"; + var countdownString = bufferCountdownEvent is null ? "null" : bufferCountdownEvent.CurrentCount.ToString(); + var cbcString = countdownCallbackAndContext is null ? "null" : countdownCallbackAndContext.ToString(); + return $"refCntGcH [{refCountedGCHandle}], gcH {bstr(gcHandle.IsAllocated)}, countdown {countdownString}, cb&c {cbcString}"; + } + + public DiskWriteCallbackContext(CountdownCallbackAndContext callbackAndContext) + { + countdownCallbackAndContext = callbackAndContext; + callbackAndContext.Increment(); + } + + public DiskWriteCallbackContext(CountdownCallbackAndContext callbackAndContext, RefCountedPinnedGCHandle refGcHandle) : this(callbackAndContext) + => SetRefCountedHandle(refGcHandle); + + public DiskWriteCallbackContext(CountdownCallbackAndContext callbackAndContext, GCHandle gcHandle) : this(callbackAndContext) + => this.gcHandle = gcHandle; + + /// This write is associated with a so we need to signal the countdown event for that buffer when we are done. + public void SetBufferCountdownEvent(CountdownEvent countdownEvent) => bufferCountdownEvent = countdownEvent; + + public long Release() + { + refCountedGCHandle?.Release(); + if (gcHandle.IsAllocated) + gcHandle.Free(); + _ = bufferCountdownEvent?.Signal(); + return countdownCallbackAndContext?.Decrement() ?? 0; + } + } + + /// + /// Hold the callback and context for a refcounted callback and context. Used to ensure global completion of multi-buffer writes (which use a "local" + /// callback) before invoking the external callback. + /// + /// + /// The sequence is illustrated for flushes: + /// + /// Initialize the field to a new instance of this at the start of a partial flush + /// AddRef and Release for each operation (for flushes, there will be two levels of refcount: + /// + /// Per-buffer + /// Globally (within the ), to await the completion of all partial flushes before invoking the external callback. + /// + /// + /// + /// When the count hits zero, if the callback is not null, call it; it will only be set to non-null when we have completed a partial flush. This allows the count to drop to 0 and + /// be increased again throughout the partial flush, as various data spans are written. + /// + internal sealed class CountdownCallbackAndContext + { + /// Original caller's callback + public DeviceIOCompletionCallback callback; + /// Original caller's callback context + public object context; + /// Number of bytes written + private uint numBytes; + /// Number of in-flight operations + internal long count; + + public override string ToString() + { + var callbackString = callback is null ? "null" : callback.ToString(); + var contextString = callback is null ? "null" : context.ToString(); + return $"numBytes {numBytes}, count {count}, callback {callbackString}, context {contextString}"; + } + + public void Set(DeviceIOCompletionCallback callback, object context, uint numBytes) + { + this.callback = callback; + this.context = context; + this.numBytes = numBytes; + } + + internal void Increment() => _ = Interlocked.Increment(ref count); + + internal long Decrement() + { + var remaining = Interlocked.Decrement(ref count); + if (remaining == 0) + callback?.Invoke(errorCode: 0, numBytes, context); + return remaining; + } + } + + /// + /// Hold a and a refcount; free the handle when the refcount reaches 0. Used when multiple sections of the + /// same byte[] are being written, such as when it is split across segments. + /// + internal sealed class RefCountedPinnedGCHandle + { + /// The being held. + internal GCHandle gcHandle; + /// Number of in-flight operations + private long count; + + public override string ToString() + { + static string bstr(bool value) => value ? "T" : "F"; + return $"gcH {bstr(gcHandle.IsAllocated)}, count {count}"; + } + + internal RefCountedPinnedGCHandle(object targetObject, long initialCount) + { + gcHandle = GCHandle.Alloc(targetObject, GCHandleType.Pinned); + count = initialCount; + } + + internal RefCountedPinnedGCHandle(GCHandle gcHandle, long initialCount) + { + this.gcHandle = gcHandle; + count = initialCount; + } + + internal void AddRef() + { + ObjectDisposedException.ThrowIf(count <= 0, $"Uninitialized or final-released {nameof(RefCountedPinnedGCHandle)}"); + _ = Interlocked.Increment(ref count); + } + + internal void Release() + { + ObjectDisposedException.ThrowIf(count <= 0, $"Uninitialized or final-released {nameof(RefCountedPinnedGCHandle)}"); + if (Interlocked.Decrement(ref count) == 0 && gcHandle.IsAllocated) + gcHandle.Free(); + } + + internal object Target + { + get { - freeBuffer2.Return(); - freeBuffer2 = null; + ObjectDisposedException.ThrowIf(count <= 0 || !gcHandle.IsAllocated, $"Uninitialized or final-released {nameof(RefCountedPinnedGCHandle)}"); + return gcHandle.Target; } + } + + internal bool IsAllocated => gcHandle.IsAllocated; + } - flushCompletionTracker?.CompleteFlush(); + /// + /// A class to carry callback and context through operations that may chain callbacks. + /// + internal sealed class DiskReadCallbackContext + { + /// If we had separate Reads directly into multiple spans of a single byte[], such as across segments, this is a refcounted wrapper for the ; + /// it is released after the write and if it is the final release, all spans have been written and the GCHandle is freed (and the object unpinned). + public RefCountedPinnedGCHandle refCountedGCHandle { get; private set; } + + /// Separate public Set() call so we ensure it is AddRef'd + /// + public void SetRefCountedHandle(RefCountedPinnedGCHandle refGcHandle) + { + Debug.Assert(!gcHandle.IsAllocated, "Cannot have both GCHandle and RefCountedPinnedGCHandle"); + refCountedGCHandle = refGcHandle; + refCountedGCHandle.AddRef(); + } + + /// An event that can be waited for; the caller's callback will signal it if non-null. + internal CountdownEvent countdownEvent; + + /// If we had a Read directly into the byte[] of an , this is the that keps it pinned during the Read. + /// After the Read it is freed (and the object unpinned). + public GCHandle gcHandle; + + public override string ToString() + { + static string bstr(bool value) => value ? "T" : "F"; + return $"refCntGcH {refCountedGCHandle}, gcH {bstr(gcHandle.IsAllocated)}, countdown {countdownEvent?.CurrentCount}"; + } + + /// If non-null, this is the target buffer to copy data to (the copy is done by the caller's callback). + public byte[] CopyTarget => (byte[])(gcHandle.IsAllocated ? gcHandle.Target : refCountedGCHandle.Target); + + internal DiskReadCallbackContext(CountdownEvent countdownEvent) => this.countdownEvent = countdownEvent; + + internal DiskReadCallbackContext(CountdownEvent countdownEvent, RefCountedPinnedGCHandle refGcHandle) : this(countdownEvent) + => SetRefCountedHandle(refGcHandle); + + internal DiskReadCallbackContext(CountdownEvent countdownEvent, GCHandle gcHandle) : this(countdownEvent) + => this.gcHandle = gcHandle; + + public void Dispose() + { + if (gcHandle.IsAllocated) + gcHandle.Free(); + _ = (countdownEvent?.Signal()); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/SimpleConcurrentStack.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/SimpleConcurrentStack.cs new file mode 100644 index 00000000000..31141119f2a --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/SimpleConcurrentStack.cs @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Tsavorite.core +{ + /// + /// This is a node in the freelists, implemented as a union of two ints and a long. The long is used for Interlocks. + /// + [StructLayout(LayoutKind.Explicit)] + internal struct SimpleFreeStackNode(int slot, int version) + { + internal const int Nil = -1; + + /// The next free node in the stack, or Empty if this is the last node. + [FieldOffset(0)] + internal int Slot = slot; + + /// The slot in the main elementArray. + [FieldOffset(4)] + internal int Version = version; + + /// The word is used for Interlocked operations, containing and . + [FieldOffset(0)] + internal long word; + + internal bool IsNil => Slot == Nil; + + public override string ToString() => $"Slot {Slot}, Version {Version}, IsNil {IsNil}"; + } + + /// + /// This is a queue containing items that may be ref or value types, but does not call Dispose; if TItem is IDisposable + /// it must be owned/disposed elsewhere. + /// + /// + /// This queue does not use latches or pointers. Instead it uses int indexes into the elementArray, and a version number + /// to avoid the ABA issue. This does mean that each item in the array is a struct containing the item and the node information, + /// so is 8 bytes (2 ints) larger than the item alone; we need to track 'next' indexes explicitly rather than rely on push/pop + /// ordering because CAS contention will alter that order. This space overhead is a tradeoff for avoiding the ABA issue without the + /// overhead of ConcurrentStack allocations or latches. + /// + class SimpleConcurrentStack + { + internal struct ArrayElement + { + internal TItem Item; + internal SimpleFreeStackNode Node; + + public override readonly string ToString() => $"[Node {Node}]; Item {Item}"; + } + + public const int DefaultInitialCapacity = 1024; + + /// The actual stack, as a simple growable vector + internal MultiLevelPageArray elementArray; + + /// + /// This is the head of the chain of stack nodes, which are used to track the stack slots in the elementArray. + /// + internal SimpleFreeStackNode stack; + + /// + /// This is the head of the chain of free nodes, which are used to track the free slots in the elementArray. + /// + internal SimpleFreeStackNode freeNodes; + + public SimpleConcurrentStack() + { + elementArray = new(); + stack = new(SimpleFreeStackNode.Nil, version: 0); + freeNodes = new(SimpleFreeStackNode.Nil, version: 0); + } + + /// + /// This is not named "Count" because our and do not adjust 's Tail. + /// So we support just the high-water mark (mostly for test). + /// + public int MaxCount => elementArray.Count; + + public bool IsEmpty => stack.IsNil; + + /// + /// Public API: Push an item onto the stack. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Push(TItem item) + { + if (GetNodeFromFreeList(out SimpleFreeStackNode node)) + node.Version++; + else + node = new(elementArray.Allocate(), version: 0); + + // This node is going onto the stack so create an ArrayElement with its item set to the passed item and the node's version. + // We'll update the element's slot to the stack head inside the retry loop. + var element = new ArrayElement { Item = item, Node = new(SimpleFreeStackNode.Nil, node.Version) }; + + while (true) + { + // The element's slot is the 'next' pointer; update it to what is currently in 'head' to maintain the chain. + var head = stack; + element.Node.Slot = head.Slot; + elementArray.Set(node.Slot, element); + + if (Interlocked.CompareExchange(ref stack.word, node.word, head.word) == head.word) + return; + _ = Thread.Yield(); + } + } + + /// + /// Public API: Pop an item from the stack. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPop(out TItem item) + { + while (true) + { + var current = stack; + if (current.IsNil) + { + item = default; + return false; + } + + // For an element in elementArray at slot elementSlot, its node.Slot refers to the next slot in the chain; node.Version is for the current slot, + // which is also the version for the node whose .Slot == elementSlot. + var element = elementArray[current.Slot]; + var nextSlot = element.Node.Slot; + var nextVersion = element.Node.IsNil ? 0 : elementArray[nextSlot].Node.Version; + + var next = new SimpleFreeStackNode(nextSlot, nextVersion); + if (Interlocked.CompareExchange(ref stack.word, next.word, current.word) == current.word) + { + item = element.Item; + AddNodeToFreeList(current); + return true; + } + _ = Thread.Yield(); + } + } + + /// Put a node that was popped from the onto the . + void AddNodeToFreeList(SimpleFreeStackNode node) + { + ++node.Version; + + // This node is going onto the freeList so create an ArrayElement with its item set to 'default' and the node's version. + // We'll update the element's slot to the freeList head inside the retry loop. + var element = new ArrayElement { Node = new(SimpleFreeStackNode.Nil, node.Version) }; + + while (true) + { + // The element's slot is the 'next' pointer; update it to what is currently in 'head' to maintain the chain. + var head = freeNodes; + element.Node.Slot = head.Slot; + elementArray.Set(node.Slot, element); + + if (Interlocked.CompareExchange(ref freeNodes.word, node.word, head.word) == head.word) + return; + _ = Thread.Yield(); + } + } + + bool GetNodeFromFreeList(out SimpleFreeStackNode node) + { + while (true) + { + node = freeNodes; + if (node.IsNil) + { + node = default; + return false; + } + + // For elementArray[elementSlot], node.Slot refers to the next slot in the chain; node.Version is for the current slot, which is also + // the version for the node whose .Slot == elementSlot. + var element = elementArray.Get(node.Slot); + var version = element.Node.IsNil ? 0 : elementArray.Get(element.Node.Slot).Node.Version; + + var head = new SimpleFreeStackNode(element.Node.Slot, version); + if (Interlocked.CompareExchange(ref freeNodes.word, head.word, node.word) == node.word) + return true; + _ = Thread.Yield(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Clear(int retainedChapterCount = 1 << MultiLevelPageArray.FreeListClearRetainedChapterSizeBits) + { + elementArray.Clear(retainedChapterCount); + stack = new(SimpleFreeStackNode.Nil, version: 0); + freeNodes = new(SimpleFreeStackNode.Nil, version: 0); + } + + public override string ToString() => $"elements {elementArray.Count}; [stack {stack}]; [freeList {freeNodes}]"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs index 912980a0a8d..3ceccaacaf5 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/Status.cs @@ -94,37 +94,42 @@ internal Status(OperationStatus operationStatus) : this() /// /// Whether a Read or RMW found the key /// - public bool Found => (Record.statusCode & StatusCode.BasicMask) == StatusCode.Found; + public readonly bool Found => (Record.statusCode & StatusCode.BasicMask) == StatusCode.Found; /// /// Whether a Read or RMW did not find the key /// - public bool NotFound => (statusCode & StatusCode.BasicMask) == StatusCode.NotFound; + public readonly bool NotFound => (statusCode & StatusCode.BasicMask) == StatusCode.NotFound; /// /// Whether the operation went pending /// - public bool IsPending => statusCode == StatusCode.Pending; + public readonly bool IsPending => statusCode == StatusCode.Pending; /// /// Whether the operation went pending /// - public bool IsCompleted => !IsPending; + public readonly bool IsCompleted => !IsPending; /// /// Whether the operation is in an error state /// - public bool IsFaulted => statusCode == StatusCode.Error; + public readonly bool IsFaulted => statusCode == StatusCode.Error; /// /// Whether the operation was canceled /// - public bool IsCanceled => statusCode == StatusCode.Canceled; + public readonly bool IsCanceled => statusCode == StatusCode.Canceled; /// /// Whether the operation found an expired record /// - public bool Expired => (statusCode & StatusCode.Expired) == StatusCode.Expired; + public readonly bool IsExpired => (statusCode & StatusCode.Expired) == StatusCode.Expired; + + /// + /// Whether the operation found an expired record + /// + public readonly bool IsWrongType => (statusCode & StatusCode.WrongType) == StatusCode.WrongType; /// /// Whether the operation completed successfully, i.e., it is not pending and did not error out diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/StatusCode.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/StatusCode.cs index 45d584f4d0e..3bdab194296 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/StatusCode.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/StatusCode.cs @@ -18,18 +18,18 @@ internal enum StatusCode : byte /// /// /// - /// Upsert ConcurrentWriter: | + /// Upsert InPlaceWriter: | /// RMW InPlaceUpdater: | /// RMW CopyUpdater: | /// /// If NeedCopyUpdate returns false: /// - /// Delete ConcurrentDeleter: | - /// Read ConcurrentReader: + /// Delete InPlaceDeleter: | + /// Read Reader: /// /// If in immutable region and copying to tail: | /// - /// Read Pending to SingleReader: + /// Read Pending to Reader: /// /// If copying to tail: | /// If copying to readCache: | @@ -44,12 +44,12 @@ internal enum StatusCode : byte /// /// /// - /// Upsert SingleWriter (not found in mutable region): | + /// Upsert InitialWriter (not found in mutable region): | /// RMW InitialUpdater (not found in mutable, immutable, or on-disk regions): | /// /// If NeedInitialUpdate returns false: /// - /// Delete SingleDeleter (not found in mutable region): | + /// Delete InitialDeleter (not found in mutable region): | /// /// NotFound = 0x01, @@ -119,7 +119,14 @@ internal enum StatusCode : byte /// CopiedRecordToReadCache = 0x50, - // unused 0x60, + /// + /// Indicates that an existing record was found but was of the wrong type for the requested operation. + /// + /// + /// See basic codes for details of usage. + /// + WrongType = 0x60, + // unused 0x70, /// diff --git a/libs/storage/Tsavorite/cs/src/core/Utilities/Utility.cs b/libs/storage/Tsavorite/cs/src/core/Utilities/Utility.cs index 940b27195e7..4e71c5a52c0 100644 --- a/libs/storage/Tsavorite/cs/src/core/Utilities/Utility.cs +++ b/libs/storage/Tsavorite/cs/src/core/Utilities/Utility.cs @@ -4,6 +4,7 @@ using System; using System.ComponentModel; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -77,8 +78,6 @@ internal static int NumBitsPreviousPowerOf2(long v, ILogger logger = null) /// /// Previous power of 2 /// - /// - /// internal static long PreviousPowerOf2(long v) { v |= v >> 1; @@ -90,6 +89,11 @@ internal static long PreviousPowerOf2(long v) return v - (v >> 1); } + /// + /// Next power of 2 + /// + internal static long NextPowerOf2(long v) => (long)BitOperations.RoundUpToPowerOf2((nuint)v); + /// /// Pretty print value /// @@ -124,24 +128,84 @@ internal static string PrettySize(long value) return v.ToString() + "B"; } + /// Rounds up to (which must be a power of two) [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool IsReadCache(long address) => (address & Constants.kReadCacheBitMask) != 0; + public static int RoundUp(int value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundUp(int) alignment must be a power of two"); + return (value + (alignment - 1)) & ~(alignment - 1); + } + /// Rounds up to (which must be a power of two) [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static long AbsoluteAddress(long address) => address & ~Constants.kReadCacheBitMask; + public static uint RoundUp(uint value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundUp(uint) alignment must be a power of two"); + return (value + ((uint)alignment - 1)) & ~((uint)alignment - 1); + } - /// Rounds up value to alignment - /// Value to be aligned - /// Align to this - /// Aligned value + /// Rounds up to (which must be a power of two) [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int RoundUp(int value, int alignment) => (value + (alignment - 1)) & ~(alignment - 1); + internal static long RoundUp(long value, long alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundUp(long) alignment must be a power of two"); + return (value + (alignment - 1)) & ~(alignment - 1); + } + /// Rounds up to (which must be a power of two) [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static long RoundUp(long value, int alignment) + internal static ulong RoundUp(ulong value, ulong alignment) { - Debug.Assert(IsPowerOfTwo(alignment), "RoundUp alignment must be a power of two"); - return (value + (alignment - 1)) & ~(alignment - 1); + Debug.Assert(IsPowerOfTwo(alignment), "RoundUp(ulong) alignment must be a power of two"); + return (value + ((uint)alignment - 1)) & ~((uint)alignment - 1); + } + + /// Rounds up to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int RoundDown(int value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundDown(int) alignment must be a power of two"); + return value & ~(alignment - 1); + } + + /// Rounds up to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static uint RoundDown(uint value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundDown(uint) alignment must be a power of two"); + return value & ~((uint)alignment - 1); + } + + /// Rounds up to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static long RoundDown(long value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundDown(long) alignment must be a power of two"); + return value & ~(alignment - 1); + } + + /// Rounds up to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ulong RoundDown(ulong value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "RoundDown(ulong) alignment must be a power of two"); + return value & ~((uint)alignment - 1); + } + + /// Verifies that is aligned to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool IsAligned(long value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "IsAligned(long) alignment must be a power of two"); + return (value & (alignment - 1)) == 0; + } + + /// Verifies that is aligned to (which must be a power of two) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool IsAligned(ulong value, int alignment) + { + Debug.Assert(IsPowerOfTwo(alignment), "IsAligned(ulong) alignment must be a power of two"); + return (value & ((uint)alignment - 1)) == 0; } /// @@ -172,29 +236,36 @@ public static long GetHashCode(long input) } /// - /// Get 64-bit hash code for a byte array + /// Get 64-bit hash code for a byte array. The array does not have to be pinned. /// - /// - /// - /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe long HashBytes(byte* pbString, int len) + public static long HashBytes(ReadOnlySpan byteSpan) { - const long magicno = 40343; - char* pwString = (char*)pbString; - int cbBuf = len / 2; - ulong hashState = (ulong)len; + unsafe + { + fixed (byte* pbString = byteSpan) + { + const long magicno = 40343; - for (int i = 0; i < cbBuf; i++, pwString++) - hashState = magicno * hashState + *pwString; + // Convert to char for faster enumeration (two bytes per iteration) + char* pwString = (char*)pbString; + int len = byteSpan.Length; + int cbBuf = len / 2; + ulong hashState = (ulong)len; - if ((len & 1) > 0) - { - byte* pC = (byte*)pwString; - hashState = magicno * hashState + *pC; - } + for (int i = 0; i < cbBuf; i++, pwString++) + hashState = magicno * hashState + *pwString; - return (long)Rotr64(magicno * hashState, 4); + // If we had an odd number of bytes, get the last byte + if ((len & 1) > 0) + { + byte* pC = (byte*)pwString; + hashState = magicno * hashState + *pC; + } + + return (long)Rotr64(magicno * hashState, 4); + } + } } /// @@ -235,14 +306,22 @@ public static unsafe ulong XorBytes(byte* src, int length) [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static ulong Rotr64(ulong x, int n) => BitOperations.RotateRight(x, n); - /// + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsPowerOfTwo(long x) => BitOperations.IsPow2(x); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsPowerOfTwo(ulong x) => BitOperations.IsPow2(x); + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int GetLogBase2(int x) => BitOperations.Log2((uint)x); + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int GetLogBase2(long x) => BitOperations.Log2((ulong)x); + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int GetLogBase2(ulong value) => BitOperations.Log2(value); @@ -288,7 +367,27 @@ public static bool MonotonicUpdate(ref long variable, long newValue, out long ol do { oldValue = variable; - if (oldValue >= newValue) return false; + if (oldValue >= newValue) + return false; + } while (Interlocked.CompareExchange(ref variable, newValue, oldValue) != oldValue); + return true; + } + + /// + /// Updates the variable to newValue only if the current value is smaller than the new value. + /// + /// The variable to possibly replace + /// The value that replaces the variable if successful + /// The orignal value in the variable + /// if oldValue less than newValue + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool MonotonicUpdate(ref ulong variable, ulong newValue, out ulong oldValue) + { + do + { + oldValue = variable; + if (oldValue >= newValue) + return false; } while (Interlocked.CompareExchange(ref variable, newValue, oldValue) != oldValue); return true; } @@ -306,7 +405,8 @@ public static bool MonotonicUpdate(ref int variable, int newValue, out int oldVa do { oldValue = variable; - if (oldValue >= newValue) return false; + if (oldValue >= newValue) + return false; } while (Interlocked.CompareExchange(ref variable, newValue, oldValue) != oldValue); return true; } @@ -334,14 +434,14 @@ private static async Task SlowWithCancellationAsync(Task task, Cancella var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); using (token.Register(s => ((TaskCompletionSource)s).TrySetResult(true), tcs, useSynchronizationContext)) { - if (task != await Task.WhenAny(task, tcs.Task)) + if (task != await Task.WhenAny(task, tcs.Task).ConfigureAwait(false)) { token.ThrowIfCancellationRequested(); } } // make sure any exceptions in the task get unwrapped and exposed to the caller. - return await task; + return await task.ConfigureAwait(false); } /// @@ -383,5 +483,16 @@ public static string GetCallbackErrorMessage(uint errorCode, uint numBytes, obje [DllImport("libc")] private static extern IntPtr strerror(int errnum); + + /// + /// Should only be called in Debug.Assert or other DEBUG-conditional code + /// + [MethodImpl(MethodImplOptions.NoInlining)] + internal static string GetCurrentMethodName([CallerMemberName] string memberName = "") => memberName; + + /// Throw Tsavorite exception with message. We use a method wrapper so that the caller method can execute inlined. + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + internal static void ThrowTsavoriteException(string message) => throw new TsavoriteException(message); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/BorrowedMemoryOwner.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/BorrowedMemoryOwner.cs new file mode 100644 index 00000000000..79e4886afb3 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/BorrowedMemoryOwner.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers; + +namespace Tsavorite.core +{ + /// + /// Lightweight wrapper around an externally-owned . + /// + /// + /// Use this when you need to expose a region of memory that lives elsewhere (e.g. inside an + /// or another long-lived allocation) through APIs that require an + /// (e.g. ) without copying. + /// is a no-op because this type does not own the underlying allocation. + /// + public sealed class BorrowedMemoryOwner : IMemoryOwner + { + /// + public Memory Memory { get; } + + public BorrowedMemoryOwner(Memory memory) + { + Memory = memory; + } + + /// + public void Dispose() + { + // No-op: the underlying memory is owned by the producer (e.g. Tsavorite log/overflow allocator). + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs index 121a051d53c..19a350b88b8 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/IVariableLengthInput.cs @@ -1,31 +1,49 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Tsavorite.core { /// - /// Interface for variable length Inputs to RMW; only implemented for of . + /// Interface for variable length Inputs to RMW. /// - public interface IVariableLengthInput + public interface IVariableLengthInput { - /// - /// Length of resulting value object when performing RMW modification of value using given input - /// - int GetRMWModifiedValueLength(ref TValue value, ref TInput input); + /// Length of resulting value object when performing RMW modification of value using given input + RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TInput input) + where TSourceLogRecord : ISourceLogRecord; + + /// Initial expected length of value object when populated by RMW using given input + RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; + + /// Length of value object, when populated by Upsert using given value and input + RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; - /// - /// Initial expected length of value object when populated by RMW using given input - /// - /// - /// - int GetRMWInitialValueLength(ref TInput input); + /// Length of value object, when populated by Upsert using given value and input + RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + ; - /// - /// Length of value object, when populated by Upsert using given value and input - /// - /// - /// - /// - int GetUpsertValueLength(ref TValue value, ref TInput input); + /// Length of value object, when populated by Upsert using given log record + RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref TInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSourceLogRecord : ISourceLogRecord; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/PinnedSpanByte.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/PinnedSpanByte.cs new file mode 100644 index 00000000000..8f50554014c --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/PinnedSpanByte.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Tsavorite.core +{ + /// + /// Represents contiguous region of arbitrary pinned memory. + /// + /// + /// SAFETY: This type is used to represent arguments that are assumed to point to pinned memory. + /// + [StructLayout(LayoutKind.Explicit, Size = Size)] + public unsafe struct PinnedSpanByte + { + public const int Size = 12; + + [FieldOffset(0)] + public byte* ptr; + + [FieldOffset(8)] + public int length; + + /// + /// Get and set length of ArgSlice. + /// + public int Length + { + readonly get => length; + set => length = value; + } + + /// Correlates to ReadOnlySpan.IsEmpty + public readonly bool IsEmpty => Length == 0; + + /// + /// Get pointer to the start of the slice + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly byte* ToPointer() => ptr; + + /// + /// Reset the contained Span + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Set(byte* newPtr, int newLength) + { + ptr = newPtr; + length = newLength; + } + + /// + /// Total size of the contained span, including the length prefix. + /// + public readonly int TotalSize => sizeof(int) + length; + + /// + /// Set this as invalid; used by to indicate the should be used. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invalidate() => ptr = null; + + /// + /// If the pointer is null, this PinnedSpanByte is not valid + /// + public readonly bool IsValid => ptr != null; + + /// + /// Defines an implicit conversion to a + /// + public static implicit operator ReadOnlySpan(PinnedSpanByte psb) => psb.ReadOnlySpan; + + /// + /// Get slice as ReadOnlySpan + /// + public readonly ReadOnlySpan ReadOnlySpan => new(ptr, length); + + /// + /// Get slice as ReadOnlySpan + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ReadOnlySpan AsReadOnlySpan(int start) => start <= length ? new(ptr + start, length - start) : throw new ArgumentOutOfRangeException(nameof(start)); + + /// + /// Get slice as ReadOnlySpan + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly ReadOnlySpan AsReadOnlySpan(int start, int len) => ((ulong)(uint)start + (uint)len <= (uint)length) ? new(ptr + start, len) : throw new ArgumentOutOfRangeException($"start {nameof(start)} + len {len} exceeds length {length}"); + + /// + /// Get slice as Span + /// + public readonly Span Span => new(ptr, length); + + /// + /// Get slice as Span + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Span AsSpan(int start) => start <= length ? new(ptr + start, length - start) : throw new ArgumentOutOfRangeException(nameof(start)); + + /// + /// Get slice as Span + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly Span AsSpan(int start, int len) => ((ulong)(uint)start + (uint)len <= (uint)length) ? new(ptr + start, len) : throw new ArgumentOutOfRangeException($"start {nameof(start)} + len {len} exceeds length {length}"); + + /// + /// Reinterprets the pinned memory as a reference to an unmanaged value of type . + /// The length of the slice must exactly match the size of . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public ref T AsRef() where T : unmanaged + { + Debug.Assert(length == Unsafe.SizeOf()); + return ref Unsafe.AsRef((T*)ptr); + } + + /// + /// Copies the contents of this slice into a new array. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly byte[] ToArray() => ReadOnlySpan.ToArray(); + + /// + /// Decodes the contents of this slice as ASCII into a new string. + /// + /// A string ASCII decoded string from the slice. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public override readonly string ToString() => IsValid ? Encoding.ASCII.GetString(ReadOnlySpan) : $", len {Length}"; + + /// + /// Create a from the given . + /// + /// + /// SAFETY: The MUST point to pinned memory. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static PinnedSpanByte FromPinnedSpan(ReadOnlySpan span) => FromPinnedPointer((byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)), span.Length); + + /// + /// Create new ArgSlice from given pointer and length + /// + /// + /// SAFETY: The MUST point to pinned memory. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static PinnedSpanByte FromPinnedPointer(byte* ptr, int length) => new() { ptr = ptr, length = length }; + + /// + /// Create a SpanByte around a pinned memory whose first sizeof(int) bytes are the length (i.e. serialized form). + /// + /// + /// SAFETY: The MUST point to pinned memory. + /// + public static PinnedSpanByte FromLengthPrefixedPinnedPointer(byte* pointer) => new() { ptr = pointer + sizeof(int), length = *(int*)pointer }; + + /// + /// Check for equality to the provided argSlice + /// + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool Equals(PinnedSpanByte argSlice) => argSlice.Span.SequenceEqual(Span); + + /// + /// Copy serialized version to specified memory location + /// + /// + /// SAFETY: The MUST point to pinned memory of at least length. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly void SerializeTo(byte* destination) + { + *(int*)destination = length; + Buffer.MemoryCopy(ptr, destination + sizeof(int), Length, Length); + } + + /// + /// Copy non-serialized version to specified memory location (do not copy the length prefix space) + /// + public readonly void CopyTo(Span destination) => ReadOnlySpan.CopyTo(destination); + + /// + /// Copy non-serialized version to specified (do not copy the length prefix space) + /// + public readonly void CopyTo(ref SpanByteAndMemory dst, MemoryPool memoryPool) => ReadOnlySpan.CopyTo(ref dst, memoryPool); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/RecordFieldInfo.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/RecordFieldInfo.cs new file mode 100644 index 00000000000..0eba16000f1 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/RecordFieldInfo.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +namespace Tsavorite.core +{ + /// + /// Struct for information about fields (Value and optional fields) of a record, to determine required allocation size. + /// + public struct RecordFieldInfo + { + /// + /// The data length of the key for the record. It is immutable unless the record is deleted and revivified. Its behavior varies between the String and Object stores: + /// + /// String store: It is the data length of the Span + /// Object store: It is the data length of the Span (which may or may not Overflow) + /// + /// + public int KeySize; + + /// + /// The data length of the value for the record. Its behavior varies between the String and Object stores: + /// + /// String store: It is the data length of the Span + /// Object store: It is either the data length of the Span (which may or may not Overflow) or if the Value is an Object + /// + /// + public int ValueSize; + + /// There is one byte reserved for the namespace in the , limited to integer values from 1-127. If more are desired, the entire length is here + /// and the namespace is stored in the record immediately after the , just before the Key data bytes. This is immutable for the life of the key + /// (unless the record is deleted and revivified). + public int ExtendedNamespaceSize; + + /// Whether the value was specified to be an object. + public bool ValueIsObject; + + /// Whether the new record will have an ETag. + public bool HasETag { get => eTagSize > 0; set => eTagSize = (byte)(value ? LogRecord.ETagSize : 0); } + internal byte eTagSize; + + /// Whether the new record will have an Expiration. + public bool HasExpiration { get => expirationSize > 0; set => expirationSize = (byte)(value ? LogRecord.ExpirationSize : 0); } + internal byte expirationSize; + + /// for the record - defaults to 0. + public byte RecordType; + + /// + public override string ToString() + => $"KeySize {KeySize}, ValSize {ValueSize}, ValIsObj {ValueIsObject}, HasETag {HasETag}, HasExpir {HasExpiration}, RecType: {RecordType}"; + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/RecordSizeInfo.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/RecordSizeInfo.cs new file mode 100644 index 00000000000..ab6a613dac0 --- /dev/null +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/RecordSizeInfo.cs @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace Tsavorite.core +{ +#pragma warning disable IDE0065 // Misplaced using directive + using static Utility; + + /// + /// Struct for information about the key and the fields and their sizes in a record. + /// + public struct RecordSizeInfo + { + // Bit layout for 'word': + // Bit 0: KeyIsInline + // Bit 1: ValueIsInline + // Bit 2: IsRevivifiedRecord + // Bits 3-5: KeyLengthBytes (max value 4) + // Bits 6-8: RecordLengthBytes (max value 4) + private const int KeyIsInlineBit = 1 << 0; + private const int ValueIsInlineBit = 1 << 1; + private const int IsRevivifiedRecordBit = 1 << 2; + private const int KeyLengthBytesShift = 3; + private const int RecordLengthBytesShift = 6; + private const int LengthBytesMask = 0x7; + + /// Packed field containing KeyIsInline, ValueIsInline, IsRevivifiedRecord, KeyLengthBytes, and RecordLengthBytes. + internal int word; + + /// The value length and whether optional fields are present. + public RecordFieldInfo FieldInfo; + + /// Whether the key was within the inline max key length. Set automatically by Tsavorite based on key size. + public readonly bool KeyIsInline + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (word & KeyIsInlineBit) != 0; + } + + /// Whether the value was within the inline max value length. + public readonly bool ValueIsInline + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (word & ValueIsInlineBit) != 0; + } + + /// Sets to true. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void SetKeyIsInline() => word |= KeyIsInlineBit; + + /// Sets to true. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void SetValueIsInline() => word |= ValueIsInlineBit; + + /// Number of bytes in key length; see . + public int KeyLengthBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + readonly get => (word >> KeyLengthBytesShift) & LengthBytesMask; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set + { + Debug.Assert(value is <= sizeof(int) and > 0, $"KeyLengthBytes value {value} should be the number of bytes needed to store an int value from 1 to int.MaxValue"); + word = (word & ~(LengthBytesMask << KeyLengthBytesShift)) | (value << KeyLengthBytesShift); + } + } + + /// Number of bytes in entire record length; see . + public int RecordLengthBytes + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + readonly get => (word >> RecordLengthBytesShift) & LengthBytesMask; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + set + { + Debug.Assert(value is <= sizeof(int) and > 0, $"RecordLengthBytes value {value} should be the number of bytes needed to store an int value from 1 to int.MaxValue"); + word = (word & ~(LengthBytesMask << RecordLengthBytesShift)) | (value << RecordLengthBytesShift); + } + } + + /// Whether the record allocation returned a revivified record. + internal readonly bool IsRevivifiedRecord + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => (word & IsRevivifiedRecordBit) != 0; + } + + /// Sets to true. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void SetIsRevivifiedRecord() => word |= IsRevivifiedRecordBit; + + /// Whether the value was specified to be an object. + public readonly bool ValueIsObject => FieldInfo.ValueIsObject; + + /// Whether the key is an overflow allocation. + public readonly bool KeyIsOverflow => !KeyIsInline; + + /// Whether the value is an overflow allocation. + public readonly bool ValueIsOverflow => !ValueIsInline && !FieldInfo.ValueIsObject; + + /// Returns the inline length of the key (the amount it will take in the record). + public readonly int InlineKeySize => KeyIsInline ? FieldInfo.KeySize : ObjectIdMap.ObjectIdSize; + + /// Returns the inline length of the value (the amount it will take in the record). + public readonly int InlineValueSize => ValueIsInline ? FieldInfo.ValueSize : ObjectIdMap.ObjectIdSize; + + /// Returns whether both the key and value are inline (no overflow or object). + public readonly bool RecordIsInline => (word & (KeyIsInlineBit | ValueIsInlineBit)) == (KeyIsInlineBit | ValueIsInlineBit); + + /// The max inline value size if this is a record in the string log. + public int MaxInlineValueSize { readonly get; internal set; } + + /// The inline size of the record (in the main log). If Key and/or Value are overflow (or value is Object), + /// then their contribution to inline length is just . + public int ActualInlineRecordSize { readonly get; internal set; } + + /// The inline size of the record rounded up to alignment. + public int AllocatedInlineRecordSize { readonly get; internal set; } + + /// Size to allocate for the 'long' offset into the Object log if this record will have objects or overflow, else 0. + public readonly int ObjectLogPositionSize => RecordIsInline ? 0 : LogRecord.ObjectLogPositionSize; + + /// Size to allocate for all optional fields that will be included; possibly 0. + public readonly int OptionalSize => FieldInfo.eTagSize + FieldInfo.expirationSize + ObjectLogPositionSize; + + /// Whether these values are set (default instances are used for Delete internally, for example). + public readonly bool IsSet => AllocatedInlineRecordSize != 0; + + /// + /// Calculate the Record sizes based on the given and sizes, which are adjusted for inline vs. overflow/object. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void CalculateSizes(int keySize, int valueSize) + { + if (FieldInfo.ExtendedNamespaceSize > sbyte.MaxValue) + ThrowTsavoriteException($"FieldInfo.ExtendedNamespaceSize ({FieldInfo.ExtendedNamespaceSize}) exceeds max allowable ({sbyte.MaxValue})"); + + // Calculate full used record size. Use the full possible RecordLengthBytes initially to reserve space in the record for it; + // then replace it with the exact size needed and update ActualInlineRecordSize. + KeyLengthBytes = RecordDataHeader.GetByteCount(keySize); + const int initialRecordLengthBytes = sizeof(int); + ActualInlineRecordSize = RecordInfo.Size + RecordDataHeader.NumIndicatorBytes + KeyLengthBytes + initialRecordLengthBytes + + FieldInfo.ExtendedNamespaceSize + keySize + valueSize + OptionalSize; + + // Adjust to the actual record length bytes needed (must include roundup). + var allocatedSize = RoundUp(ActualInlineRecordSize, Constants.kRecordAlignment); + RecordLengthBytes = RecordDataHeader.GetByteCount(allocatedSize); + ActualInlineRecordSize -= initialRecordLengthBytes - RecordLengthBytes; + + // Finally, calculate allocated size (record-aligned). Round up again as our subtraction might have knocked us down + // by one kRecordAlignment. This may leave us with one extra byte of RecordLengthBytes if for example ActualInlineRecordSize + // went down from 257 to 255, so recalculate the size. This cannot reduce RecordLengthBytes by more than 1. + AllocatedInlineRecordSize = RoundUp(ActualInlineRecordSize, Constants.kRecordAlignment); + if (AllocatedInlineRecordSize != allocatedSize) + RecordLengthBytes = RecordDataHeader.GetByteCount(AllocatedInlineRecordSize); + } + + /// + /// Called from Upsert or RMW methods for Span Values with the actual data size of the update value; ensures consistency between the Get*FieldInfo methods and the actual update methods. + /// Usually called directly to save the cost of calculating actualDataSize twice (in Get*FieldInfo and the actual update methods). + /// + [Conditional("DEBUG")] + public static void AssertValueDataLength(int dataSize, in RecordSizeInfo sizeInfo) + { + Debug.Assert(sizeInfo.FieldInfo.ValueSize == dataSize, $"Mismatch between expected value size {sizeInfo.FieldInfo.ValueSize} and actual value size {dataSize}"); + } + + /// Called from Upsert or RMW methods with the final record info; ensures consistency between the Get*FieldInfo methods and the actual update methods./// + [Conditional("DEBUG")] + public void AssertOptionalsIfSet(RecordInfo recordInfo, bool checkETag = true, bool checkExpiration = true) + { + if (!IsSet) + return; + if (checkETag) + Debug.Assert(FieldInfo.HasETag == recordInfo.HasETag, $"Mismatch between expected HasETag {FieldInfo.HasETag} and actual ETag {recordInfo.HasETag}"); + if (checkExpiration) + Debug.Assert(FieldInfo.HasExpiration == recordInfo.HasExpiration, $"Mismatch between expected HasExpiration {FieldInfo.HasExpiration} and actual HasExpiration {recordInfo.HasExpiration}"); + } + + /// + public override readonly string ToString() + { + var keyString = KeyIsInline ? "inl" : "ovf"; + var valString = ValueIsInline ? "inl" : (ValueIsObject ? "obj" : "ovf"); + return $"[{FieldInfo}] | Key::{keyString}, Val::{valString}, ActRecSize {ActualInlineRecordSize}, AllocRecSize {AllocatedInlineRecordSize}, OptSize {OptionalSize}"; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs index 856da36c78e..26411e08e93 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByte.cs @@ -3,7 +3,6 @@ using System; using System.Buffers; -using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -11,319 +10,18 @@ namespace Tsavorite.core { /// - /// Represents a pinned variable length byte array that is viewable as a pinned Span<byte> - /// Important: AOF header version needs to be updated if this struct's disk representation changes + /// Span{byte} static utility functions for Span{byte} and ReadOnlySpan{byte}. /// - /// - /// Format: [4-byte (int) length of payload][[optional 8-byte metadata] payload bytes...] - /// First 2 bits of length are used as a mask for properties, so max payload length is 1GB - /// - [StructLayout(LayoutKind.Explicit, Pack = 4)] - public unsafe struct SpanByte + public static unsafe class SpanByte { - // Byte #31 is used to denote unserialized (1) or serialized (0) data - private const int UnserializedBitMask = 1 << 31; - // Byte #30 is used to denote extra metadata present (1) or absent (0) in payload - private const int ExtraMetadataBitMask = 1 << 30; - // Bit #29 used to denote if a namespace is present in payload - private const int NamespaceBitMask = 1 << 29; - // Mask for header - private const int HeaderMask = UnserializedBitMask | ExtraMetadataBitMask | NamespaceBitMask; - - /// - /// Length of the payload - /// - [FieldOffset(0)] - private int length; - - /// - /// Start of payload - /// - [FieldOffset(4)] - private IntPtr payload; - - internal readonly IntPtr Pointer => payload; - - /// - /// Pointer to the beginning of payload, not including metadata if any - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public byte* ToPointer() - { - if (Serialized) - return MetadataSize + (byte*)Unsafe.AsPointer(ref payload); - else - return MetadataSize + (byte*)payload; - } - - /// - /// Pointer to the beginning of payload, including metadata if any - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public byte* ToPointerWithMetadata() - { - if (Serialized) - return (byte*)Unsafe.AsPointer(ref payload); - else - return (byte*)payload; - } - - /// - /// Length of payload, including metadata if any - /// - public int Length - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get => length & ~HeaderMask; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - set { length = (length & HeaderMask) | value; } - } - - /// - /// Length of payload, not including metadata if any - /// - public readonly int LengthWithoutMetadata => (length & ~HeaderMask) - MetadataSize; - - /// - /// Format of structure - /// - public readonly bool Serialized => (length & UnserializedBitMask) == 0; - - /// - /// Total serialized size in bytes, including header and metadata if any - /// - public readonly int TotalSize => sizeof(int) + Length; - - /// - /// Size of metadata header, if any (returns 0, 1, 8, or 9) - /// - public readonly int MetadataSize => ((length & ExtraMetadataBitMask) >> (30 - 3)) + ((length & NamespaceBitMask) >> 29); - - /// - /// Create a around a given pointer and given - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public SpanByte(int length, IntPtr payload) - { - Debug.Assert(length <= ~HeaderMask); - this.length = length | UnserializedBitMask; - this.payload = payload; - } - - /// - /// Extra metadata header - /// - public long ExtraMetadata - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - get - { - if (Serialized) - return MetadataSize > 0 ? *(long*)Unsafe.AsPointer(ref payload) : 0; - else - return MetadataSize > 0 ? *(long*)payload : 0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - set - { - if (value > 0) - { - length |= ExtraMetadataBitMask; - Debug.Assert(Length >= MetadataSize); - if (Serialized) - *(long*)Unsafe.AsPointer(ref payload) = value; - else - *(long*)payload = value; - } - } - } - - /// - /// Mark as having 8-byte metadata in header of payload - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void MarkExtraMetadata() - { - Debug.Assert(Length >= 8); - Debug.Assert((length & NamespaceBitMask) == 0, "Don't use both extension for now"); - length |= ExtraMetadataBitMask; - } - - /// - /// Unmark as having 8-byte metadata in header of payload - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnmarkExtraMetadata() => length &= ~ExtraMetadataBitMask; - - /// - /// Mark as having 1-byte namespace in header of payload - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void MarkNamespace() - { - Debug.Assert(Length >= 1); - Debug.Assert((length & ExtraMetadataBitMask) == 0, "Don't use both extension for now"); - length |= NamespaceBitMask; - } - - /// - /// Unmark as having 1-byte namespace in header of payload - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void UnmarkNamespace() => length &= ~NamespaceBitMask; - - /// - /// Check or set struct as invalid - /// - public bool Invalid - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - readonly get => ((length & UnserializedBitMask) != 0) && payload == IntPtr.Zero; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - set - { - Debug.Assert(value, "Cannot restore an Invalid SpanByte to Valid; must reassign the SpanByte as a full value"); - - // Set the actual length to 0; any metadata is no longer available, and a zero length will cause callers' length checks to go - // through the ConvertToHeap path automatically. Keep the UnserializedBitMask. - length = UnserializedBitMask; - payload = IntPtr.Zero; - } - } - - /// - /// Get Span<byte> for this 's payload (excluding metadata if any) - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpan() - { - if (Serialized) - return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); - else - return new Span(MetadataSize + (byte*)payload, Length - MetadataSize); - } - - /// - /// Get Span<byte> for this 's payload (excluding metadata if any) - /// - /// Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpan(int offset) - { - if (Serialized) - return new Span(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); - else - return new Span(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); - } - /// - /// Get ReadOnlySpan<byte> for this 's payload (excluding metadata if any) - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpan() - { - if (Serialized) - return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload), Length - MetadataSize); - else - return new ReadOnlySpan(MetadataSize + (byte*)payload, Length - MetadataSize); - } - - /// - /// Get ReadOnlySpan<byte> for this 's payload (excluding metadata if any) - /// - /// Parameter to avoid having to call slice when wanting to interact directly with payload skipping ETag at the front of the payload - /// - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpan(int offset) - { - if (Serialized) - return new ReadOnlySpan(MetadataSize + (byte*)Unsafe.AsPointer(ref payload) + offset, Length - MetadataSize - offset); - else - return new ReadOnlySpan(MetadataSize + (byte*)payload + offset, Length - MetadataSize - offset); - } - - /// - /// Get Span<byte> for this 's payload (including metadata if any) - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpanWithMetadata() - { - if (Serialized) - return new Span((byte*)Unsafe.AsPointer(ref payload), Length); - else - return new Span((byte*)payload, Length); - } - - /// - /// Get ReadOnlySpan<byte> for this 's payload (including metadata if any) - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpanWithMetadata() - { - if (Serialized) - return new ReadOnlySpan((byte*)Unsafe.AsPointer(ref payload), Length); - else - return new ReadOnlySpan((byte*)payload, Length); - } - - /// - /// If is in a serialized form, return a non-serialized wrapper that points to the same payload. + /// Create a Span{byte} around a stack variable. /// /// - /// SAFETY: The resulting is safe to heap-copy, as long as the underlying payload remains pinned. + /// SAFETY: The MUST be non-movable, such as on the stack. /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public SpanByte Deserialize() - { - if (!Serialized) return this; - return new SpanByte(Length - MetadataSize, (IntPtr)(MetadataSize + (byte*)Unsafe.AsPointer(ref payload))); - } - - /// - /// Reinterpret a fixed Span<byte> as a serialized . Automatically adds Span length to the first 4 bytes. - /// - public static ref SpanByte Reinterpret(Span span) - { - Debug.Assert(span.Length - sizeof(int) <= ~HeaderMask); - - fixed (byte* ptr = span) - { - *(int*)ptr = span.Length - sizeof(int); - return ref Unsafe.AsRef(ptr); - } - } - - /// - /// Reinterpret a fixed ReadOnlySpan<byte> as a serialized , without adding length header - /// - public static ref SpanByte ReinterpretWithoutLength(ReadOnlySpan span) - { - fixed (byte* ptr = span) - { - return ref Unsafe.AsRef(ptr); - } - } - - /// - /// Reinterpret a fixed pointer as a serialized - /// - public static ref SpanByte Reinterpret(byte* ptr) - { - return ref Unsafe.AsRef(ptr); - } - - /// - /// Reinterpret a fixed ref as a serialized (user needs to write the payload length to the first 4 bytes) - /// - public static ref SpanByte Reinterpret(ref T t) - { - return ref Unsafe.As(ref t); - } + public static Span FromPinnedVariable(ref T stackVar) where T : unmanaged + => new(Unsafe.AsPointer(ref stackVar), Unsafe.SizeOf()); /// /// Create a SpanByte around a pinned memory of given . @@ -331,268 +29,106 @@ public static ref SpanByte Reinterpret(ref T t) /// /// SAFETY: The MUST point to pinned memory. /// - public static SpanByte FromPinnedPointer(byte* pointer, int length) => new(length, (nint)pointer); - - /// - /// Create a SpanByte around a pinned unmanaged struct. - /// - /// - /// SAFETY: The provided unmanaged struct MUST be on the stack or point to pinned memory. - /// - public static SpanByte FromPinnedStruct(T* ptr) where T : unmanaged - => new(Unsafe.SizeOf(), (nint)ptr); - - /// - /// Create a from the given . - /// - /// - /// SAFETY: The MUST point to pinned memory. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static SpanByte FromPinnedSpan(ReadOnlySpan span) - { - return new SpanByte(span.Length, (nint)Unsafe.AsPointer(ref MemoryMarshal.GetReference(span))); - } + public static Span FromPinnedPointer(byte* pointer, int length) => new(pointer, length); /// - /// Create SpanByte around a pinned . + /// Create a SpanByte around a pinned memory whose first sizeof(int) bytes are the length (i.e. serialized form). /// /// - /// SAFETY: The MUST be pinned. + /// SAFETY: The MUST point to pinned memory. /// - public static SpanByte FromPinnedMemory(Memory memory) => FromPinnedSpan(memory.Span); - - /// - /// Convert payload to new byte array - /// - public byte[] ToByteArray() => AsReadOnlySpan().ToArray(); - - /// - /// Convert payload to specified (disposable) memory owner - /// - public (IMemoryOwner memory, int length) ToMemoryOwner(MemoryPool pool) - { - var dst = pool.Rent(Length); - AsReadOnlySpan().CopyTo(dst.Memory.Span); - return (dst, Length); - } - - /// - /// Convert to wrapper - /// - public readonly SpanByteAndMemory ToSpanByteAndMemory() => new(this); - - /// - /// Try to copy to given pre-allocated , checking if space permits at destination - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TryCopyTo(ref SpanByte dst) - { - if (dst.Length < Length) return false; - CopyTo(ref dst); - return true; - } + public static Span FromLengthPrefixedPinnedPointer(byte* pointer) => new(pointer + sizeof(int), *(int*)pointer); - /// - /// Blindly copy to given pre-allocated , assuming sufficient space. - /// Does not change length of destination. - /// + /// Total size, including length prefix, of a Span + /// This must be a methods instead of a property due to extension limitations [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopyTo(ref SpanByte dst, long metadata = 0) - { - dst.UnmarkExtraMetadata(); - dst.ExtraMetadata = metadata == 0 ? ExtraMetadata : metadata; - AsReadOnlySpan().CopyTo(dst.AsSpan()); - } + public static int TotalSize(this ReadOnlySpan span) => sizeof(int) + span.Length; - /// - /// Try to copy to given pre-allocated , checking if space permits at destination - /// - /// The target of the copy - /// Optional metadata to add to the destination - /// The size available at the destination (e.g. dst.TotalSize or the log-space Value allocation size) + /// Total size, including length prefix, of a Span + /// This must be a methods instead of a property due to extension limitations [MethodImpl(MethodImplOptions.AggressiveInlining)] - public bool TrySafeCopyTo(ref SpanByte dst, int fullDestSize, long metadata = 0) - { - // If the incoming caller wants to addMetadata and the destination does not already have metadata, the new length needs to account for it. - var addMetadata = metadata > 0 && MetadataSize == 0; - - var newTotalSize = addMetadata ? TotalSize + sizeof(long) : TotalSize; - if (fullDestSize < newTotalSize) - return false; - - var newLength = addMetadata ? Length + sizeof(long) : Length; - dst.ShrinkSerializedLength(newLength); - // Note: If dst is shorter than src we have already verified there is enough extra value space to grow dst to store src. - dst.Length = newLength; - CopyTo(ref dst, metadata); + public static int TotalSize(this Span span) => sizeof(int) + span.Length; - return true; - } - - /// - /// Shrink the length header of the in-place allocated buffer on - /// Tsavorite hybrid log, pointed to by the given . - /// Zeroes out the extra space to retain log scan correctness. - /// - /// New length of payload (including metadata) + /// Copy to given , using the Span{byte} if possible, else allocating from [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void ShrinkSerializedLength(int newLength) - { - // Zero-fill extra space - needed so log scan does not see spurious data - *before* setting length to 0. - if (newLength < Length) - { - Unsafe.InitBlockUnaligned(ToPointerWithMetadata() + newLength, 0, (uint)(Length - newLength)); - Length = newLength; - } - } - - /// - /// Utility to zero out an arbitrary span of bytes. - /// One use is to zero extra space after in-place update shrinks a value, to retain log scan correctness. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Clear(byte* pointer, int length) => new Span(pointer, length).Clear(); - - /// - /// Copy to given (only payload copied to actual span/memory) - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopyTo(ref SpanByteAndMemory dst, MemoryPool memoryPool) + public static void CopyTo(this ReadOnlySpan src, ref SpanByteAndMemory dst, MemoryPool memoryPool) { if (dst.IsSpanByte) { - if (dst.Length >= Length) + if (dst.Length >= src.Length) { - dst.Length = Length; - AsReadOnlySpan().CopyTo(dst.SpanByte.AsSpan()); + dst.Length = src.Length; + src.CopyTo(dst.SpanByte.Span); return; } dst.ConvertToHeap(); } - dst.Memory = memoryPool.Rent(Length); - dst.Length = Length; - AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span); + dst.Memory = memoryPool.Rent(src.Length); + dst.Length = src.Length; + src.CopyTo(dst.MemorySpan); } - /// - /// Copy to given (only payload copied to actual span/memory) - /// + /// Copy to given , using the Span{byte} if possible, else allocating from [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopySliceTo(int sliceLength, ref SpanByteAndMemory dst, MemoryPool memoryPool) - { - if (dst.IsSpanByte) - { - if (dst.Length >= Length) - { - dst.Length = Length; - AsReadOnlySpan().Slice(0, sliceLength).CopyTo(dst.SpanByte.AsSpan()); - return; - } - dst.ConvertToHeap(); - } - - dst.Memory = memoryPool.Rent(Length); - dst.Length = Length; - AsReadOnlySpan().Slice(0, sliceLength).CopyTo(dst.Memory.Memory.Span); - } + public static void CopyTo(this Span src, ref SpanByteAndMemory dst, MemoryPool memoryPool) + => ((ReadOnlySpan)src).CopyTo(ref dst, memoryPool); /// - /// Copy to given (header and payload copied to actual span/memory) + /// Unchecked Unsafe cast to a different type; for speed, it does not do the checking for "contains references" etc. that + /// does. /// - public void CopyWithHeaderTo(ref SpanByteAndMemory dst, MemoryPool memoryPool) - { - if (dst.IsSpanByte) - { - if (dst.Length >= TotalSize) - { - dst.Length = TotalSize; - var span = dst.SpanByte.AsSpan(); - fixed (byte* ptr = span) - *(int*)ptr = Length; - dst.SpanByte.ExtraMetadata = ExtraMetadata; - - AsReadOnlySpan().CopyTo(span.Slice(sizeof(int) + MetadataSize)); - return; - } - dst.ConvertToHeap(); - } - - dst.Memory = memoryPool.Rent(TotalSize); - dst.Length = TotalSize; - fixed (byte* ptr = dst.Memory.Memory.Span) - *(int*)ptr = Length; - dst.SpanByte.ExtraMetadata = ExtraMetadata; - AsReadOnlySpan().CopyTo(dst.Memory.Memory.Span.Slice(sizeof(int) + MetadataSize)); - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static ReadOnlySpan UncheckedCast(this ReadOnlySpan src) + => MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(src)), src.Length / Unsafe.SizeOf()); /// /// Copy serialized version to specified memory location /// + /// + /// SAFETY: The MUST point to pinned memory of at least source.TotalSize(). + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopyTo(Span buffer) + public static void SerializeTo(this ReadOnlySpan source, byte* destination, int destinationSize) { - fixed (byte* ptr = buffer) - CopyTo(ptr); + *(int*)destination = source.Length; + source.CopyTo(new Span(destination + sizeof(int), destinationSize - sizeof(int))); } /// /// Copy serialized version to specified memory location /// + /// + /// SAFETY: The MUST point to pinned memory of at least source.TotalSize(). + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopyTo(byte* destination) + public static void SerializeTo(this ReadOnlySpan source, Span destination) { - if (Serialized) - { - *(int*)destination = length; - Buffer.MemoryCopy(Unsafe.AsPointer(ref payload), destination + sizeof(int), Length, Length); - } - else - { - *(int*)destination = length & ~UnserializedBitMask; - Buffer.MemoryCopy((void*)payload, destination + sizeof(int), Length, Length); - } + if (destination.Length < source.Length + sizeof(int)) + throw new ArgumentException($"Destination length {destination.Length} is less than source length {source.Length} + sizeof(int)"); + Unsafe.As(ref destination[0]) = source.Length; + source.CopyTo(destination.Slice(sizeof(int))); } - /// - /// Gets an Etag from the payload of the SpanByte, caller should make sure the SpanByte has an Etag for the record by checking RecordInfo - /// + /// Length-limited string representation of a Span [MethodImpl(MethodImplOptions.AggressiveInlining)] - public long GetEtagInPayload() => *(long*)this.ToPointer(); - - /// - /// Gets an Etag from the payload of the SpanByte, caller should make sure the SpanByte has an Etag for the record by checking RecordInfo - /// - /// The Etag value to set - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetEtagInPayload(long etag) => *(long*)this.ToPointer() = etag; - - /// - /// Gets a namespace from the payload of the SpanByte, caller should make sure the SpanByte has a namespace for the record by checking RecordInfo - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public byte GetNamespaceInPayload() => *(byte*)this.ToPointerWithMetadata(); - - /// - /// Gets a namespace from the payload of the SpanByte, caller should make sure the SpanByte has a namespace for the record by checking RecordInfo - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void SetNamespaceInPayload(byte ns) => *(byte*)this.ToPointerWithMetadata() = ns; - - /// - public override string ToString() + public static string ToShortString(this ReadOnlySpan span, int maxLen = 20) { - if (Invalid) - return "Invalid"; - var bytes = AsSpan(); - var len = Math.Min(Length, bytes.Length); - StringBuilder sb = new($"len: {Length}, mdLen: {MetadataSize}, isSer {Serialized}, "); + var len = Math.Min(span.Length, maxLen); + StringBuilder sb = new(); for (var ii = 0; ii < len; ++ii) - sb.Append(bytes[ii].ToString("x2")); - if (bytes.Length > len) - sb.Append("..."); + { + if (ii > 0 && ii % 4 == 0) + _ = sb.Append(' '); + _ = sb.Append(span[ii].ToString("x2")); + } + _ = sb.Append(span.Length > maxLen ? '+' : '~'); return sb.ToString(); } + + /// Length-limited string representation of a Span + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string ToShortString(this Span span, int maxLen = 20) + => ToShortString((ReadOnlySpan)span, maxLen); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteAndMemory.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteAndMemory.cs index cf6a1c5c9d0..a9819fc3e1d 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteAndMemory.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteAndMemory.cs @@ -9,14 +9,14 @@ namespace Tsavorite.core { /// - /// Output that encapsulates sync stack output (via ) and async heap output (via IMemoryOwner) + /// Output that encapsulates sync stack output (via ) and async heap output (via IMemoryOwner) /// - public unsafe struct SpanByteAndMemory + public unsafe struct SpanByteAndMemory : IDisposable { /// /// Stack output as /// - public SpanByte SpanByte; + public PinnedSpanByte SpanByte; /// /// Heap output as IMemoryOwner @@ -26,22 +26,12 @@ public unsafe struct SpanByteAndMemory /// /// Constructor using given /// - public SpanByteAndMemory(SpanByte spanByte) + public SpanByteAndMemory(PinnedSpanByte spanByte) { - if (spanByte.Serialized) throw new Exception("Cannot create new SpanByteAndMemory using serialized SpanByte"); SpanByte = spanByte; Memory = default; } - /// - /// Constructor using at given pinned , of given - /// - public SpanByteAndMemory(void* pointer, int length) - { - SpanByte = new SpanByte(length, (IntPtr)pointer); - Memory = default; - } - /// /// Get length /// @@ -54,25 +44,25 @@ public int Length /// /// Is it allocated as (on stack)? /// - public readonly bool IsSpanByte => !SpanByte.Invalid; + public readonly bool IsSpanByte => SpanByte.IsValid; /// /// Constructor using given IMemoryOwner /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public SpanByteAndMemory(IMemoryOwner memory) { - SpanByte = default; - SpanByte.Invalid = true; + SpanByte.Invalidate(); Memory = memory; } /// /// Constructor using given IMemoryOwner and length /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public SpanByteAndMemory(IMemoryOwner memory, int length) { - SpanByte = default; - SpanByte.Invalid = true; + SpanByte.Invalidate(); Memory = memory; SpanByte.Length = length; } @@ -80,45 +70,57 @@ public SpanByteAndMemory(IMemoryOwner memory, int length) /// /// As a span of the contained data. Use this when you haven't tested . /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ReadOnlySpan AsReadOnlySpan() => IsSpanByte ? SpanByte.AsReadOnlySpan() : Memory.Memory.Span.Slice(0, Length); + /// + /// SAFETY: This returns a null pointer in the Span if ! and is null; + /// it is the caller's responsibility to check the length and allocate if necessary. + /// + public ReadOnlySpan ReadOnlySpan + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => IsSpanByte + ? SpanByte.ReadOnlySpan + : (Memory != null ? Memory.Memory.Span.Slice(0, Length) : new(null, 0)); + } /// /// As a span of the contained data. Use this when you haven't tested . /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Span AsSpan() => IsSpanByte ? SpanByte.AsSpan() : Memory.Memory.Span.Slice(0, Length); + /// + /// SAFETY: This returns a null pointer in the Span if ! and is null; + /// it is the caller's responsibility to check the length and allocate if necessary. + /// + public Span Span + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => IsSpanByte + ? SpanByte.Span + : (Memory != null ? MemorySpan.Slice(0, Length) : new(null, 0)); + } /// - /// As a span of the contained data. Use this when you have already tested . + /// As a ReadOnlySpan of the contained data. Use this when you have already tested . /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public readonly ReadOnlySpan AsMemoryReadOnlySpan() + public readonly ReadOnlySpan MemoryReadOnlySpan { - Debug.Assert(!IsSpanByte, "Cannot call AsMemoryReadOnlySpan when IsSpanByte"); - return Memory.Memory.Span.Slice(0, Length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + Debug.Assert(!IsSpanByte, "Cannot call MemoryReadOnlySpan when IsSpanByte"); + return Memory.Memory.Span.Slice(0, Length); + } } /// - /// Copy from the passed ReadOnlySpan{byte}. Use this when you have not tested . + /// As a Span of the contained data. Use this when you have already tested . /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void CopyFrom(ReadOnlySpan srcSpan, MemoryPool memoryPool) + public readonly Span MemorySpan { - if (IsSpanByte) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get { - if (srcSpan.Length < Length) - { - srcSpan.CopyTo(SpanByte.AsSpan()); - Length = srcSpan.Length; - return; - } - ConvertToHeap(); + Debug.Assert(!IsSpanByte, "Cannot call MemoryReadOnlySpan when IsSpanByte"); + return Memory.Memory.Span.Slice(0, Length); } - - Length = srcSpan.Length; - Memory = memoryPool.Rent(srcSpan.Length); - srcSpan.CopyTo(Memory.Memory.Span); } /// @@ -127,11 +129,59 @@ public void CopyFrom(ReadOnlySpan srcSpan, MemoryPool memoryPool) /// /// SAFETY: The MUST point to pinned memory. /// - public static SpanByteAndMemory FromPinnedSpan(ReadOnlySpan span) => new(SpanByte.FromPinnedSpan(span)); + public static SpanByteAndMemory FromPinnedSpan(ReadOnlySpan span) => new() { SpanByte = PinnedSpanByte.FromPinnedSpan(span), Memory = default }; + + /// + /// Create a from a given pinned , of given + /// + /// + /// SAFETY: The MUST point to pinned memory. + /// + public static SpanByteAndMemory FromPinnedPointer(byte* pointer, int length) => new() { SpanByte = PinnedSpanByte.FromPinnedPointer(pointer, length), Memory = default }; /// /// Convert to be used on heap (IMemoryOwner) /// - public void ConvertToHeap() { SpanByte.Invalid = true; } + public void ConvertToHeap() { SpanByte.Invalidate(); } + + /// + /// Ensure the required size is available in this structure via the Span or the Memory. + /// + public void EnsureHeapMemorySize(int size, MemoryPool memoryPool = null) + { + if (memoryPool is null) + memoryPool = MemoryPool.Shared; + + // In case it is still SpanByte, we need to convert it to heap. This should only be done when the SpanByte is too small. + Debug.Assert(!IsSpanByte || SpanByte.Length < size, $"SpanByte Length of {SpanByte.Length} is sufficient for size of {size}, so this calling path should have used the SpanByte."); + ConvertToHeap(); + + SpanByte.Length = 0; + if (Memory is null) + { + Memory = memoryPool.Rent(size); + SpanByte.Length = size; + return; + } + + if (Memory.Memory.Length >= size) + { + SpanByte.Length = size; + return; + } + + // We have a Memory that is too small, so we need to release it and allocate a new one. + Memory.Dispose(); + Memory = null; // In case the following throws OOM + Memory = memoryPool.Rent(size); + SpanByte.Length = size; + } + + public void Dispose() + { + var memory = Memory; + Memory = null; + memory?.Dispose(); + } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs index 6ab7bbf1d1b..59d0dcb23bd 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteComparer.cs @@ -7,9 +7,9 @@ namespace Tsavorite.core { /// - /// Equality comparer for + /// Equality comparer for /// - public struct SpanByteComparer : IKeyComparer + public struct SpanByteComparer : IKeyComparer { /// /// The default instance. @@ -18,37 +18,37 @@ public struct SpanByteComparer : IKeyComparer public static readonly SpanByteComparer Instance = new(); /// - public readonly unsafe long GetHashCode64(ref SpanByte spanByte) => StaticGetHashCode64(ref spanByte); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => StaticGetHashCode64(key.KeyBytes); /// /// Get 64-bit hash code /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe long StaticGetHashCode64(ref SpanByte spanByte) - { - if (spanByte.Serialized) - { - byte* ptr = (byte*)Unsafe.AsPointer(ref spanByte); - return Utility.HashBytes(ptr + sizeof(int), spanByte.Length); - } - else - { - byte* ptr = (byte*)spanByte.Pointer; - return Utility.HashBytes(ptr, spanByte.Length); - } - } + public static long StaticGetHashCode64(ReadOnlySpan key) => Utility.HashBytes(key); /// - public readonly unsafe bool Equals(ref SpanByte k1, ref SpanByte k2) => StaticEquals(ref k1, ref k2); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => StaticEquals(k1.KeyBytes, k2.KeyBytes); /// /// Equality comparison /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool StaticEquals(ref SpanByte k1, ref SpanByte k2) - { - return k1.AsReadOnlySpanWithMetadata().SequenceEqual(k2.AsReadOnlySpanWithMetadata()) - && (k1.MetadataSize == k2.MetadataSize); - } + public static bool StaticEquals(ReadOnlySpan k1, ReadOnlySpan k2) => k1.SequenceEqual(k2); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs index f647d9297e5..212ebae1d0b 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteFunctions.cs @@ -1,15 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Buffers; -using System.Runtime.CompilerServices; namespace Tsavorite.core { /// - /// Callback functions for Key, Value, Input; Output; and specified + /// Callback functions for Value and Input; Output; and specified /// - public class SpanByteFunctions : SpanByteFunctions + public class SpanByteFunctions : SessionFunctionsBase { protected readonly MemoryPool memoryPool; @@ -23,113 +23,33 @@ public SpanByteFunctions(MemoryPool memoryPool = default) } /// - public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref ReadInfo readInfo) { - value.CopyTo(ref dst, memoryPool); + srcLogRecord.ValueSpan.CopyTo(ref output, memoryPool); return true; } - /// - public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - value.CopyTo(ref dst, memoryPool); - return true; - } + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = input.Length }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref PinnedSpanByte input) + // TODO: Namespaces! + => new() { KeySize = key.KeyBytes.Length, ValueSize = input.Length }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref PinnedSpanByte input) + // TODO: Namespaces! + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref PinnedSpanByte input) + // TODO: Namespaces! + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; /// - public override void ConvertOutputToHeap(ref SpanByte input, ref SpanByteAndMemory output) + public override void ConvertOutputToHeap(ref PinnedSpanByte input, ref SpanByteAndMemory output) { // Currently the default is a no-op; the derived class inspects 'input' to decide whether to ConvertToHeap(). //output.ConvertToHeap(); } } - - /// - /// Callback functions for key, value; specified , , and - /// - public class SpanByteFunctions : SessionFunctionsBase - { - /// - public override bool SingleWriter(ref SpanByte key, ref TInput input, ref SpanByte src, ref SpanByte dst, ref TOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo); - - /// - public override bool ConcurrentWriter(ref SpanByte key, ref TInput input, ref SpanByte src, ref SpanByte dst, ref TOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - => DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo); - - /// - /// Utility function for copying, Upsert version. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool DoSafeCopy(ref SpanByte src, ref SpanByte dst, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo, long metadata = 0) - { - // First get the full record length and clear it from the extra value space (if there is any). - // This ensures all bytes after the used value space are 0, which retains log-scan correctness. - - // For non-in-place operations, the new record may have been revivified, so standard copying procedure must be done; - // For SpanByte we don't implement DisposeForRevivification, so any previous value is still there, and thus we must - // zero unused value space to ensure log-scan correctness, just like in in-place updates. - - // IMPORTANT: usedValueLength and fullValueLength use .TotalSize, not .Length, to account for the leading "Length" int. - upsertInfo.ClearExtraValueLength(ref recordInfo, ref dst, dst.TotalSize); - - // We want to set the used and extra lengths and Filler whether we succeed (to the new length) or fail (to the original length). - var result = src.TrySafeCopyTo(ref dst, upsertInfo.FullValueLength, metadata); - upsertInfo.SetUsedValueLength(ref recordInfo, ref dst, dst.TotalSize); - return result; - } - - /// - /// Utility function for copying, RMW version. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool DoSafeCopy(ref SpanByte src, ref SpanByte dst, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - // See comments in upsertInfo overload of this function. - rmwInfo.ClearExtraValueLength(ref recordInfo, ref dst, dst.TotalSize); - var result = src.TrySafeCopyTo(ref dst, rmwInfo.FullValueLength); - rmwInfo.SetUsedValueLength(ref recordInfo, ref dst, dst.TotalSize); - return result; - } - - /// - /// Avoids the "value = default" for added tombstone record, which do not have space for the payload - public override bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => true; - } - - /// - /// Callback functions for key, value, input; specified and - /// - public class SpanByteFunctions : SpanByteFunctions - { - /// - public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => DoSafeCopy(ref input, ref value, ref rmwInfo, ref recordInfo); - - /// - public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => DoSafeCopy(ref oldValue, ref newValue, ref rmwInfo, ref recordInfo); - - /// - // The default implementation of IPU simply writes input to destination, if there is space - public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref TOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => DoSafeCopy(ref input, ref value, ref rmwInfo, ref recordInfo); - - /// - /// Length of resulting object when doing RMW with given value and input. Here we set the length - /// to the max of input and old value lengths. You can provide a custom implementation for other cases. - /// - public override int GetRMWModifiedValueLength(ref SpanByte t, ref SpanByte input) - => sizeof(int) + (t.Length > input.Length ? t.Length : input.Length); - - /// - public override int GetRMWInitialValueLength(ref SpanByte input) => input.TotalSize; - - /// - /// Length of resulting object when doing Upsert with given value and input. Here we set the length to the - /// length of the provided value, ignoring input. You can provide a custom implementation for other cases. - /// - public override int GetUpsertValueLength(ref SpanByte t, ref SpanByte input) - => t.TotalSize; - } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteHeapContainer.cs b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteHeapContainer.cs index 9220411a89c..295c5baaa08 100644 --- a/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteHeapContainer.cs +++ b/libs/storage/Tsavorite/cs/src/core/VarLen/SpanByteHeapContainer.cs @@ -1,25 +1,34 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using System.Runtime.CompilerServices; +using System; namespace Tsavorite.core { /// /// Heap container for structs /// - internal sealed class SpanByteHeapContainer : IHeapContainer + public sealed class SpanByteHeapContainer : IHeapContainer { readonly SectorAlignedMemory mem; + PinnedSpanByte pinnedSpanByte; - public unsafe SpanByteHeapContainer(ref SpanByte obj, SectorAlignedBufferPool pool) + public unsafe SpanByteHeapContainer(ReadOnlySpan item, SectorAlignedBufferPool pool) { - mem = pool.Get(obj.TotalSize); - obj.CopyTo(mem.GetValidPointer()); + if (item.Length == 0) + { + pinnedSpanByte = default; + return; + } + + var size = item.TotalSize(); + mem = pool.Get(size); + item.SerializeTo(mem.GetValidPointer(), size); + pinnedSpanByte = PinnedSpanByte.FromLengthPrefixedPinnedPointer(mem.GetValidPointer()); } - public unsafe ref SpanByte Get() => ref Unsafe.AsRef(mem.GetValidPointer()); + public ref PinnedSpanByte Get() => ref pinnedSpanByte; - public void Dispose() => mem.Return(); + public void Dispose() => mem?.Return(); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureCheckpointNamingScheme.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureCheckpointNamingScheme.cs index 2f91419590d..3743c152bfe 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureCheckpointNamingScheme.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureCheckpointNamingScheme.cs @@ -32,9 +32,6 @@ public AzureCheckpointNamingScheme(string baseName = "") public FileDescriptor LogSnapshot(Guid token) => new(string.Join('/', LogCheckpointBasePath, token.ToString()), "snapshot.dat"); /// public FileDescriptor ObjectLogSnapshot(Guid token) => new(string.Join('/', LogCheckpointBasePath, token.ToString()), "snapshot.obj.dat"); - /// - public FileDescriptor DeltaLog(Guid token) => new(string.Join('/', LogCheckpointBasePath, token.ToString()), "delta.dat"); - /// public FileDescriptor IndexCheckpointBase(Guid token) => new(string.Join('/', IndexCheckpointBasePath, token.ToString()), null); /// diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageDevice.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageDevice.cs index 42d20a41305..67b4b952c17 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageDevice.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/AzureStorageDevice.cs @@ -28,7 +28,7 @@ public class AzureStorageDevice : StorageDeviceBase readonly ConcurrentDictionary pendingReadWriteOperations; readonly ConcurrentDictionary pendingRemoveOperations; readonly Timer hangCheckTimer; - readonly SemaphoreSlim singleWriterSemaphore; + readonly SemaphoreSlim initialWriterSemaphore; readonly TimeSpan limit; readonly bool localBlobManager; @@ -53,7 +53,7 @@ struct RemoveRequestInfo public DateTime TimeStamp; } - SemaphoreSlim SingleWriterSemaphore => singleWriterSemaphore; + SemaphoreSlim InitialWriterSemaphore => initialWriterSemaphore; internal IStorageErrorHandler StorageErrorHandler { get; private set; } @@ -106,7 +106,7 @@ public AzureStorageDevice(string connectionString, string containerName, string StorageErrorHandler.Token.Register(CancelAllRequests); this.underLease = underLease; hangCheckTimer = new Timer(DetectHangs, null, 0, 20000); - singleWriterSemaphore = underLease ? new SemaphoreSlim(1) : null; + initialWriterSemaphore = underLease ? new SemaphoreSlim(1) : null; limit = TimeSpan.FromSeconds(90); StartAsync().Wait(); @@ -141,7 +141,7 @@ internal AzureStorageDevice(string blobName, BlobUtilsV12.BlobDirectory pageBlob StorageErrorHandler.Token.Register(CancelAllRequests); this.underLease = underLease; hangCheckTimer = new Timer(DetectHangs, null, 0, 20000); - singleWriterSemaphore = underLease ? new SemaphoreSlim(1) : null; + initialWriterSemaphore = underLease ? new SemaphoreSlim(1) : null; limit = TimeSpan.FromSeconds(90); StartAsync().Wait(); @@ -197,7 +197,7 @@ await BlobManager.PerformWithRetriesAsync( pageResults = page.Values; continuationToken = page.ContinuationToken; return page.Values.Count; // not accurate, in terms of bytes, but still useful for tracing purposes - }); + }).ConfigureAwait(false); foreach (var item in pageResults) { @@ -217,7 +217,7 @@ await BlobManager.PerformWithRetriesAsync( while (!string.IsNullOrEmpty(continuationToken)); // make sure we did not lose the lease while iterating to find the blobs - await BlobManager.ConfirmLeaseIsGoodForAWhileAsync(); + await BlobManager.ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false); StorageErrorHandler.Token.ThrowIfCancellationRequested(); @@ -319,7 +319,7 @@ public override void Dispose() BlobManager.StopAsync().Wait(); hangCheckTimer.Dispose(); - singleWriterSemaphore?.Dispose(); + initialWriterSemaphore?.Dispose(); // Unlike in LocalStorageDevice, we explicitly remove all page blobs if the deleteOnClose flag is set, instead of relying on the operating system // to delete files after the end of our process. This leads to potential problems if multiple instances are sharing the same underlying page blobs. @@ -384,7 +384,7 @@ public override void RemoveSegmentAsync(int segment, AsyncCallback callback, IAs async (numAttempts) => { var client = (numAttempts > 1) ? entry.PageBlob.Default : entry.PageBlob.Aggressive; - await client.DeleteAsync(cancellationToken: StorageErrorHandler.Token); + await client.DeleteAsync(cancellationToken: StorageErrorHandler.Token).ConfigureAwait(false); return 1; }); } @@ -419,7 +419,7 @@ Task Delete(BlobEntry entry) async (numAttempts) => { var client = (numAttempts > 1) ? entry.PageBlob.Default : entry.PageBlob.Aggressive; - await client.DeleteAsync(cancellationToken: StorageErrorHandler.Token); + await client.DeleteAsync(cancellationToken: StorageErrorHandler.Token).ConfigureAwait(false); return 1; }); } @@ -585,7 +585,7 @@ await BlobManager.PerformWithRetriesAsync( }, async () => { - var response = await blobEntry.PageBlob.Default.GetPropertiesAsync(); + var response = await blobEntry.PageBlob.Default.GetPropertiesAsync().ConfigureAwait(false); blobEntry.ETag = response.Value.ETag; }).ConfigureAwait(false); @@ -642,7 +642,7 @@ await BlobManager.PerformWithRetriesAsync( } return length; - }); + }).ConfigureAwait(false); readLength -= length; offset += length; @@ -666,34 +666,34 @@ unsafe void WriteToBlobAsync(BlobEntry blobEntry, IntPtr sourceAddress, ulong de { WriteToBlobAsync(blobEntry, sourceAddress, (long)destinationAddress, numBytesToWrite, id) .ContinueWith((Task t) => + { + if (pendingReadWriteOperations.TryRemove(id, out ReadWriteRequestInfo request)) { - if (pendingReadWriteOperations.TryRemove(id, out ReadWriteRequestInfo request)) + if (t.IsFaulted) { - if (t.IsFaulted) - { - BlobManager?.StorageTracer?.TsavoriteStorageProgress($"StorageOpReturned AzureStorageDevice.WriteAsync id={id} (Failure)"); - request.Callback(uint.MaxValue, request.NumBytes, request.Context); - } - else - { - BlobManager?.StorageTracer?.TsavoriteStorageProgress($"StorageOpReturned AzureStorageDevice.WriteAsync id={id}"); - request.Callback(0, request.NumBytes, request.Context); - } + BlobManager?.StorageTracer?.TsavoriteStorageProgress($"StorageOpReturned AzureStorageDevice.WriteAsync id={id} (Failure)"); + request.Callback(uint.MaxValue, request.NumBytes, request.Context); } - - if (underLease) + else { - SingleWriterSemaphore.Release(); + BlobManager?.StorageTracer?.TsavoriteStorageProgress($"StorageOpReturned AzureStorageDevice.WriteAsync id={id}"); + request.Callback(0, request.NumBytes, request.Context); } + } - }, TaskContinuationOptions.ExecuteSynchronously); + if (underLease) + { + InitialWriterSemaphore.Release(); + } + + }, TaskContinuationOptions.ExecuteSynchronously); } async Task WriteToBlobAsync(BlobEntry blobEntry, IntPtr sourceAddress, long destinationAddress, uint numBytesToWrite, long id) { if (underLease) { - await SingleWriterSemaphore.WaitAsync(); + await InitialWriterSemaphore.WaitAsync().ConfigureAwait(false); } long offset = 0; diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobEntry.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobEntry.cs index 9068b8543a8..f201d322e3c 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobEntry.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobEntry.cs @@ -71,16 +71,16 @@ await azureStorageDevice.BlobManager.PerformWithRetriesAsync( var response = await client.CreateAsync( size: size, conditions: new Azure.Storage.Blobs.Models.PageBlobRequestConditions() { IfNoneMatch = Azure.ETag.All }, - cancellationToken: azureStorageDevice.StorageErrorHandler.Token); + cancellationToken: azureStorageDevice.StorageErrorHandler.Token).ConfigureAwait(false); ETag = response.Value.ETag; return 1; }, async () => { - var response = await pageBlob.Default.GetPropertiesAsync(); + var response = await pageBlob.Default.GetPropertiesAsync().ConfigureAwait(false); ETag = response.Value.ETag; - }); + }).ConfigureAwait(false); // At this point the blob is fully created. After this line all consequent writers will write immediately. We just // need to clear the queue of pending writers. diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobManager.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobManager.cs index c4fc8ef4368..e6758171179 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobManager.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobManager.cs @@ -107,7 +107,7 @@ async Task StartAsync() { leaseBlob = leaseBlobDirectory.GetBlockBlobClient(LeaseBlobName); leaseClient = leaseBlob.WithRetries.GetBlobLeaseClient(); - await AcquireOwnership(); + await AcquireOwnership().ConfigureAwait(false); } /// @@ -130,7 +130,7 @@ public async Task StopAsync() { shutDownOrTermination.Cancel(); // has no effect if already cancelled - await LeaseMaintenanceLoopTask; // wait for loop to terminate cleanly + await LeaseMaintenanceLoopTask.ConfigureAwait(false); // wait for loop to terminate cleanly } /// @@ -186,7 +186,7 @@ await leaseClient.AcquireAsync( // the previous owner has not released the lease yet, // try again until it becomes available, should be relatively soon // as the transport layer is supposed to shut down the previous owner when starting this - await Task.Delay(TimeSpan.FromSeconds(1), StorageErrorHandler.Token); + await Task.Delay(TimeSpan.FromSeconds(1), StorageErrorHandler.Token).ConfigureAwait(false); continue; } @@ -207,7 +207,7 @@ await PerformWithRetriesAsync( try { var client = numAttempts > 2 ? leaseBlob.Default : leaseBlob.Aggressive; - await client.UploadAsync(new MemoryStream()); + await client.UploadAsync(new MemoryStream()).ConfigureAwait(false); } catch (Azure.RequestFailedException ex2) when (BlobUtilsV12.LeaseConflictOrExpired(ex2)) { @@ -216,7 +216,7 @@ await PerformWithRetriesAsync( } return 1; - }); + }).ConfigureAwait(false); continue; } @@ -241,7 +241,7 @@ await PerformWithRetriesAsync( { TimeSpan nextRetryIn = GetDelayBetweenRetries(numAttempts); TraceHelper.TsavoritePerfWarning($"Lease acquisition failed transiently, retrying in {nextRetryIn}"); - await Task.Delay(nextRetryIn); + await Task.Delay(nextRetryIn).ConfigureAwait(false); } continue; } @@ -303,7 +303,7 @@ public async Task MaintenanceLoopAsync() } // wait for successful renewal, or exit the loop as this throws - await NextLeaseRenewalTask; + await NextLeaseRenewalTask.ConfigureAwait(false); } } catch (OperationCanceledException) @@ -332,7 +332,7 @@ public async Task MaintenanceLoopAsync() && !StorageErrorHandler.IsTerminated && (leaseTimer?.Elapsed < LeaseDuration)) { - await Task.Delay(20); // give storage accesses that are in progress and require the lease a chance to complete + await Task.Delay(20).ConfigureAwait(false); // give storage accesses that are in progress and require the lease a chance to complete } TraceHelper.LeaseProgress("Waited for lease users to complete"); diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs index 9109dfa73c3..c4b1aad05e9 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/BlobUtilsV12.cs @@ -203,7 +203,7 @@ public static async Task ForceDeleteAsync(BlobContainerClient containerCli try { - await blob.DeleteAsync(); + await blob.DeleteAsync().ConfigureAwait(false); return true; } catch (Azure.RequestFailedException e) when (BlobDoesNotExist(e)) @@ -215,7 +215,7 @@ public static async Task ForceDeleteAsync(BlobContainerClient containerCli try { var leaseClient = new BlobLeaseClient(blob); - await leaseClient.BreakAsync(TimeSpan.Zero); + await leaseClient.BreakAsync(TimeSpan.Zero).ConfigureAwait(false); } catch { @@ -225,7 +225,7 @@ public static async Task ForceDeleteAsync(BlobContainerClient containerCli // retry the delete try { - await blob.DeleteAsync(); + await blob.DeleteAsync().ConfigureAwait(false); return true; } catch (Azure.RequestFailedException ex) when (BlobDoesNotExist(ex)) diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageErrorHandler.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageErrorHandler.cs index 484503ab158..1425f56a818 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageErrorHandler.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageErrorHandler.cs @@ -145,7 +145,7 @@ void Shutdown() public async Task WaitForTermination(TimeSpan timeout) { Task timeoutTask = Task.Delay(timeout); - var first = await Task.WhenAny(timeoutTask, shutdownComplete.Task); + var first = await Task.WhenAny(timeoutTask, shutdownComplete.Task).ConfigureAwait(false); return first == shutdownComplete.Task; } } diff --git a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageOperations.cs b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageOperations.cs index 9c2d8e12820..67aa3e0bb21 100644 --- a/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageOperations.cs +++ b/libs/storage/Tsavorite/cs/src/devices/AzureStorageDevice/StorageOperations.cs @@ -27,7 +27,7 @@ public async Task PerformWithRetriesAsync( { if (semaphore != null) { - await semaphore.WaitAsync(); + await semaphore.WaitAsync().ConfigureAwait(false); } Stopwatch stopwatch = new(); @@ -49,7 +49,7 @@ public async Task PerformWithRetriesAsync( } Interlocked.Increment(ref LeaseUsers); - await ConfirmLeaseIsGoodForAWhileAsync(); + await ConfirmLeaseIsGoodForAWhileAsync().ConfigureAwait(false); } StorageErrorHandler.Token.ThrowIfCancellationRequested(); @@ -90,7 +90,7 @@ public async Task PerformWithRetriesAsync( { TimeSpan nextRetryIn = GetDelayBetweenRetries(numAttempts); HandleStorageError(name, $"storage operation {name} ({intent}) failed transiently on attempt {numAttempts}, retry in {nextRetryIn}s", target, e, false, true); - await Task.Delay(nextRetryIn); + await Task.Delay(nextRetryIn).ConfigureAwait(false); } continue; } diff --git a/libs/storage/Tsavorite/cs/test/BasicLockTests.cs b/libs/storage/Tsavorite/cs/test/BasicLockTests.cs index 637cdd32427..2634ddb4274 100644 --- a/libs/storage/Tsavorite/cs/test/BasicLockTests.cs +++ b/libs/storage/Tsavorite/cs/test/BasicLockTests.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,78 +13,67 @@ namespace Tsavorite.test.LockTests { - // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - internal sealed class LocalIntKeyComparer : IKeyComparer - { - internal int mod; - - internal LocalIntKeyComparer(int mod) => this.mod = mod; - - public bool Equals(ref int k1, ref int k2) => k1 == k2; - - public long GetHashCode64(ref int k) => Utility.GetHashCode(k % mod); - } -} - -namespace Tsavorite.test.LockTests -{ - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructStoreFunctions = StoreFunctions; [TestFixture] - public class BasicLockTests : AllureTestBase + public class BasicLockTests : TestBase { - internal class Functions : SimpleSimpleFunctions + internal class Functions : SimpleLongSimpleFunctions { internal bool throwOnInitialUpdater; internal long initialUpdaterThrowAddress; - static bool Increment(ref int dst) + static bool Increment(Span field) { - ++dst; + ++field.AsRef(); return true; } - public override bool ConcurrentWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) => Increment(ref dst); + public override bool InPlaceWriter(ref LogRecord logRecord, ref long input, ReadOnlySpan srcValue, ref long output, ref UpsertInfo upsertInfo) + { + return Increment(logRecord.ValueSpan); + } - public override bool InPlaceUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => Increment(ref value); + public override bool InPlaceUpdater(ref LogRecord logRecord, ref long input, ref long output, ref RMWInfo rmwInfo) + { + return Increment(logRecord.ValueSpan); + } - public override bool InitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref long input, ref long output, ref RMWInfo rmwInfo) { if (throwOnInitialUpdater) { initialUpdaterThrowAddress = rmwInfo.Address; throw new TsavoriteException(nameof(throwOnInitialUpdater)); } - return base.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + return base.InitialUpdater(ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); } - public override bool SingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref long input, ReadOnlySpan srcValue, ref long output, ref UpsertInfo upsertInfo) { if (throwOnInitialUpdater) { initialUpdaterThrowAddress = upsertInfo.Address; throw new TsavoriteException(nameof(throwOnInitialUpdater)); } - return base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); + return base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); } - public override bool SingleDeleter(ref int key, ref int value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public override bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { if (throwOnInitialUpdater) { initialUpdaterThrowAddress = deleteInfo.Address; throw new TsavoriteException(nameof(throwOnInitialUpdater)); } - return base.SingleDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + return base.InitialDeleter(ref logRecord, ref deleteInfo); } } - private TsavoriteKV> store; - private ClientSession> session; - private BasicContext> bContext; + private TsavoriteKV> store; + private ClientSession> session; + private BasicContext> bContext; private IDevice log; - private LocalIntKeyComparer keyComparer = new(NumRecords); + private LongKeyComparerModulo keyComparer = new(NumRecords); const int NumRecords = 100; const int ValueMult = 1000000; @@ -94,15 +82,15 @@ public override bool SingleDeleter(ref int key, ref int value, ref DeleteInfo de public void Setup() { DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericStringTests.log"), deleteOnClose: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectStringTests.log"), deleteOnClose: true); store = new(new() { IndexSize = 1L << 26, LogDevice = log - }, StoreFunctions.Create(keyComparer) + }, StoreFunctions.Create(keyComparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new Functions()); + session = store.NewSession(new Functions()); bContext = session.BasicContext; } @@ -124,10 +112,11 @@ public void TearDown() public void FunctionsLockTest([Values(1, 20)] int numThreads) { // Populate - for (var key = 0; key < NumRecords; key++) + for (long key = 0; key < NumRecords; key++) { // For this test we should be in-memory, so no pending - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + long valueNum = key * ValueMult; + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref valueNum)).IsPending); } // Update @@ -136,25 +125,30 @@ public void FunctionsLockTest([Values(1, 20)] int numThreads) Task.WaitAll(tasks); // Verify - for (var key = 0; key < NumRecords; key++) + for (long key = 0; key < NumRecords; key++) { - var expectedValue = key * ValueMult + numThreads * numIters; - ClassicAssert.IsFalse(bContext.Read(key, out var value).IsPending); - ClassicAssert.AreEqual(expectedValue, value); + var expectedOutput = key * ValueMult + numThreads * numIters; + long output = 0; + ClassicAssert.IsFalse(bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output).IsPending); + ClassicAssert.AreEqual(expectedOutput, output); } } void UpdateFunc(bool useRMW, int numRecords, int numIters) { - for (var key = 0; key < numRecords; ++key) + using var localSession = store.NewSession(new Functions()); + var localBContext = localSession.BasicContext; + for (long keyNum = 0; keyNum < numRecords; ++keyNum) { + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); for (var iter = 0; iter < numIters; iter++) { if ((iter & 7) == 7) - ClassicAssert.IsFalse(bContext.Read(key).status.IsPending); + ClassicAssert.IsFalse(localBContext.Read(key).status.IsPending); // These will both just increment the stored value, ignoring the input argument. - _ = useRMW ? bContext.RMW(key, default) : bContext.Upsert(key, default); + long input = default; + _ = useRMW ? localBContext.RMW(key, ref input) : localBContext.Upsert(key, SpanByte.FromPinnedVariable(ref input)); } } } @@ -164,50 +158,55 @@ void UpdateFunc(bool useRMW, int numRecords, int numIters) public unsafe void CollidingDeletedRecordTest([Values(UpdateOp.RMW, UpdateOp.Upsert)] UpdateOp updateOp, [Values(FlushMode.NoFlush, FlushMode.OnDisk)] FlushMode flushMode) { // Populate - for (var key = 0; key < NumRecords; key++) + long keyNum = 0, valueNum = 0; + for (keyNum = 0; keyNum < NumRecords; keyNum++) { // For this test we should be in-memory, so no pending - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + valueNum = keyNum * ValueMult; + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref valueNum)).IsPending); } // Insert a colliding key so we don't elide the deleted key from the hash chain. - var deleteKey = NumRecords / 2; - var collidingKey = deleteKey + NumRecords; - ClassicAssert.IsFalse(bContext.Upsert(collidingKey, collidingKey * ValueMult).IsPending); + long deleteKeyNum = NumRecords / 2; + long collidingKeyNum = deleteKeyNum + NumRecords; + keyNum = collidingKeyNum; + valueNum = collidingKeyNum * ValueMult; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + var deleteKey = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref deleteKeyNum)); + ClassicAssert.IsFalse(bContext.Upsert(key, value).IsPending); // Now make sure we did collide - HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(ref deleteKey)); + HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(deleteKey)); ClassicAssert.IsTrue(store.FindTag(ref hei), "Cannot find deleteKey entry"); - ClassicAssert.Greater(hei.Address, Constants.kInvalidAddress, "Couldn't find deleteKey Address"); - var physicalAddress = store.hlog.GetPhysicalAddress(hei.Address); - ref var recordInfo = ref store.hlog.GetInfo(physicalAddress); - ref var lookupKey = ref store.hlog.GetKey(physicalAddress); - ClassicAssert.AreEqual(collidingKey, lookupKey, "Expected collidingKey"); + ClassicAssert.Greater(hei.Address, LogAddress.kInvalidAddress, "Couldn't find deleteKey Address"); + var physicalAddress = store.hlogBase.GetPhysicalAddress(hei.Address); + var lookupKey = LogRecord.GetInlineKey(physicalAddress); + ClassicAssert.AreEqual(collidingKeyNum, lookupKey.AsRef(), "Expected collidingKey"); // Backtrace to deleteKey - physicalAddress = store.hlog.GetPhysicalAddress(recordInfo.PreviousAddress); - recordInfo = ref store.hlog.GetInfo(physicalAddress); - lookupKey = ref store.hlog.GetKey(physicalAddress); - ClassicAssert.AreEqual(deleteKey, lookupKey, "Expected deleteKey"); - ClassicAssert.IsFalse(recordInfo.Tombstone, "Tombstone should be false"); + physicalAddress = store.hlogBase.GetPhysicalAddress(LogRecord.GetInfo(physicalAddress).PreviousAddress); + lookupKey = LogRecord.GetInlineKey(physicalAddress); + ClassicAssert.AreEqual(deleteKey.AsRef(), lookupKey.AsRef(), "Expected deleteKey"); + ClassicAssert.IsFalse(LogRecord.GetInfo(physicalAddress).Tombstone, "Tombstone should be false"); // In-place delete. ClassicAssert.IsFalse(bContext.Delete(deleteKey).IsPending); - ClassicAssert.IsTrue(recordInfo.Tombstone, "Tombstone should be true after Delete"); + ClassicAssert.IsTrue(LogRecord.GetInfo(physicalAddress).Tombstone, "Tombstone should be true after Delete"); if (flushMode == FlushMode.ReadOnly) _ = store.hlogBase.ShiftReadOnlyAddress(store.Log.TailAddress); var status = updateOp switch { - UpdateOp.RMW => bContext.RMW(deleteKey, default), - UpdateOp.Upsert => bContext.Upsert(deleteKey, default), + UpdateOp.RMW => bContext.RMW(deleteKey, ref valueNum), + UpdateOp.Upsert => bContext.Upsert(deleteKey, value), UpdateOp.Delete => throw new InvalidOperationException("UpdateOp.Delete not expected in this test"), _ => throw new InvalidOperationException($"Unknown updateOp {updateOp}") }; ClassicAssert.IsFalse(status.IsPending); - ClassicAssert.IsTrue(recordInfo.Tombstone, "Tombstone should be true after Update"); + ClassicAssert.IsTrue(LogRecord.GetInfo(physicalAddress).Tombstone, "Tombstone should be true after Update"); } [Test] @@ -218,18 +217,19 @@ public unsafe void SetInvalidOnException([Values] UpdateOp updateOp) keyComparer.mod = int.MaxValue; // Populate - for (var key = 0; key < NumRecords; key++) + for (long keyNum = 0; keyNum < NumRecords; keyNum++) { // For this test we should be in-memory, so no pending - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + long valueNum = keyNum * ValueMult; + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref valueNum)).IsPending); } var expectedThrowAddress = store.Log.TailAddress; session.functions.throwOnInitialUpdater = true; // Delete must try with an existing key; Upsert and Delete should insert a new key - var deleteKey = NumRecords / 2; - var insertKey = NumRecords + 1; + long deleteKeyNum = NumRecords / 2; + long insertKeyNum = NumRecords + 1; // Make sure everything will create a new record. store.Log.FlushAndEvict(wait: true); @@ -237,11 +237,12 @@ public unsafe void SetInvalidOnException([Values] UpdateOp updateOp) var threw = false; try { + long input = 0; var status = updateOp switch { - UpdateOp.RMW => bContext.RMW(insertKey, default), - UpdateOp.Upsert => bContext.Upsert(insertKey, default), - UpdateOp.Delete => bContext.Delete(deleteKey), + UpdateOp.RMW => bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref insertKeyNum)), ref input), + UpdateOp.Upsert => bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref insertKeyNum)), SpanByte.FromPinnedVariable(ref input)), + UpdateOp.Delete => bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref deleteKeyNum))), _ => throw new InvalidOperationException($"Unknown updateOp {updateOp}") }; ClassicAssert.IsFalse(status.IsPending); @@ -255,8 +256,8 @@ public unsafe void SetInvalidOnException([Values] UpdateOp updateOp) ClassicAssert.IsTrue(threw, "Test should have thrown"); ClassicAssert.AreEqual(expectedThrowAddress, session.functions.initialUpdaterThrowAddress, "Unexpected throw address"); - var physicalAddress = store.hlog.GetPhysicalAddress(expectedThrowAddress); - ref var recordInfo = ref store.hlog.GetInfo(physicalAddress); + var physicalAddress = store.hlogBase.GetPhysicalAddress(expectedThrowAddress); + var recordInfo = LogRecord.GetInfo(physicalAddress); ClassicAssert.IsTrue(recordInfo.Invalid, "Expected Invalid record"); } } diff --git a/libs/storage/Tsavorite/cs/test/BasicTests.cs b/libs/storage/Tsavorite/cs/test/BasicTests.cs index f1a76564c3b..e95235023ee 100644 --- a/libs/storage/Tsavorite/cs/test/BasicTests.cs +++ b/libs/storage/Tsavorite/cs/test/BasicTests.cs @@ -1,11 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Diagnostics; using System.IO; using System.Linq; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,24 +13,21 @@ namespace Tsavorite.test { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; //** NOTE - more detailed / in depth Read tests in ReadAddressTests.cs //** These tests ensure the basics are fully covered - - [AllureNUnit] [TestFixture] - internal class BasicTests : AllureTestBase + internal class BasicTests : TestBase { - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; - TestDeviceType deviceType; [SetUp] public void Setup() @@ -40,7 +36,7 @@ public void Setup() DeleteDirectory(MethodTestDir, wait: true); } - private void Setup(KVSettings kvSettings, TestDeviceType deviceType, int latencyMs = DefaultLocalMemoryDeviceLatencyMs) + private void Setup(KVSettings kvSettings, TestDeviceType deviceType, int latencyMs = DefaultLocalMemoryDeviceLatencyMs) { kvSettings.IndexSize = 1L << 13; @@ -49,11 +45,11 @@ private void Setup(KVSettings kvSettings, TestDeviceType kvSettings.LogDevice = log; store = new(kvSettings - , StoreFunctions.Create(KeyStruct.Comparer.Instance) + , StoreFunctions.Create(KeyStruct.Comparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new Functions()); + session = store.NewSession(new Functions()); bContext = session.BasicContext; } @@ -87,7 +83,7 @@ private void AssertCompleted(Status expected, Status actual) [Category("Smoke")] public void NativeInMemWriteRead([Values] TestDeviceType deviceType) { - Setup(new() { PageSize = 1L << 10, MemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); + Setup(new() { PageSize = 1L << 10, LogMemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); InputStruct input = default; OutputStruct output = default; @@ -95,8 +91,8 @@ public void NativeInMemWriteRead([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); @@ -108,7 +104,7 @@ public void NativeInMemWriteRead([Values] TestDeviceType deviceType) [Category("Smoke")] public void NativeInMemWriteReadDelete([Values] TestDeviceType deviceType) { - Setup(new() { PageSize = 1L << 10, MemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); + Setup(new() { PageSize = 1L << 10, LogMemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); InputStruct input = default; OutputStruct output = default; @@ -116,38 +112,37 @@ public void NativeInMemWriteReadDelete([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); - _ = bContext.Delete(ref key1, Empty.Default); + _ = bContext.Delete(key1, Empty.Default); - status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); var key2 = new KeyStruct { kfield1 = 14, kfield2 = 15 }; var value2 = new ValueStruct { vfield1 = 24, vfield2 = 25 }; - _ = bContext.Upsert(ref key2, ref value2, Empty.Default); - status = bContext.Read(ref key2, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key2, SpanByte.FromPinnedVariable(ref value2), Empty.Default); + status = bContext.Read(key2, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(value2.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value2.vfield2, output.value.vfield2); } - [Test] [Category("TsavoriteKV")] [Category("Smoke")] public void NativeInMemWriteReadDelete2() { // Just set this one since Write Read Delete already does all four devices - deviceType = TestDeviceType.MLSD; + var deviceType = TestDeviceType.MLSD; const int count = 10; - Setup(new() { MemorySize = 1L << 29 }, deviceType); + Setup(new() { LogMemorySize = 1L << 29 }, deviceType); InputStruct input = default; OutputStruct output = default; @@ -157,13 +152,13 @@ public void NativeInMemWriteReadDelete2() var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } for (var i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; - _ = bContext.Delete(ref key1, Empty.Default); + _ = bContext.Delete(key1, Empty.Default); } for (var i = 0; i < 10 * count; i++) @@ -171,16 +166,16 @@ public void NativeInMemWriteReadDelete2() var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } for (var i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); } } @@ -191,12 +186,12 @@ public void NativeInMemWriteReadDelete2() public unsafe void NativeInMemWriteRead2() { // Just use this one instead of all four devices since InMemWriteRead covers all four devices - deviceType = TestDeviceType.MLSD; + var deviceType = TestDeviceType.MLSD; const int count = 200; - Setup(new() { MemorySize = 1L << 29 }, deviceType); - session = store.NewSession(new Functions()); + Setup(new() { LogMemorySize = 1L << 29 }, deviceType); + session = store.NewSession(new Functions()); InputStruct input = default; @@ -206,7 +201,7 @@ public unsafe void NativeInMemWriteRead2() var i = r.Next(10000); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } r = new Random(10); @@ -218,7 +213,7 @@ public unsafe void NativeInMemWriteRead2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - if (bContext.Read(ref key1, ref input, ref output, Empty.Default).IsPending) + if (bContext.Read(key1, ref input, ref output, Empty.Default).IsPending) { _ = bContext.CompletePending(true); } @@ -236,14 +231,14 @@ public unsafe void NativeInMemWriteRead2() var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - ClassicAssert.IsFalse(bContext.Read(ref key1, ref input, ref output, Empty.Default).Found); + ClassicAssert.IsFalse(bContext.Read(key1, ref input, ref output, Empty.Default).Found); } } [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Values] BatchMode batchMode) + public void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Values] BatchMode batchMode) { InputStruct input = default; const int RandSeed = 10; @@ -254,14 +249,14 @@ public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Val var sw = Stopwatch.StartNew(); var latencyMs = batchMode == BatchMode.NoBatch ? 0 : DefaultLocalMemoryDeviceLatencyMs; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType, latencyMs: latencyMs); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType, latencyMs: latencyMs); for (var c = 0; c < NumRecs; c++) { var i = r.Next(RandRange); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } r = new Random(RandSeed); @@ -274,7 +269,7 @@ public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Val var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - if (bContext.Read(ref key1, ref input, ref output, Empty.Default).IsPending) + if (bContext.Read(key1, ref input, ref output, Empty.Default).IsPending) { ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); @@ -294,7 +289,7 @@ public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Val var i = r.Next(RandRange); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var foundStatus = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var foundStatus = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(foundStatus.IsPending); if (batchMode == BatchMode.NoBatch) { @@ -313,8 +308,8 @@ public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Val while (outputs.Next()) { count++; - ClassicAssert.AreEqual(outputs.Current.Key.kfield1, outputs.Current.Output.value.vfield1); - ClassicAssert.AreEqual(outputs.Current.Key.kfield2, outputs.Current.Output.value.vfield2); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield1, outputs.Current.Output.value.vfield1); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield2, outputs.Current.Output.value.vfield2); } outputs.Dispose(); ClassicAssert.AreEqual(batchSize + (c == batchSize ? 1 : 0), count); @@ -325,74 +320,12 @@ public unsafe void TestShiftHeadAddress([Values] TestDeviceType deviceType, [Val [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) + public void NativeInMemRMWKeys([Values] TestDeviceType deviceType) { InputStruct input = default; OutputStruct output = default; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); - - var nums = Enumerable.Range(0, 1000).ToArray(); - var rnd = new Random(11); - for (var i = 0; i < nums.Length; ++i) - { - var randomIndex = rnd.Next(nums.Length); - (nums[i], nums[randomIndex]) = (nums[randomIndex], nums[i]); - } - - for (var j = 0; j < nums.Length; ++j) - { - var i = nums[j]; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = bContext.RMW(ref key1, ref input, Empty.Default); - } - for (var j = 0; j < nums.Length; ++j) - { - var i = nums[j]; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - if (bContext.RMW(ref key1, ref input, ref output, Empty.Default).IsPending) - { - _ = bContext.CompletePending(true); - } - else - { - ClassicAssert.AreEqual(2 * i, output.value.vfield1); - ClassicAssert.AreEqual(2 * (i + 1), output.value.vfield2); - } - } - - Status status; - KeyStruct key; - - for (var j = 0; j < nums.Length; ++j) - { - var i = nums[j]; - - key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - ValueStruct value = new() { vfield1 = i, vfield2 = i + 1 }; - - status = bContext.Read(ref key, ref input, ref output, Empty.Default); - - AssertCompleted(new(StatusCode.Found), status); - ClassicAssert.AreEqual(2 * value.vfield1, output.value.vfield1); - ClassicAssert.AreEqual(2 * value.vfield2, output.value.vfield2); - } - - key = new KeyStruct { kfield1 = nums.Length, kfield2 = nums.Length + 1 }; - status = bContext.Read(ref key, ref input, ref output, Empty.Default); - AssertCompleted(new(StatusCode.NotFound), status); - } - - // Tests the overload where no reference params used: key,input,userContext - [Test] - [Category("TsavoriteKV")] - public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) - { - InputStruct input = default; - - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); var nums = Enumerable.Range(0, 1000).ToArray(); var rnd = new Random(11); @@ -408,7 +341,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = bContext.RMW(ref key1, ref input, Empty.Default); + _ = bContext.RMW(key1, ref input, Empty.Default); } // CopyUpdater @@ -417,10 +350,15 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = bContext.RMW(key1, input); // no ref and do not set any other params + if (bContext.RMW(key1, ref input, ref output, Empty.Default).IsPending) + { + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + (_ /*status*/, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.AreEqual(2 * i, output.value.vfield1); + ClassicAssert.AreEqual(2 * (i + 1), output.value.vfield2); } - OutputStruct output = default; Status status; KeyStruct key; @@ -431,7 +369,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; ValueStruct value = new() { vfield1 = i, vfield2 = i + 1 }; - status = bContext.Read(ref key, ref input, ref output, Empty.Default); + status = bContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(2 * value.vfield1, output.value.vfield1); @@ -439,7 +377,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) } key = new KeyStruct { kfield1 = nums.Length, kfield2 = nums.Length + 1 }; - status = bContext.Read(ref key, ref input, ref output, Empty.Default); + status = bContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); } @@ -447,17 +385,18 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public void ReadNoRefKeyInputOutput([Values] TestDeviceType deviceType) + public void ReadKeyInputOutput([Values] TestDeviceType deviceType) { InputStruct input = default; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(key1, input, out var output, Empty.Default); + OutputStruct output = default; + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -472,13 +411,14 @@ public void ReadNoRefKeyInputOutput([Values] TestDeviceType deviceType) [Category("TsavoriteKV")] public void ReadNoRefKey([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(key1, out var output, Empty.Default); + OutputStruct output = default; + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -495,15 +435,15 @@ public void ReadNoRefKey([Values] TestDeviceType deviceType) [Category("Smoke")] public void ReadWithoutInput([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(ref key1, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -519,12 +459,12 @@ public void ReadWithoutInput([Values] TestDeviceType deviceType) [Category("Smoke")] public void ReadBareMinParams([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); var (status, output) = bContext.Read(key1); AssertCompleted(new(StatusCode.Found), status); @@ -541,9 +481,9 @@ public void ReadBareMinParams([Values] TestDeviceType deviceType) public void ReadAtAddressDefaultOptions() { // Just functional test of ReadFlag so one device is enough - deviceType = TestDeviceType.MLSD; + var deviceType = TestDeviceType.MLSD; - Setup(new() { MemorySize = 1L << 29 }, deviceType); + Setup(new() { LogMemorySize = 1L << 29 }, deviceType); InputStruct input = default; OutputStruct output = default; @@ -552,7 +492,7 @@ public void ReadAtAddressDefaultOptions() var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; ReadOptions readOptions = default; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); var status = bContext.ReadAtAddress(store.Log.BeginAddress, ref input, ref output, ref readOptions, out _, Empty.Default); AssertCompleted(new(StatusCode.Found), status); @@ -566,25 +506,15 @@ class SkipReadCacheFunctions : Functions { internal long expectedReadAddress; - public override bool SingleReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord logRecord, ref InputStruct input, ref OutputStruct output, ref ReadInfo readInfo) { - Assign(ref value, ref dst, ref readInfo); - return true; - } - - public override bool ConcurrentReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - Assign(ref value, ref dst, ref readInfo); - return true; - } - - void Assign(ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) - { - dst.value = value; + output.value = logRecord.ValueSpan.AsRef(); ClassicAssert.AreEqual(expectedReadAddress, readInfo.Address); expectedReadAddress = -1; // show that the test executed + return true; } - public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) + + public override void ReadCompletionCallback(ref DiskLogRecord logRecord, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) { // Do no data verifications here; they're done in the test } @@ -596,12 +526,12 @@ public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct i public void ReadAtAddressIgnoreReadCache() { // Another ReadFlag functional test so one device is enough - deviceType = TestDeviceType.MLSD; + var deviceType = TestDeviceType.MLSD; - Setup(new() { MemorySize = 1L << 29, ReadCacheEnabled = true }, deviceType); + Setup(new() { LogMemorySize = 1L << 29, ReadCacheEnabled = true }, deviceType); SkipReadCacheFunctions functions = new(); - using var skipReadCacheSession = store.NewSession(functions); + using var skipReadCacheSession = store.NewSession(functions); var skipReadCachebContext = skipReadCacheSession.BasicContext; InputStruct input = default; @@ -611,7 +541,7 @@ public void ReadAtAddressIgnoreReadCache() var readAtAddress = store.Log.BeginAddress; Status status; - _ = skipReadCachebContext.Upsert(ref key1, ref value, Empty.Default); + _ = skipReadCachebContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); void VerifyOutput() { @@ -635,7 +565,7 @@ void VerifyResult() // This will just be an ordinary read, as the record is in memory. functions.expectedReadAddress = readAtAddress; - status = skipReadCachebContext.Read(ref key1, ref input, ref output); + status = skipReadCachebContext.Read(key1, ref input, ref output); ClassicAssert.IsTrue(status.Found); VerifyOutput(); @@ -645,7 +575,7 @@ void VerifyResult() // Do not put it into the read cache. functions.expectedReadAddress = readAtAddress; ReadOptions readOptions = new() { CopyOptions = ReadCopyOptions.None }; - status = skipReadCachebContext.ReadAtAddress(readAtAddress, ref key1, ref input, ref output, ref readOptions, out _); + status = skipReadCachebContext.ReadAtAddress(readAtAddress, key1, ref input, ref output, ref readOptions, out _); VerifyResult(); ClassicAssert.AreEqual(store.ReadCache.BeginAddress, store.ReadCache.TailAddress); @@ -653,15 +583,15 @@ void VerifyResult() // Put it into the read cache. functions.expectedReadAddress = readAtAddress; readOptions.CopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.ReadCache); - status = skipReadCachebContext.ReadAtAddress(readAtAddress, ref key1, ref input, ref output, ref readOptions, out _); + status = skipReadCachebContext.ReadAtAddress(readAtAddress, key1, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(status.IsPending); VerifyResult(); ClassicAssert.Less(store.ReadCache.BeginAddress, store.ReadCache.TailAddress); // Now this will read from the read cache. - functions.expectedReadAddress = Constants.kInvalidAddress; - status = skipReadCachebContext.Read(ref key1, ref input, ref output); + functions.expectedReadAddress = LogAddress.kInvalidAddress; + status = skipReadCachebContext.Read(key1, ref input, ref output); ClassicAssert.IsFalse(status.IsPending); ClassicAssert.IsTrue(status.Found); VerifyOutput(); @@ -673,7 +603,7 @@ void VerifyResult() [Category("Smoke")] public void UpsertDefaultsTest([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); InputStruct input = default; OutputStruct output = default; @@ -683,8 +613,8 @@ public void UpsertDefaultsTest([Values] TestDeviceType deviceType) ClassicAssert.AreEqual(0, store.EntryCount); - _ = bContext.Upsert(ref key1, ref value); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value)); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(1, store.EntryCount); @@ -692,31 +622,6 @@ public void UpsertDefaultsTest([Values] TestDeviceType deviceType) ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); } - // Simple Upsert test of overload where not using Ref for key and value and setting all parameters - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void UpsertNoRefNoDefaultsTest() - { - // Just checking more parameter values so one device is enough - deviceType = TestDeviceType.MLSD; - - Setup(new() { MemorySize = 1L << 29 }, deviceType); - - InputStruct input = default; - OutputStruct output = default; - - var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; - var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - - _ = bContext.Upsert(key1, value, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); - AssertCompleted(new(StatusCode.Found), status); - - ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); - ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); - } - //**** Quick End to End Sample code from help docs *** // Very minor changes to LogDevice call and type of Asserts to use but basically code from Sample code in docs // Also tests the overload call of .Read (ref key ref output) @@ -726,25 +631,27 @@ public static void KVBasicsSampleEndToEndInDocs() { using var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog.log"), deleteOnClose: false); - using var store = new TsavoriteKV( + using var store = new TsavoriteKV( new() { IndexSize = 1L << 26, LogDevice = log, - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var s = store.NewSession>(new SimpleSimpleFunctions()); + using var s = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = s.BasicContext; - long key = 1, value = 1, input = 10, output = 0; - _ = bContext.Upsert(ref key, ref value); - _ = bContext.Read(ref key, ref output); - ClassicAssert.AreEqual(value, output); - _ = bContext.RMW(ref key, ref input); - _ = bContext.RMW(ref key, ref input); - _ = bContext.Read(ref key, ref output); + long keyNum = 1, valueNum = 1, input = 10, output = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + _ = bContext.Upsert(key, value); + _ = bContext.Read(key, ref output); + ClassicAssert.AreEqual(valueNum, output); + _ = bContext.RMW(key, ref input); + _ = bContext.RMW(key, ref input); + _ = bContext.Read(key, ref output); ClassicAssert.AreEqual(10, output); } @@ -757,7 +664,7 @@ public static void LogPathtooLong() string testDir = new('x', Native32.WIN32_MAX_PATH - 11); // As in LSD, -11 for "." using var log = Devices.CreateLogDevice(testDir, deleteOnClose: true); // Should succeed - _ = Assert.Throws(typeof(TsavoriteException), () => Devices.CreateLogDevice(testDir + "y", deleteOnClose: true)); + _ = Assert.Throws(() => Devices.CreateLogDevice(testDir + "y", deleteOnClose: true)); } [Test] @@ -766,16 +673,16 @@ public static void BasicSyncOperationsTest() { using var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog.log"), deleteOnClose: false); - using var store = new TsavoriteKV( + using var store = new TsavoriteKV( new() { IndexSize = 1L << 26, LogDevice = log, - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; const int numRecords = 500; @@ -783,28 +690,33 @@ public static void BasicSyncOperationsTest() var hashes = new long[numRecords]; Status status; - long output; + long output = 0; - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult; - hashes[key] = store.storeFunctions.GetKeyHashCode64(ref key); + var valueNum = keyNum + valueMult; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + + hashes[keyNum] = store.storeFunctions.GetKeyHashCode64(key); status = bContext.Upsert(key, value); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); - status = bContext.Read(key, out output); + status = bContext.Read(key, ref output); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } void doUpdate(bool useRMW) { // Update and Read without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult * 2; + var valueNum = keyNum + valueMult * 2; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); if (useRMW) { - status = bContext.RMW(key, value); + status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } else @@ -812,31 +724,33 @@ void doUpdate(bool useRMW) status = bContext.Upsert(key, value); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } - status = bContext.Read(key, out output); + status = bContext.Read(key, ref output); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } // Update and Read with keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult * 3; + var valueNum = keyNum + valueMult * 3; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); if (useRMW) { - RMWOptions rmwOptions = new() { KeyHash = hashes[key] }; - status = bContext.RMW(key, value, ref rmwOptions); + RMWOptions rmwOptions = new() { KeyHash = hashes[keyNum] }; + status = bContext.RMW(key, ref valueNum, ref rmwOptions); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } else { - UpsertOptions upsertOptions = new() { KeyHash = hashes[key] }; + UpsertOptions upsertOptions = new() { KeyHash = hashes[keyNum] }; status = bContext.Upsert(key, value, ref upsertOptions); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } - ReadOptions readOptions = new() { KeyHash = hashes[key] }; - status = bContext.Read(key, out output, ref readOptions); + ReadOptions readOptions = new() { KeyHash = hashes[keyNum] }; + status = bContext.Read(key, ref output, ref readOptions); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } } @@ -844,21 +758,23 @@ void doUpdate(bool useRMW) doUpdate(useRMW: true); // Delete without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); status = bContext.Delete(key); ClassicAssert.IsTrue(status.Found, status.ToString()); - status = bContext.Read(key, out _); + status = bContext.Read(key, ref output); ClassicAssert.IsTrue(status.NotFound, status.ToString()); } // Update and Read without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - DeleteOptions deleteOptions = new() { KeyHash = hashes[key] }; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + DeleteOptions deleteOptions = new() { KeyHash = hashes[keyNum] }; status = bContext.Delete(key, ref deleteOptions); - ReadOptions readOptions = new() { KeyHash = hashes[key] }; - status = bContext.Read(key, out _, ref readOptions); + ReadOptions readOptions = new() { KeyHash = hashes[keyNum] }; + status = bContext.Read(key, ref output, ref readOptions); ClassicAssert.IsTrue(status.NotFound, status.ToString()); } } @@ -869,16 +785,16 @@ public static void BasicOperationsTest() { using var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog.log"), deleteOnClose: false); - using var store = new TsavoriteKV( + using var store = new TsavoriteKV( new() { IndexSize = 1L << 26, LogDevice = log, - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; const int numRecords = 500; @@ -888,26 +804,32 @@ public static void BasicOperationsTest() Status status; long output; - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult; - hashes[key] = store.storeFunctions.GetKeyHashCode64(ref key); + var valueNum = keyNum + valueMult; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + + hashes[keyNum] = store.storeFunctions.GetKeyHashCode64(key); status = bContext.Upsert(key, value); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); (status, output) = bContext.Read(key); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } void doUpdate(bool useRMW) { // Update and Read without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult * 2; + var valueNum = keyNum + valueMult * 2; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + if (useRMW) { - status = bContext.RMW(key, value); + status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } else @@ -917,29 +839,32 @@ void doUpdate(bool useRMW) } (status, output) = bContext.Read(key); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } // Update and Read with keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - var value = key + valueMult * 3; + var valueNum = keyNum + valueMult * 3; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + if (useRMW) { - RMWOptions rmwOptions = new() { KeyHash = hashes[key] }; - status = bContext.RMW(key, value, ref rmwOptions); + RMWOptions rmwOptions = new() { KeyHash = hashes[keyNum] }; + status = bContext.RMW(key, ref valueNum, ref rmwOptions); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } else { - UpsertOptions upsertOptions = new() { KeyHash = hashes[key] }; + UpsertOptions upsertOptions = new() { KeyHash = hashes[keyNum] }; status = bContext.Upsert(key, value, ref upsertOptions); ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); } - ReadOptions readOptions = new() { KeyHash = hashes[key] }; + ReadOptions readOptions = new() { KeyHash = hashes[keyNum] }; (status, output) = bContext.Read(key, ref readOptions); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(value, output); + ClassicAssert.AreEqual(valueNum, output); } } @@ -947,8 +872,10 @@ void doUpdate(bool useRMW) doUpdate(useRMW: true); // Delete without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + status = bContext.Delete(key); ClassicAssert.IsTrue(status.Found, status.ToString()); (status, _) = bContext.Read(key); @@ -956,11 +883,13 @@ void doUpdate(bool useRMW) } // Update and Read without keyHash - for (var key = 0L; key < numRecords; key++) + for (var keyNum = 0L; keyNum < numRecords; keyNum++) { - DeleteOptions deleteOptions = new() { KeyHash = hashes[key] }; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + + DeleteOptions deleteOptions = new() { KeyHash = hashes[keyNum] }; status = bContext.Delete(key, ref deleteOptions); - ReadOptions readOptions = new() { KeyHash = hashes[key] }; + ReadOptions readOptions = new() { KeyHash = hashes[keyNum] }; (status, _) = bContext.Read(key, ref readOptions); ClassicAssert.IsTrue(status.NotFound, status.ToString()); } diff --git a/libs/storage/Tsavorite/cs/test/BlittableIterationTests.cs b/libs/storage/Tsavorite/cs/test/BlittableIterationTests.cs deleted file mode 100644 index 7e992af0bc0..00000000000 --- a/libs/storage/Tsavorite/cs/test/BlittableIterationTests.cs +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test -{ - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class BlittableIterationTests : AllureTestBase - { - private TsavoriteKV> store; - private IDevice log; - - [SetUp] - public void Setup() - { - // Clean up log files from previous test runs in case they weren't cleaned up - DeleteDirectory(MethodTestDir, wait: true); - } - - [TearDown] - public void TearDown() - { - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - OnTearDown(); - } - - internal struct BlittablePushIterationTestFunctions : IScanIteratorFunctions - { - internal int keyMultToValue; - internal long numRecords; - internal int stopAt; - - public bool SingleReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - if (keyMultToValue > 0) - ClassicAssert.AreEqual(key.kfield1 * keyMultToValue, value.vfield1); - return stopAt != ++numRecords; - } - - public bool ConcurrentReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public readonly bool OnStart(long beginAddress, long endAddress) => true; - public readonly void OnException(Exception exception, long numberOfRecords) { } - public readonly void OnStop(bool completed, long numberOfRecords) { } - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public void BlittableIterationBasicTest([Values] TestDeviceType deviceType, [Values] ScanIteratorType scanIteratorType) - { - log = CreateTestDevice(deviceType, Path.Join(MethodTestDir, $"{deviceType}.log")); - - store = new( - new() - { - IndexSize = 1L << 26, - LogDevice = log, - MemorySize = 1L << 15, - PageSize = 1L << 9, - SegmentSize = 1L << 22 - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var session = store.NewSession(new FunctionsCompaction()); - var bContext = session.BasicContext; - - BlittablePushIterationTestFunctions scanIteratorFunctions = new(); - - const int totalRecords = 500; - - void iterateAndVerify(int keyMultToValue, int expectedRecs) - { - scanIteratorFunctions.keyMultToValue = keyMultToValue; - scanIteratorFunctions.numRecords = 0; - - if (scanIteratorType == ScanIteratorType.Pull) - { - using var iter = session.Iterate(); - while (iter.GetNext(out var recordInfo)) - _ = scanIteratorFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), default, default, out _); - } - else - ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); - - ClassicAssert.AreEqual(expectedRecs, scanIteratorFunctions.numRecords); - } - - // Initial population - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(1, totalRecords); - - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = 2 * i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(2, totalRecords); - - for (var i = totalRecords / 2; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(0, totalRecords); - - for (var i = 0; i < totalRecords; i += 2) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(0, totalRecords); - - for (var i = 0; i < totalRecords; i += 2) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - _ = bContext.Delete(ref key1); - } - iterateAndVerify(0, totalRecords / 2); - - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = 3 * i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(3, totalRecords); - - store.Log.FlushAndEvict(wait: true); - iterateAndVerify(3, totalRecords); - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public void BlittableIterationPushStopTest() - { - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "stop_test.log")); - - store = new( - new() - { - IndexSize = 1L << 26, - LogDevice = log, - MemorySize = 1L << 15, - PageSize = 1L << 9, - SegmentSize = 1L << 22 - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var session = store.NewSession(new FunctionsCompaction()); - var bContext = session.BasicContext; - BlittablePushIterationTestFunctions scanIteratorFunctions = new(); - - const int totalRecords = 2000; - var start = store.Log.TailAddress; - - void scanAndVerify(int stopAt, bool useScan) - { - scanIteratorFunctions.numRecords = 0; - scanIteratorFunctions.stopAt = stopAt; - if (useScan) - ClassicAssert.IsFalse(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); - else - ClassicAssert.IsFalse(session.Iterate(ref scanIteratorFunctions), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); - ClassicAssert.AreEqual(stopAt, scanIteratorFunctions.numRecords); - } - - // Initial population - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - - scanAndVerify(42, useScan: true); - scanAndVerify(42, useScan: false); - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public unsafe void BlittableIterationPushLockTest([Values(1, 4)] int scanThreads, [Values(1, 4)] int updateThreads, [Values] ScanMode scanMode) - { - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "lock_test.log")); - - // Must be large enough to contain all records in memory to exercise locking - store = new( - new() - { - IndexSize = 1L << 26, - LogDevice = log, - MemorySize = 1L << 25, - PageSize = 1L << 20, - SegmentSize = 1L << 22 - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - const int totalRecords = 2000; - var start = store.Log.TailAddress; - - void LocalScan(int i) - { - using var session = store.NewSession(new FunctionsCompaction()); - BlittablePushIterationTestFunctions scanIteratorFunctions = new(); - if (scanMode == ScanMode.Scan) - ClassicAssert.IsTrue(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to complete push scan; numRecords = {scanIteratorFunctions.numRecords}"); - else - ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); - ClassicAssert.AreEqual(totalRecords, scanIteratorFunctions.numRecords); - } - - void LocalUpdate(int tid) - { - using var session = store.NewSession(new FunctionsCompaction()); - var bContext = session.BasicContext; - for (var iteration = 0; iteration < 2; ++iteration) - { - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = (tid + 1) * i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); - } - } - } - - { // Initial population - using var session = store.NewSession(new FunctionsCompaction()); - var bContext = session.BasicContext; - for (var i = 0; i < totalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - } - - List tasks = []; // Task rather than Thread for propagation of exception. - var numThreads = scanThreads + updateThreads; - for (var t = 0; t < numThreads; t++) - { - var tid = t; - if (t < scanThreads) - tasks.Add(Task.Factory.StartNew(() => LocalScan(tid))); - else - tasks.Add(Task.Factory.StartNew(() => LocalUpdate(tid))); - } - Task.WaitAll([.. tasks]); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/BlittableLogScanTests.cs b/libs/storage/Tsavorite/cs/test/BlittableLogScanTests.cs deleted file mode 100644 index 061d1bf5633..00000000000 --- a/libs/storage/Tsavorite/cs/test/BlittableLogScanTests.cs +++ /dev/null @@ -1,399 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test -{ - // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - struct KeyStructComparerModulo : IKeyComparer - { - readonly long mod; - - internal KeyStructComparerModulo(long mod) => this.mod = mod; - - public readonly bool Equals(ref KeyStruct k1, ref KeyStruct k2) => k1.kfield1 == k2.kfield1 && k1.kfield2 == k2.kfield2; - - // Force collisions to create a chain - public readonly long GetHashCode64(ref KeyStruct key) - { - long hash = Utility.GetHashCode(key.kfield1); - return mod > 0 ? hash % mod : hash; - } - } -} - -namespace Tsavorite.test -{ - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class BlittableLogScanTests : AllureTestBase - { - private TsavoriteKV store; - private IDevice log; - const int TotalRecords = 2000; - const int PageSizeBits = 10; - - [SetUp] - public void Setup() - { - DeleteDirectory(MethodTestDir, wait: true); - - KeyStructComparerModulo comparer = new(0); - foreach (var arg in TestContext.CurrentContext.Test.Arguments) - { - if (arg is HashModulo mod && mod == HashModulo.Hundred) - { - comparer = new(100); - continue; - } - } - - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.log"), deleteOnClose: true); - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - MemorySize = 1L << 24, - PageSize = 1L << PageSizeBits - }, StoreFunctions.Create(comparer) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - } - - [TearDown] - public void TearDown() - { - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - OnTearDown(); - } - - internal struct BlittablePushScanTestFunctions : IScanIteratorFunctions - { - internal long numRecords; - - public readonly bool OnStart(long beginAddress, long endAddress) => true; - - public bool ConcurrentReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public bool SingleReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - ClassicAssert.AreEqual(numRecords, key.kfield1); - ClassicAssert.AreEqual(numRecords + 1, key.kfield2); - ClassicAssert.AreEqual(numRecords, value.vfield1); - ClassicAssert.AreEqual(numRecords + 1, value.vfield2); - - ++numRecords; - return true; - } - - public readonly void OnException(Exception exception, long numberOfRecords) { } - - public readonly void OnStop(bool completed, long numberOfRecords) { } - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - - public void BlittableDiskWriteScan([Values] ScanIteratorType scanIteratorType) - { - using var session = store.NewSession(new Functions()); - var bContext = session.BasicContext; - - using var s = store.Log.Subscribe(new LogObserver()); - var start = store.Log.TailAddress; - - for (int i = 0; i < TotalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - } - store.Log.FlushAndEvict(true); - - BlittablePushScanTestFunctions scanIteratorFunctions = new(); - void scanAndVerify(ScanBufferingMode sbm) - { - scanIteratorFunctions.numRecords = 0; - - if (scanIteratorType == ScanIteratorType.Pull) - { - using var iter = store.Log.Scan(start, store.Log.TailAddress, sbm); - while (iter.GetNext(out var recordInfo)) - _ = scanIteratorFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), default, default, out _); - } - else - ClassicAssert.IsTrue(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress, sbm), "Failed to complete push iteration"); - - ClassicAssert.AreEqual(TotalRecords, scanIteratorFunctions.numRecords); - } - - scanAndVerify(ScanBufferingMode.SinglePageBuffering); - scanAndVerify(ScanBufferingMode.DoublePageBuffering); - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - - public void BlittableScanJumpToBeginAddressTest() - { - using var session = store.NewSession(new Functions()); - var bContext = session.BasicContext; - - const int numRecords = 200; - const int numTailRecords = 10; - long shiftBeginAddressTo = 0; - int shiftToKey = 0; - for (int i = 0; i < numRecords; i++) - { - if (i == numRecords - numTailRecords) - { - shiftBeginAddressTo = store.Log.TailAddress; - shiftToKey = i; - } - var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key, ref value, Empty.Default); - } - - using var iter = store.Log.Scan(store.Log.HeadAddress, store.Log.TailAddress); - - for (int i = 0; i < 100; ++i) - { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); - ClassicAssert.AreEqual(i, iter.GetKey().kfield1); - ClassicAssert.AreEqual(i, iter.GetValue().vfield1); - } - - store.Log.ShiftBeginAddress(shiftBeginAddressTo); - - for (int i = 0; i < numTailRecords; ++i) - { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); - if (i == 0) - ClassicAssert.AreEqual(store.Log.BeginAddress, iter.CurrentAddress); - var expectedKey = numRecords - numTailRecords + i; - ClassicAssert.AreEqual(expectedKey, iter.GetKey().kfield1); - ClassicAssert.AreEqual(expectedKey, iter.GetValue().vfield1); - } - } - - public class ScanFunctions : FunctionsWithContext - { - // Right now this is unused but helped with debugging so I'm keeping it around. - internal long insertedAddress; - - public override bool SingleWriter(ref KeyStruct key, ref InputStruct input, ref ValueStruct src, ref ValueStruct dst, ref OutputStruct output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - insertedAddress = upsertInfo.Address; - return base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); - } - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - - public void BlittableScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) - { - const long PageSize = 1L << PageSizeBits; - var recordSize = BlittableAllocatorImpl.RecordSize; - - using var session = store.NewSession(new ScanFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < TotalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - - var scanCursorFuncs = new ScanCursorFuncs(); - - // Normal operations - var endAddresses = new long[] { store.Log.TailAddress, long.MaxValue }; - var counts = new long[] { 10, 100, long.MaxValue }; - - long cursor = 0; - for (var iAddr = 0; iAddr < endAddresses.Length; ++iAddr) - { - for (var iCount = 0; iCount < counts.Length; ++iCount) - { - scanCursorFuncs.Initialize(verifyKeys: true); - while (session.ScanCursor(ref cursor, counts[iCount], scanCursorFuncs, endAddresses[iAddr])) - ; - ClassicAssert.AreEqual(TotalRecords, scanCursorFuncs.numRecords, $"count: {counts[iCount]}, endAddress {endAddresses[iAddr]}"); - ClassicAssert.AreEqual(0, cursor, "Expected cursor to be 0, pt 1"); - } - } - - // After FlushAndEvict, we will be doing pending IO. With collision chains, this means we may be returning colliding keys from in-memory - // before the sequential keys from pending IO. Therefore we do not want to verify keys if we are causing collisions. - store.Log.FlushAndEvict(wait: true); - bool verifyKeys = hashMod == HashModulo.NoMod; - - // Scan and verify we see them all - scanCursorFuncs.Initialize(verifyKeys); - ClassicAssert.IsFalse(session.ScanCursor(ref cursor, long.MaxValue, scanCursorFuncs, long.MaxValue), "Expected scan to finish and return false, pt 1"); - ClassicAssert.AreEqual(TotalRecords, scanCursorFuncs.numRecords, "Unexpected count for all on-disk"); - ClassicAssert.AreEqual(0, cursor, "Expected cursor to be 0, pt 2"); - - // Add another totalRecords, with keys incremented by totalRecords to remain distinct, and verify we see all keys. - for (int i = 0; i < TotalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i + TotalRecords, kfield2 = i + TotalRecords + 1 }; - var value = new ValueStruct { vfield1 = i + TotalRecords, vfield2 = i + TotalRecords + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - scanCursorFuncs.Initialize(verifyKeys); - ClassicAssert.IsFalse(session.ScanCursor(ref cursor, long.MaxValue, scanCursorFuncs, long.MaxValue), "Expected scan to finish and return false, pt 1"); - ClassicAssert.AreEqual(TotalRecords * 2, scanCursorFuncs.numRecords, "Unexpected count for on-disk + in-mem"); - ClassicAssert.AreEqual(0, cursor, "Expected cursor to be 0, pt 3"); - - // Try an invalid cursor (not a multiple of 8) on-disk and verify we get one correct record. Use 3x page size to make sure page boundaries are tested. - ClassicAssert.Greater(store.hlogBase.GetTailAddress(), PageSize * 10, "Need enough space to exercise this"); - scanCursorFuncs.Initialize(verifyKeys); - cursor = store.hlogBase.BeginAddress - 1; - do - { - ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 1, scanCursorFuncs, long.MaxValue, validateCursor: true), "Expected scan to finish and return false, pt 1"); - cursor = scanCursorFuncs.lastAddress + recordSize + 1; - } while (cursor < PageSize * 3); - - // Now try an invalid cursor in-memory. First we have to read what's at the target start address (let's use HeadAddress) to find what the value is. - InputStruct input = default; - OutputStruct output = default; - ReadOptions readOptions = default; - var readStatus = bContext.ReadAtAddress(store.hlogBase.HeadAddress, ref input, ref output, ref readOptions, out _); - ClassicAssert.IsTrue(readStatus.Found, $"Could not read at HeadAddress; {readStatus}"); - - scanCursorFuncs.Initialize(verifyKeys); - scanCursorFuncs.numRecords = (int)output.value.vfield1; - cursor = store.Log.HeadAddress + 1; - do - { - ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 1, scanCursorFuncs, long.MaxValue, validateCursor: true), "Expected scan to finish and return false, pt 1"); - cursor = scanCursorFuncs.lastAddress + recordSize + 1; - } while (cursor < store.hlogBase.HeadAddress + PageSize * 3); - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - - public void BlittableScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) - { - var recordSize = BlittableAllocatorImpl.RecordSize; - - using var session = store.NewSession(new ScanFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < TotalRecords; i++) - { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value); - } - - var scanCursorFuncs = new ScanCursorFuncs(); - - long cursor = 0; - scanCursorFuncs.Initialize(verifyKeys: false, k => k.kfield1 % 10 == 0); - ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 10, scanCursorFuncs, store.Log.TailAddress), "ScanCursor failed, pt 1"); - ClassicAssert.AreEqual(10, scanCursorFuncs.numRecords, "count at first 10"); - ClassicAssert.Greater(cursor, 0, "Expected cursor to be > 0, pt 1"); - - // Now fake out the key verification to make it think we got all the previous keys; this ensures we are aligned as expected. - scanCursorFuncs.Initialize(verifyKeys: true, k => true); - scanCursorFuncs.numRecords = 91; // (filter accepts: 0-9) * 10 + 1 - ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 100, scanCursorFuncs, store.Log.TailAddress), "ScanCursor failed, pt 2"); - ClassicAssert.AreEqual(191, scanCursorFuncs.numRecords, "count at second 100"); - ClassicAssert.Greater(cursor, 0, "Expected cursor to be > 0, pt 1"); - } - - internal sealed class ScanCursorFuncs : IScanIteratorFunctions - { - internal int numRecords; - internal long lastAddress; - internal bool verifyKeys; - internal Func filter; - - internal void Initialize(bool verifyKeys) => Initialize(verifyKeys, k => true); - - internal void Initialize(bool verifyKeys, Func filter) - { - numRecords = 0; - this.verifyKeys = verifyKeys; - this.filter = filter; - } - - public bool ConcurrentReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = filter(key) ? CursorRecordResult.Accept : CursorRecordResult.Skip; - if (cursorRecordResult != CursorRecordResult.Accept) - return true; - - if (verifyKeys) - ClassicAssert.AreEqual(numRecords, key.kfield1, "Mismatched key field on Scan"); - ClassicAssert.Greater(recordMetadata.Address, 0); - ++numRecords; - lastAddress = recordMetadata.Address; - return true; - } - - public void OnException(Exception exception, long numberOfRecords) - => Assert.Fail($"Unexpected exception at {numberOfRecords} records: {exception.Message}"); - - public bool OnStart(long beginAddress, long endAddress) => true; - - public void OnStop(bool completed, long numberOfRecords) { } - - public bool SingleReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - } - - class LogObserver : IObserver> - { - int val = 0; - - public void OnCompleted() - { - ClassicAssert.AreEqual(TotalRecords, val); - } - - public void OnError(Exception error) - { - } - - public void OnNext(ITsavoriteScanIterator iter) - { - while (iter.GetNext(out _, out KeyStruct key, out ValueStruct value)) - { - ClassicAssert.AreEqual(val, key.kfield1); - ClassicAssert.AreEqual(val + 1, key.kfield2); - ClassicAssert.AreEqual(val, value.vfield1); - ClassicAssert.AreEqual(val + 1, value.vfield2); - val++; - } - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/CancellationTests.cs b/libs/storage/Tsavorite/cs/test/CancellationTests.cs index 9736b2f1b04..ac670c64f5c 100644 --- a/libs/storage/Tsavorite/cs/test/CancellationTests.cs +++ b/libs/storage/Tsavorite/cs/test/CancellationTests.cs @@ -1,8 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,12 +11,11 @@ namespace Tsavorite.test.Cancellation { - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; - - [AllureNUnit] + // Use an int in these tests just to get a different length underlying the SpanByte + using IntAllocator = SpanByteAllocator>; + using IntStoreFunctions = StoreFunctions; [TestFixture] - class CancellationTests : AllureTestBase + class CancellationTests : TestBase { internal enum CancelLocation { @@ -26,16 +25,16 @@ internal enum CancelLocation NeedCopyUpdate, CopyUpdater, InPlaceUpdater, - SingleWriter, - ConcurrentWriter + InitialWriter, + InPlaceWriter } - public class CancellationFunctions : SessionFunctionsBase + public class CancellationFunctions : SessionFunctionsBase { internal CancelLocation cancelLocation = CancelLocation.None; internal CancelLocation lastFunc = CancelLocation.None; - public override bool NeedInitialUpdate(ref int key, ref int input, ref int output, ref RMWInfo rmwInfo) + public override bool NeedInitialUpdate(TKey key, ref int input, ref int output, ref RMWInfo rmwInfo) { lastFunc = CancelLocation.NeedInitialUpdate; if (cancelLocation == CancelLocation.NeedInitialUpdate) @@ -46,7 +45,7 @@ public override bool NeedInitialUpdate(ref int key, ref int input, ref int outpu return true; } - public override bool NeedCopyUpdate(ref int key, ref int input, ref int oldValue, ref int output, ref RMWInfo rmwInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref int input, ref int output, ref RMWInfo rmwInfo) { lastFunc = CancelLocation.NeedCopyUpdate; if (cancelLocation == CancelLocation.NeedCopyUpdate) @@ -58,7 +57,7 @@ public override bool NeedCopyUpdate(ref int key, ref int input, ref int oldValue } /// - public override bool CopyUpdater(ref int key, ref int input, ref int oldValue, ref int newValue, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) { lastFunc = CancelLocation.CopyUpdater; ClassicAssert.AreNotEqual(CancelLocation.NeedCopyUpdate, cancelLocation); @@ -67,11 +66,10 @@ public override bool CopyUpdater(ref int key, ref int input, ref int oldValue, r rmwInfo.Action = RMWAction.CancelOperation; return false; } - newValue = oldValue; - return true; + return dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo); } - public override bool InitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) { lastFunc = CancelLocation.InitialUpdater; ClassicAssert.AreNotEqual(CancelLocation.NeedInitialUpdate, cancelLocation); @@ -81,11 +79,10 @@ public override bool InitialUpdater(ref int key, ref int input, ref int value, r rmwInfo.Action = RMWAction.CancelOperation; return false; } - value = input; - return true; + return logRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref input), in sizeInfo); } - public override bool InPlaceUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref int input, ref int output, ref RMWInfo rmwInfo) { lastFunc = CancelLocation.InPlaceUpdater; if (cancelLocation == CancelLocation.InPlaceUpdater) @@ -93,41 +90,54 @@ public override bool InPlaceUpdater(ref int key, ref int input, ref int value, r rmwInfo.Action = RMWAction.CancelOperation; return false; } - value = input; - return true; + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + return logRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref input), in sizeInfo); } // Upsert functions - public override bool SingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref int input, ReadOnlySpan srcValue, ref int output, ref UpsertInfo upsertInfo) { - lastFunc = CancelLocation.SingleWriter; - if (cancelLocation == CancelLocation.SingleWriter) + lastFunc = CancelLocation.InitialWriter; + if (cancelLocation == CancelLocation.InitialWriter) { upsertInfo.Action = UpsertAction.CancelOperation; return false; } - dst = src; - return true; + return logRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); } - public override bool ConcurrentWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref int input, ReadOnlySpan srcValue, ref int output, ref UpsertInfo upsertInfo) { - lastFunc = CancelLocation.ConcurrentWriter; - if (cancelLocation == CancelLocation.ConcurrentWriter) + lastFunc = CancelLocation.InPlaceWriter; + if (cancelLocation == CancelLocation.InPlaceWriter) { upsertInfo.Action = UpsertAction.CancelOperation; return false; } - dst = src; - return true; + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); } + + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref int input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(int) }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref int input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(int) }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref int input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref int input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; } IDevice log; CancellationFunctions functions; - TsavoriteKV store; - ClientSession session; - BasicContext bContext; + TsavoriteKV store; + ClientSession session; + BasicContext bContext; const int NumRecs = 100; @@ -141,14 +151,14 @@ public void Setup() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 17, + LogMemorySize = 1L << 17, PageSize = 1L << 12 - }, StoreFunctions.Create(IntKeyComparer.Instance) + }, StoreFunctions.Create(IntKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); functions = new CancellationFunctions(); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -167,8 +177,11 @@ public void TearDown() private unsafe void Populate() { // Single alloc outside the loop, to the max length we'll need. - for (int ii = 0; ii < NumRecs; ii++) - _ = bContext.Upsert(ii, ii * NumRecs * 10); + for (int keyNum = 0; keyNum < NumRecs; keyNum++) + { + var valueNum = keyNum * NumRecs * 10; + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref valueNum)); + } } [Test] @@ -178,15 +191,17 @@ public void InitialUpdaterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) { Populate(); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - int key = NumRecs; + int keyNum = NumRecs, valueNum = keyNum * NumRecs * 10; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); functions.cancelLocation = CancelLocation.NeedInitialUpdate; - var status = bContext.RMW(key, key * NumRecs * 10); + var status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.IsCanceled); ClassicAssert.AreEqual(CancelLocation.NeedInitialUpdate, functions.lastFunc); functions.cancelLocation = CancelLocation.InitialUpdater; - status = bContext.RMW(key, key * NumRecs * 10); + valueNum *= 2; + status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.IsCanceled); ClassicAssert.AreEqual(CancelLocation.InitialUpdater, functions.lastFunc); } @@ -198,19 +213,20 @@ public void CopyUpdaterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) { Populate(); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - int key = NumRecs / 2; + int keyNum = NumRecs / 2, valueNum = keyNum * NumRecs * 10; void do_it() { + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); for (int lap = 0; lap < 2; ++lap) { functions.cancelLocation = CancelLocation.NeedCopyUpdate; - var status = bContext.RMW(key, key * NumRecs * 10); + var status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.IsCanceled); ClassicAssert.AreEqual(CancelLocation.NeedCopyUpdate, functions.lastFunc); functions.cancelLocation = CancelLocation.CopyUpdater; - status = bContext.RMW(key, key * NumRecs * 10); + status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.IsCanceled); ClassicAssert.AreEqual(CancelLocation.CopyUpdater, functions.lastFunc); } @@ -231,11 +247,12 @@ public void InPlaceUpdaterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) { Populate(); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - int key = NumRecs / 2; + int keyNum = NumRecs / 2, valueNum = keyNum * NumRecs * 10; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); // Note: ExpirationTests tests the combination of CancelOperation and DeleteRecord functions.cancelLocation = CancelLocation.InPlaceUpdater; - var status = bContext.RMW(key, key * NumRecs * 10); + var status = bContext.RMW(key, ref valueNum); ClassicAssert.IsTrue(status.IsCanceled); ClassicAssert.AreEqual(CancelLocation.InPlaceUpdater, functions.lastFunc); } @@ -243,31 +260,35 @@ public void InPlaceUpdaterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) [Test] [Category("TsavoriteKV")] [Category("Smoke"), Category("RMW")] - public void SingleWriterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) + public void InitialWriterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) { Populate(); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - int key = NumRecs + 1; + int keyNum = NumRecs + 1, valueNum = keyNum * NumRecs * 10; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); - functions.cancelLocation = CancelLocation.SingleWriter; - var status = bContext.Upsert(key, key * NumRecs * 10); + functions.cancelLocation = CancelLocation.InitialWriter; + var status = bContext.Upsert(key, value); ClassicAssert.IsTrue(status.IsCanceled); - ClassicAssert.AreEqual(CancelLocation.SingleWriter, functions.lastFunc); + ClassicAssert.AreEqual(CancelLocation.InitialWriter, functions.lastFunc); } [Test] [Category("TsavoriteKV")] [Category("Smoke"), Category("RMW")] - public void ConcurrentWriterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) + public void InPlaceWriterTest([Values(Phase.REST, Phase.PREPARE)] Phase phase) { Populate(); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - int key = NumRecs / 2; + int keyNum = NumRecs / 2, valueNum = keyNum * NumRecs * 10; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); - functions.cancelLocation = CancelLocation.ConcurrentWriter; - var status = bContext.Upsert(key, key * NumRecs * 10); + functions.cancelLocation = CancelLocation.InPlaceWriter; + var status = bContext.Upsert(key, value); ClassicAssert.IsTrue(status.IsCanceled); - ClassicAssert.AreEqual(CancelLocation.ConcurrentWriter, functions.lastFunc); + ClassicAssert.AreEqual(CancelLocation.InPlaceWriter, functions.lastFunc); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/CompletePendingTests.cs b/libs/storage/Tsavorite/cs/test/CompletePendingTests.cs index 58df9447d87..ba90cebe78a 100644 --- a/libs/storage/Tsavorite/cs/test/CompletePendingTests.cs +++ b/libs/storage/Tsavorite/cs/test/CompletePendingTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,13 +13,27 @@ namespace Tsavorite.test { // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - public class LocalKeyStructComparer : IKeyComparer + public class LocalKeyStructComparer : IKeyComparer { internal long? forceCollisionHash; - public long GetHashCode64(ref KeyStruct key) => forceCollisionHash.HasValue ? forceCollisionHash.Value : Utility.GetHashCode(key.kfield1); - - public bool Equals(ref KeyStruct k1, ref KeyStruct k2) => k1.kfield1 == k2.kfield1 && k1.kfield2 == k2.kfield2; + public long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => forceCollisionHash ?? Utility.GetHashCode(key.KeyBytes.AsRef().kfield1); + + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef().kfield1 == k2.KeyBytes.AsRef().kfield1 && k1.KeyBytes.AsRef().kfield2 == k2.KeyBytes.AsRef().kfield2; public override string ToString() => $"forceHashCollision: {forceCollisionHash}"; } @@ -28,16 +41,14 @@ public class LocalKeyStructComparer : IKeyComparer namespace Tsavorite.test { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - class CompletePendingTests : AllureTestBase + class CompletePendingTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; - LocalKeyStructComparer comparer = new(); + readonly LocalKeyStructComparer comparer = new(); [SetUp] public void Setup() @@ -50,8 +61,8 @@ public void Setup() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 29 - }, StoreFunctions.Create(comparer) + LogMemorySize = 1L << 29 + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -74,7 +85,7 @@ public void TearDown() static InputStruct NewInputStruct(int key) => new() { ifield1 = key + NumRecords * 30, ifield2 = key + NumRecords * 40 }; static ContextStruct NewContextStruct(int key) => new() { cfield1 = key + NumRecords * 50, cfield2 = key + NumRecords * 60 }; - static void VerifyStructs(int key, ref KeyStruct keyStruct, ref InputStruct inputStruct, ref OutputStruct outputStruct, ref ContextStruct contextStruct, bool useRMW) + static void VerifyStructs(int key, in KeyStruct keyStruct, in InputStruct inputStruct, ref OutputStruct outputStruct, ref ContextStruct contextStruct, bool useRMW) { ClassicAssert.AreEqual(key, keyStruct.kfield1); ClassicAssert.AreEqual(key + NumRecords * 10, keyStruct.kfield2); @@ -91,7 +102,7 @@ static void VerifyStructs(int key, ref KeyStruct keyStruct, ref InputStruct inpu class ProcessPending { // Get the first chunk of outputs as a group, testing realloc. - private int deferredPendingMax = CompletedOutputIterator.kInitialAlloc + 1; + private int deferredPendingMax = CompletedOutputIterator.kInitialAlloc + 1; private int deferredPending = 0; internal Dictionary keyAddressDict = []; private bool isFirst = true; @@ -113,11 +124,11 @@ internal bool DeferPending() return false; } - internal void Process(CompletedOutputIterator completedOutputs, List<(KeyStruct, long)> rmwCopyUpdatedAddresses) + internal void Process(CompletedOutputIterator completedOutputs, List<(KeyStruct, long)> rmwCopyUpdatedAddresses) { var useRMW = rmwCopyUpdatedAddresses is not null; - ClassicAssert.AreEqual(CompletedOutputIterator.kInitialAlloc * - CompletedOutputIterator.kReallocMultuple, completedOutputs.vector.Length); + ClassicAssert.AreEqual(CompletedOutputIterator.kInitialAlloc * + CompletedOutputIterator.kReallocMultuple, completedOutputs.vector.Length); ClassicAssert.AreEqual(deferredPending, completedOutputs.maxIndex); ClassicAssert.AreEqual(-1, completedOutputs.currentIndex); @@ -125,11 +136,11 @@ internal void Process(CompletedOutputIterator().kfield1, in result.Key.KeyBytes.AsRef(), in result.Input, ref result.Output, ref result.Context, useRMW); if (!useRMW) - ClassicAssert.AreEqual(keyAddressDict[(int)result.Key.kfield1], result.RecordMetadata.Address); - else if (keyAddressDict[(int)result.Key.kfield1] != result.RecordMetadata.Address) - rmwCopyUpdatedAddresses.Add((result.Key, result.RecordMetadata.Address)); + ClassicAssert.AreEqual(keyAddressDict[(int)result.Key.KeyBytes.AsRef().kfield1], result.RecordMetadata.Address); + else if (keyAddressDict[(int)result.Key.KeyBytes.AsRef().kfield1] != result.RecordMetadata.Address) + rmwCopyUpdatedAddresses.Add((result.Key.KeyBytes.AsRef(), result.RecordMetadata.Address)); } completedOutputs.Dispose(); ClassicAssert.AreEqual(deferredPending + 1, count); @@ -146,11 +157,11 @@ internal void VerifyNoDeferredPending() ClassicAssert.AreEqual(0, deferredPending); } - internal static void VerifyOneNotFound(CompletedOutputIterator completedOutputs, ref KeyStruct keyStruct) + internal static void VerifyOneNotFound(CompletedOutputIterator completedOutputs, ref KeyStruct keyStruct) { ClassicAssert.IsTrue(completedOutputs.Next()); ClassicAssert.IsFalse(completedOutputs.Current.Status.Found); - ClassicAssert.AreEqual(keyStruct, completedOutputs.Current.Key); + ClassicAssert.AreEqual(keyStruct, completedOutputs.Current.Key.KeyBytes.AsRef()); ClassicAssert.IsFalse(completedOutputs.Next()); completedOutputs.Dispose(); } @@ -160,7 +171,7 @@ internal static void VerifyOneNotFound(CompletedOutputIterator>(new FunctionsWithContext()); + using var session = store.NewSession>(new FunctionsWithContext()); var bContext = session.BasicContext; ClassicAssert.IsNull(session.completedOutputs); // Do not instantiate until we need it @@ -171,7 +182,7 @@ public async ValueTask ReadAndCompleteWithPendingOutput([Values] bool useRMW) var keyStruct = NewKeyStruct(key); var valueStruct = NewValueStruct(key); processPending.keyAddressDict[key] = store.Log.TailAddress; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); } // Flush to make reads or RMWs go pending. @@ -190,9 +201,9 @@ public async ValueTask ReadAndCompleteWithPendingOutput([Values] bool useRMW) { var ksUnfound = keyStruct; ksUnfound.kfield1 += NumRecords * 10; - if (bContext.Read(ref ksUnfound, ref inputStruct, ref outputStruct, contextStruct).IsPending) + if (bContext.Read(ksUnfound, ref inputStruct, ref outputStruct, contextStruct).IsPending) { - CompletedOutputIterator completedOutputs; + CompletedOutputIterator completedOutputs; if ((key & 1) == 0) completedOutputs = await bContext.CompletePendingWithOutputsAsync().ConfigureAwait(false); else @@ -203,8 +214,8 @@ public async ValueTask ReadAndCompleteWithPendingOutput([Values] bool useRMW) // We don't use context (though we verify it), and Read does not use input. var status = useRMW - ? bContext.RMW(ref keyStruct, ref inputStruct, ref outputStruct, contextStruct) - : bContext.Read(ref keyStruct, ref inputStruct, ref outputStruct, contextStruct); + ? bContext.RMW(keyStruct, ref inputStruct, ref outputStruct, contextStruct) + : bContext.Read(keyStruct, ref inputStruct, ref outputStruct, contextStruct); if (status.IsPending) { if (processPending.IsFirst()) @@ -216,7 +227,7 @@ public async ValueTask ReadAndCompleteWithPendingOutput([Values] bool useRMW) if (!processPending.DeferPending()) { - CompletedOutputIterator completedOutputs; + CompletedOutputIterator completedOutputs; if ((key & 1) == 0) completedOutputs = await bContext.CompletePendingWithOutputsAsync().ConfigureAwait(false); else @@ -235,37 +246,34 @@ public async ValueTask ReadAndCompleteWithPendingOutput([Values] bool useRMW) foreach (var (key, address) in rmwCopyUpdatedAddresses) { - // ConcurrentReader does not verify the input struct. + // Reader does not verify the input struct. InputStruct inputStruct = default; OutputStruct outputStruct = default; ReadOptions readOptions = default; // This should not be pending since we've not flushed. var localKey = key; - var status = bContext.Read(ref localKey, ref inputStruct, ref outputStruct, ref readOptions, out RecordMetadata recordMetadata); + var status = bContext.Read(localKey, ref inputStruct, ref outputStruct, ref readOptions, out RecordMetadata recordMetadata); ClassicAssert.IsFalse(status.IsPending); ClassicAssert.AreEqual(address, recordMetadata.Address); } } - public class PendingReadFunctions : SessionFunctionsBase + public class PendingReadFunctions : SessionFunctionsBase { - public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key.kfield1, output.value.vfield1); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().kfield1, output.value.vfield1); // Do not compare field2; that's our updated value, and the key won't be found if we change kfield2 } // Read functions - public override bool SingleReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref ReadInfo readInfo) { - ClassicAssert.IsFalse(readInfo.RecordInfo.IsNull()); - dst.value = value; + ClassicAssert.IsFalse(srcLogRecord.Info.IsNull); + output.value = srcLogRecord.ValueSpan.AsRef(); return true; } - - public override bool ConcurrentReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => SingleReader(ref key, ref input, ref value, ref dst, ref readInfo); } [Test] @@ -274,29 +282,29 @@ public void ReadPendingWithNewSameKey([Values(FlushMode.NoFlush, FlushMode.OnDis { const int valueMult = 1000; - using var session = store.NewSession>(new PendingReadFunctions()); + using var session = store.NewSession>(new PendingReadFunctions()); var bContext = session.BasicContext; // Insert first record var firstValue = 0; // same as key var keyStruct = new KeyStruct { kfield1 = firstValue, kfield2 = firstValue * valueMult }; var valueStruct = new ValueStruct { vfield1 = firstValue, vfield2 = firstValue * valueMult }; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); // Flush to make the Read() go pending. store.Log.FlushAndEvict(wait: true); - var (status, outputStruct) = bContext.Read(keyStruct); + var (status, _ /*outputStruct*/) = bContext.Read(keyStruct); ClassicAssert.IsTrue(status.IsPending, $"Expected status.IsPending: {status}"); // Insert next record with the same key and flush this too if requested. var secondValue = firstValue + 1; valueStruct.vfield2 = secondValue * valueMult; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); if (secondRecordFlushMode == FlushMode.OnDisk) store.Log.FlushAndEvict(wait: true); - (status, outputStruct) = bContext.GetSinglePendingResult(); + (status, var outputStruct) = bContext.GetSinglePendingResult(); ClassicAssert.AreEqual(secondValue * valueMult, outputStruct.value.vfield2, "Should have returned second value"); } @@ -306,33 +314,33 @@ public void ReadPendingWithNewDifferentKeyInChain([Values(FlushMode.NoFlush, Flu { const int valueMult = 1000; - using var session = store.NewSession>(new PendingReadFunctions()); + using var session = store.NewSession>(new PendingReadFunctions()); var bContext = session.BasicContext; // Insert first record var firstValue = 0; // same as key var keyStruct = new KeyStruct { kfield1 = firstValue, kfield2 = firstValue * valueMult }; var valueStruct = new ValueStruct { vfield1 = firstValue, vfield2 = firstValue * valueMult }; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); // Force collisions to test having another key in the chain - comparer.forceCollisionHash = comparer.GetHashCode64(ref keyStruct); + comparer.forceCollisionHash = comparer.GetHashCode64(keyStruct); // Flush to make the Read() go pending. store.Log.FlushAndEvict(wait: true); - var (status, outputStruct) = bContext.Read(keyStruct); + var (status, _ /*outputStruct*/) = bContext.Read(keyStruct); ClassicAssert.IsTrue(status.IsPending, $"Expected status.IsPending: {status}"); // Insert next record with a different key and flush this too if requested. var secondValue = firstValue + 1; keyStruct = new() { kfield1 = secondValue, kfield2 = secondValue * valueMult }; valueStruct = new() { vfield1 = secondValue, vfield2 = secondValue * valueMult }; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); if (secondRecordFlushMode == FlushMode.OnDisk) store.Log.FlushAndEvict(wait: true); - (status, outputStruct) = bContext.GetSinglePendingResult(); + (status, var outputStruct) = bContext.GetSinglePendingResult(); ClassicAssert.AreEqual(firstValue * valueMult, outputStruct.value.vfield2, "Should have returned first value"); } @@ -343,22 +351,23 @@ public void ReadPendingWithNoNewKey() // Basic test of pending read const int valueMult = 1000; - using var session = store.NewSession>(new PendingReadFunctions()); + using var session = store.NewSession>(new PendingReadFunctions()); var bContext = session.BasicContext; // Insert first record var firstValue = 0; // same as key var keyStruct = new KeyStruct { kfield1 = firstValue, kfield2 = firstValue * valueMult }; var valueStruct = new ValueStruct { vfield1 = firstValue, vfield2 = firstValue * valueMult }; - _ = bContext.Upsert(ref keyStruct, ref valueStruct); + _ = bContext.Upsert(keyStruct, SpanByte.FromPinnedVariable(ref valueStruct)); // Flush to make the Read() go pending. store.Log.FlushAndEvict(wait: true); - var (status, outputStruct) = bContext.Read(keyStruct); + var (status, _ /*outputStruct*/) = bContext.Read(keyStruct); ClassicAssert.IsTrue(status.IsPending, $"Expected status.IsPending: {status}"); - (status, outputStruct) = bContext.GetSinglePendingResult(); + (status, var outputStruct) = bContext.GetSinglePendingResult(); + ClassicAssert.IsTrue(status.Found, $"Expected status.Found: {status}"); ClassicAssert.AreEqual(firstValue * valueMult, outputStruct.value.vfield2, "Should have returned first value"); } } diff --git a/libs/storage/Tsavorite/cs/test/ConcurrentCounterTests.cs b/libs/storage/Tsavorite/cs/test/ConcurrentCounterTests.cs index 92a15844619..03fcf70b2ae 100644 --- a/libs/storage/Tsavorite/cs/test/ConcurrentCounterTests.cs +++ b/libs/storage/Tsavorite/cs/test/ConcurrentCounterTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -9,9 +8,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - public class ConcurrentCounterTests : AllureTestBase + public class ConcurrentCounterTests : TestBase { [Test] public void Increment_IncreasesCounterValue() diff --git a/libs/storage/Tsavorite/cs/test/DeltaLogTests.cs b/libs/storage/Tsavorite/cs/test/DeltaLogTests.cs deleted file mode 100644 index 4f1fbc17fd0..00000000000 --- a/libs/storage/Tsavorite/cs/test/DeltaLogTests.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Tsavorite.test -{ - [AllureNUnit] - [TestFixture] - internal class DeltaLogStandAloneTests : AllureTestBase - { - private TsavoriteLog log; - private IDevice device; - - [SetUp] - public void Setup() - { - // Clean up log files from previous test runs in case they weren't cleaned up - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - } - - [TearDown] - public void TearDown() - { - log?.Dispose(); - log = null; - device?.Dispose(); - device = null; - - // Clean up log files - TestUtils.OnTearDown(waitForDelete: true); - } - - [Test] - [Category("TsavoriteLog")] - [Category("Smoke")] - public void DeltaLogTest1([Values] TestUtils.TestDeviceType deviceType) - { - const int TotalCount = 200; - string filename = Path.Join(TestUtils.MethodTestDir, $"delta_{deviceType}.log"); - TestUtils.RecreateDirectory(TestUtils.MethodTestDir); - - device = TestUtils.CreateTestDevice(deviceType, filename); - device.Initialize(-1); - using DeltaLog deltaLog = new DeltaLog(device, 12, 0); - Random r = new(20); - int i; - - SectorAlignedBufferPool bufferPool = new(1, (int)device.SectorSize); - deltaLog.InitializeForWrites(bufferPool); - for (i = 0; i < TotalCount; i++) - { - int _len = 1 + r.Next(254); - long address; - while (true) - { - deltaLog.Allocate(out int maxLen, out address); - if (_len <= maxLen) break; - deltaLog.Seal(0); - } - for (int j = 0; j < _len; j++) - { - unsafe { *(byte*)(address + j) = (byte)_len; } - } - deltaLog.Seal(_len, i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA); - } - deltaLog.FlushAsync().Wait(); - - deltaLog.InitializeForReads(); - r = new(20); - for (i = 0; deltaLog.GetNext(out long address, out int len, out var type); i++) - { - int _len = 1 + r.Next(254); - ClassicAssert.AreEqual(i % 2 == 0 ? DeltaLogEntryType.DELTA : DeltaLogEntryType.CHECKPOINT_METADATA, type); - ClassicAssert.AreEqual(len, _len); - for (int j = 0; j < len; j++) - { - unsafe { ClassicAssert.AreEqual((byte)_len, *(byte*)(address + j)); } - } - } - ClassicAssert.AreEqual(TotalCount, i, $"i={i} and TotalCount={TotalCount}"); - bufferPool.Free(); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/Directory.Build.props b/libs/storage/Tsavorite/cs/test/Directory.Build.props new file mode 100644 index 00000000000..6ab027dce47 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/Directory.Build.props @@ -0,0 +1,6 @@ + + + + $(MSBuildThisFileDirectory)tsavorite.runsettings + + diff --git a/libs/storage/Tsavorite/cs/test/EnqueueAndWaitForCommit.cs b/libs/storage/Tsavorite/cs/test/EnqueueAndWaitForCommit.cs index 2d1e68b6a50..2d978ced0c3 100644 --- a/libs/storage/Tsavorite/cs/test/EnqueueAndWaitForCommit.cs +++ b/libs/storage/Tsavorite/cs/test/EnqueueAndWaitForCommit.cs @@ -3,7 +3,6 @@ using System; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,9 +10,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class EnqWaitCommitTest : AllureTestBase + internal class EnqWaitCommitTest : TestBase { const int entryLength = 500; const int numEntries = 100; @@ -70,9 +68,7 @@ public async ValueTask EnqueueWaitCommitBasicTest([Values] EnqueueIteratorType i { // Set Default entry data for (int i = 0; i < entryLength; i++) - { entry[i] = (byte)i; - } // Add to TsavoriteLog on a separate thread, which will wait for the commit from this thread var currentTask = Task.Run(() => LogWriter(log, entry, iteratorType)); @@ -85,7 +81,7 @@ public async ValueTask EnqueueWaitCommitBasicTest([Values] EnqueueIteratorType i await currentTask.ConfigureAwait(false); // Read the log - Look for the flag so know each entry is unique - using var iter = log.Scan(0, 1000); + using var iter = log.Scan(0, LogAddress.MaxValidAddress); int currentEntry = 0; while (iter.GetNext(out byte[] result, out _, out _)) { diff --git a/libs/storage/Tsavorite/cs/test/EnqueueTests.cs b/libs/storage/Tsavorite/cs/test/EnqueueTests.cs index 12cca4532f4..bbad14137b2 100644 --- a/libs/storage/Tsavorite/cs/test/EnqueueTests.cs +++ b/libs/storage/Tsavorite/cs/test/EnqueueTests.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,9 +12,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class EnqueueTests : AllureTestBase + internal class EnqueueTests : TestBase { private TsavoriteLog log; private IDevice device; @@ -137,7 +135,7 @@ public void EnqueueBasicTest([Values] EnqueueIteratorType iteratorType, [Values] // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -145,13 +143,9 @@ public void EnqueueBasicTest([Values] EnqueueIteratorType iteratorType, [Values] { // Span Batch only added first entry several times so have separate verification if (iteratorType == EnqueueIteratorType.SpanBatch) - { ClassicAssert.AreEqual((byte)entryFlag, result[0]); - } else - { ClassicAssert.AreEqual((byte)entryFlag, result[currentEntry]); - } currentEntry++; } @@ -160,10 +154,8 @@ public void EnqueueBasicTest([Values] EnqueueIteratorType iteratorType, [Values] // Make sure expected length (entryLength) is same as current - also makes sure that data verification was not skipped ClassicAssert.AreEqual(entryLength, currentEntry); - } - [Test] [Category("TsavoriteLog")] [Category("Smoke")] diff --git a/libs/storage/Tsavorite/cs/test/EpochProtectedVersionScheme.cs b/libs/storage/Tsavorite/cs/test/EpochProtectedVersionScheme.cs index bb8f14777a1..d004b2a79a6 100644 --- a/libs/storage/Tsavorite/cs/test/EpochProtectedVersionScheme.cs +++ b/libs/storage/Tsavorite/cs/test/EpochProtectedVersionScheme.cs @@ -286,7 +286,7 @@ private bool MakeTransition(VersionSchemeState expectedState, VersionSchemeState { if (Interlocked.CompareExchange(ref state.Word, nextState.Word, expectedState.Word) != expectedState.Word) return false; - Debug.WriteLine("Moved to {0}, {1}", nextState.Phase, nextState.Version); + Debug.WriteLine("EPVS: Moved to {0}, {1}", nextState.Phase, nextState.Version); return true; } diff --git a/libs/storage/Tsavorite/cs/test/ExpirationTests.cs b/libs/storage/Tsavorite/cs/test/ExpirationTests.cs index aeaf2726761..43e164e8fa1 100644 --- a/libs/storage/Tsavorite/cs/test/ExpirationTests.cs +++ b/libs/storage/Tsavorite/cs/test/ExpirationTests.cs @@ -1,9 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; +using System.Runtime.InteropServices; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,11 +12,9 @@ namespace Tsavorite.test.Expiration { - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] + using SpanByteStoreFunctions = StoreFunctions; [TestFixture] - internal class ExpirationTests : AllureTestBase + internal class ExpirationTests : TestBase { const int StackAllocMax = 12; const int NumRecs = 5000; @@ -36,8 +34,7 @@ internal class ExpirationTests : AllureTestBase [Flags] internal enum Funcs { - Invalid = 0, NeedInitialUpdate = 0x0001, NeedCopyUpdate = 0x0002, InPlaceUpdater = 0x0004, InitialUpdater = 0x0008, CopyUpdater = 0x0010, - SingleReader = 0x0020, ConcurrentReader = 0x0040, + Invalid = 0, NeedInitialUpdate = 0x0001, NeedCopyUpdate = 0x0002, InPlaceUpdater = 0x0004, InitialUpdater = 0x0008, CopyUpdater = 0x0010, Reader = 0x0020, RMWCompletionCallback = 0x0100, ReadCompletionCallback = 0x0200, SkippedCopyUpdate = NeedCopyUpdate | RMWCompletionCallback, DidCopyUpdate = NeedCopyUpdate | CopyUpdater, @@ -153,23 +150,16 @@ internal enum TestOp Revivify // TODO - NYI: An Update or RMW operation encounters a tombstoned record of >= size of the new value, so the record is updated. // Test with newsize < space, then again with newsize == original space - // Verify tombstone is revivified on later insert (SingleWriter called within Tsavorite-acquired RecordInfo.SpinLock) + // Verify tombstone is revivified on later insert (InitialWriter called within Tsavorite-acquired RecordInfo.SpinLock) // Verify tombstone is revivified on later simple RMW (IU called within Tsavorite-acquired RecordInfo.SpinLock) #pragma warning restore format }; - public class ExpirationFunctions : SessionFunctionsBase + public class ExpirationFunctions : SessionFunctionsBase { - private static unsafe void VerifyValue(int key, ref SpanByte valueSpanByte) - { - Span valueSpan = valueSpanByte.AsSpan(); - for (int j = 0; j < valueSpan.Length; j++) - ClassicAssert.AreEqual(key, valueSpan[j]); - } - - static bool IsExpired(int key, int value) => value == GetValue(key) + 2; + static bool ShouldExpire(int key, int value) => value == GetValue(key) + 2; - public override unsafe bool NeedInitialUpdate(ref SpanByte key, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) + public override bool NeedInitialUpdate(TKey key, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) { output.AddFunc(Funcs.NeedInitialUpdate); switch (input.testOp) @@ -214,9 +204,9 @@ public override unsafe bool NeedInitialUpdate(ref SpanByte key, ref ExpirationIn } } - public override unsafe bool NeedCopyUpdate(ref SpanByte key, ref ExpirationInput input, ref SpanByte oldValue, ref ExpirationOutput output, ref RMWInfo rmwInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord logRecord, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) { - int field1 = oldValue.AsSpan()[0]; + int field1 = logRecord.ValueSpan.UncheckedCast()[0]; output.AddFunc(Funcs.NeedCopyUpdate); switch (input.testOp) @@ -251,11 +241,11 @@ public override unsafe bool NeedCopyUpdate(ref SpanByte key, ref ExpirationInput } /// - public override unsafe bool CopyUpdater(ref SpanByte key, ref ExpirationInput input, ref SpanByte oldValue, ref SpanByte newValue, ref ExpirationOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) { - int key1 = key.AsSpan()[0]; - int oldField1 = oldValue.AsSpan()[0]; - ref int newField1 = ref newValue.AsSpan()[0]; + int key1 = srcLogRecord.Key.AsRef(); + var oldField1 = srcLogRecord.ValueSpan.AsRef(); + ref int newField1 = ref dstLogRecord.ValueSpan.AsRef(); output.AddFunc(Funcs.CopyUpdater); switch (input.testOp) @@ -346,12 +336,12 @@ public override unsafe bool CopyUpdater(ref SpanByte key, ref ExpirationInput in } } - public override unsafe bool InitialUpdater(ref SpanByte key, ref ExpirationInput input, ref SpanByte value, ref ExpirationOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) { - ref int field1 = ref value.AsSpan()[0]; + ref int newField1 = ref logRecord.ValueSpan.AsRef(); output.AddFunc(Funcs.InitialUpdater); - field1 = input.value; + newField1 = input.value; // If InPlaceUpdater returned Delete, let the caller know both operations happened. Similarly, we may be output.result = output.result switch @@ -361,7 +351,7 @@ public override unsafe bool InitialUpdater(ref SpanByte key, ref ExpirationInput ExpirationResult.DeletedThenUpdateRejected => ExpirationResult.DeletedThenInserted, _ => ExpirationResult.Updated }; - output.retrievedValue = field1; + output.retrievedValue = newField1; // If this is the first InitialUpdater after a Delete and the testOp is *ThenInsert, we have to fail the first InitialUpdater // (which is the InitialUpdater call on the deleted record's space) and will pass the second InitialUpdater (which is into a new record). @@ -371,65 +361,65 @@ public override unsafe bool InitialUpdater(ref SpanByte key, ref ExpirationInput return true; } - public override unsafe bool InPlaceUpdater(ref SpanByte key, ref ExpirationInput input, ref SpanByte value, ref ExpirationOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref ExpirationInput input, ref ExpirationOutput output, ref RMWInfo rmwInfo) { - int key1 = key.AsSpan()[0]; - ref int field1 = ref value.AsSpan()[0]; + int key1 = logRecord.Key.AsRef(); + ref int newField1 = ref logRecord.ValueSpan.AsRef(); output.AddFunc(Funcs.InPlaceUpdater); switch (input.testOp) { case TestOp.Increment: - ClassicAssert.AreEqual(GetValue(key1), field1); + ClassicAssert.AreEqual(GetValue(key1), newField1); goto case TestOp.PassiveExpire; case TestOp.PassiveExpire: - ++field1; + ++newField1; output.result = ExpirationResult.Incremented; return true; case TestOp.ExpireDelete: - ClassicAssert.AreEqual(GetValue(key1) + 1, field1); // For this test we only call this operation when the value will expire - ++field1; + ClassicAssert.AreEqual(GetValue(key1) + 1, newField1); // For this test we only call this operation when the value will expire + ++newField1; rmwInfo.Action = RMWAction.ExpireAndStop; output.result = ExpirationResult.ExpireDelete; return false; case TestOp.ExpireRollover: - ClassicAssert.AreEqual(GetValue(key1) + 1, field1); // For this test we only call this operation when the value will expire - field1 = GetValue(key1); + ClassicAssert.AreEqual(GetValue(key1) + 1, newField1); // For this test we only call this operation when the value will expire + newField1 = GetValue(key1); output.result = ExpirationResult.ExpireRollover; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.SetIfKeyExists: - field1 = input.value; + newField1 = input.value; output.result = ExpirationResult.Updated; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.SetIfKeyNotExists: // No-op return true; case TestOp.SetIfValueEquals: - if (field1 == input.comparisonValue) + if (newField1 == input.comparisonValue) { - field1 = input.value; + newField1 = input.value; output.result = ExpirationResult.Updated; } else output.result = ExpirationResult.NotUpdated; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.SetIfValueNotEquals: - if (field1 != input.comparisonValue) + if (newField1 != input.comparisonValue) { - field1 = input.value; + newField1 = input.value; output.result = ExpirationResult.Updated; } else output.result = ExpirationResult.NotUpdated; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.DeleteIfValueEqualsThenUpdate: case TestOp.DeleteIfValueEqualsThenInsert: case TestOp.DeleteIfValueEqualsAndStop: - if (field1 == input.comparisonValue) + if (newField1 == input.comparisonValue) { // Both "ThenXxx" options will go to InitialUpdater; that will decide whether to return false rmwInfo.Action = input.testOp == TestOp.DeleteIfValueEqualsAndStop ? RMWAction.ExpireAndStop : RMWAction.ExpireAndResume; @@ -437,13 +427,13 @@ public override unsafe bool InPlaceUpdater(ref SpanByte key, ref ExpirationInput return false; } output.result = ExpirationResult.NotDeleted; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.DeleteIfValueNotEqualsThenUpdate: case TestOp.DeleteIfValueNotEqualsThenInsert: case TestOp.DeleteIfValueNotEqualsAndStop: - if (field1 != input.comparisonValue) + if (newField1 != input.comparisonValue) { // Both "ThenXxx" options will go to InitialUpdater; that will decide whether to return false rmwInfo.Action = input.testOp == TestOp.DeleteIfValueNotEqualsAndStop ? RMWAction.ExpireAndStop : RMWAction.ExpireAndResume; @@ -451,7 +441,7 @@ public override unsafe bool InPlaceUpdater(ref SpanByte key, ref ExpirationInput return false; } output.result = ExpirationResult.NotDeleted; - output.retrievedValue = field1; + output.retrievedValue = newField1; return true; case TestOp.Revivify: Assert.Fail($"{input.testOp} should not get here"); @@ -462,48 +452,36 @@ public override unsafe bool InPlaceUpdater(ref SpanByte key, ref ExpirationInput } } - public override void RMWCompletionCallback(ref SpanByte key, ref ExpirationInput input, ref ExpirationOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref ExpirationInput input, ref ExpirationOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { output.AddFunc(Funcs.RMWCompletionCallback); } - public override void ReadCompletionCallback(ref SpanByte key, ref ExpirationInput input, ref ExpirationOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref ExpirationInput input, ref ExpirationOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { output.AddFunc(Funcs.ReadCompletionCallback); } /// - public override int GetRMWModifiedValueLength(ref SpanByte value, ref ExpirationInput input) => value.TotalSize; + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref ExpirationInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = srcLogRecord.ValueSpan.Length, ValueIsObject = false }; /// - public override int GetRMWInitialValueLength(ref ExpirationInput input) => MinValueLen; + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref ExpirationInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = MinValueLen, ValueIsObject = false }; /// - public override int GetUpsertValueLength(ref SpanByte value, ref ExpirationInput input) => value.TotalSize; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref ExpirationInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length, ValueIsObject = false }; // Read functions - public override unsafe bool SingleReader(ref SpanByte key, ref ExpirationInput input, ref SpanByte value, ref ExpirationOutput output, ref ReadInfo readInfo) - { - int key1 = key.AsSpan()[0]; - ref int field1 = ref value.AsSpan()[0]; - - output.AddFunc(Funcs.SingleReader); - if (IsExpired(key1, field1)) - { - readInfo.Action = ReadAction.Expire; - return false; - } - output.retrievedValue = field1; - return true; - } - - public override unsafe bool ConcurrentReader(ref SpanByte key, ref ExpirationInput input, ref SpanByte value, ref ExpirationOutput output, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref ExpirationInput input, ref ExpirationOutput output, ref ReadInfo readInfo) { - int key1 = key.AsSpan()[0]; - ref int field1 = ref value.AsSpan()[0]; + int key1 = srcLogRecord.Key.AsRef(); + ref int field1 = ref srcLogRecord.ValueSpan.AsRef(); - output.AddFunc(Funcs.ConcurrentReader); - if (IsExpired(key1, field1)) + output.AddFunc(Funcs.Reader); + if (ShouldExpire(key1, field1)) { readInfo.Action = ReadAction.Expire; return false; @@ -513,18 +491,22 @@ public override unsafe bool ConcurrentReader(ref SpanByte key, ref ExpirationInp } // Upsert functions - public override bool SingleWriter(ref SpanByte key, ref ExpirationInput input, ref SpanByte src, ref SpanByte dst, ref ExpirationOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo); + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ExpirationInput input, ReadOnlySpan srcValue, ref ExpirationOutput output, ref UpsertInfo upsertInfo) + => dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); - public override bool ConcurrentWriter(ref SpanByte key, ref ExpirationInput input, ref SpanByte src, ref SpanByte dst, ref ExpirationOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - => SpanByteFunctions.DoSafeCopy(ref src, ref dst, ref upsertInfo, ref recordInfo); + public override bool InPlaceWriter(ref LogRecord dstLogRecord, ref ExpirationInput input, ReadOnlySpan srcValue, ref ExpirationOutput output, ref UpsertInfo upsertInfo) + { + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(dstLogRecord, srcValue, ref input) }; + dstLogRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo); + } } IDevice log; ExpirationFunctions functions; - TsavoriteKV> store; - ClientSession> session; - BasicContext> bContext; + TsavoriteKV> store; + ClientSession> session; + BasicContext> bContext; [SetUp] public void Setup() @@ -536,14 +518,14 @@ public void Setup() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 19, + LogMemorySize = 1L << 19, PageSize = 1L << 14 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); functions = new ExpirationFunctions(); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -559,7 +541,7 @@ public void TearDown() TestUtils.OnTearDown(); } - private unsafe void Populate(Random rng) + private void Populate(Random rng) { // Single alloc outside the loop, to the max length we'll need. Span keySpan = stackalloc int[1]; @@ -568,28 +550,28 @@ private unsafe void Populate(Random rng) for (int i = 0; i < NumRecs; i++) { keySpan[0] = i; - var keySpanByte = keySpan.AsSpanByte(); + var keySpanByte = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); var valueLen = GetRandomLength(rng); for (int j = 0; j < valueLen; j++) valueSpan[j] = GetValue(i); - var valueSpanByte = valueSpan.AsSpanByte(); + var valueSpanByte = MemoryMarshal.Cast(valueSpan); - bContext.Upsert(ref keySpanByte, ref valueSpanByte, Empty.Default); + _ = bContext.Upsert(keySpanByte, valueSpanByte, Empty.Default); } } - private unsafe ExpirationOutput GetRecord(int key, Status expectedStatus, FlushMode flushMode) + private ExpirationOutput GetRecord(int key, Status expectedStatus, FlushMode flushMode) { Span keySpan = [key]; - var keySpanByte = keySpan.AsSpanByte(); + var keySpanByte = MemoryMarshal.Cast(keySpan); ExpirationOutput output = new(); - var status = bContext.Read(ref keySpanByte, ref output, Empty.Default); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(keySpanByte), ref output, Empty.Default); if (status.IsPending) { ClassicAssert.AreNotEqual(FlushMode.NoFlush, flushMode); - bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); (status, output) = GetSinglePendingResult(completedOutputs); } @@ -600,19 +582,19 @@ private unsafe ExpirationOutput GetRecord(int key, Status expectedStatus, FlushM private unsafe ExpirationOutput ExecuteRMW(int key, ref ExpirationInput input, FlushMode flushMode, Status expectedStatus = default) { Span keySpan = [key]; - var keySpanByte = keySpan.AsSpanByte(); + var keySpanByte = MemoryMarshal.Cast(keySpan); ExpirationOutput output = new(); - var status = bContext.RMW(ref keySpanByte, ref input, ref output); + var status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(keySpanByte), ref input, ref output); if (status.IsPending) { ClassicAssert.AreNotEqual(FlushMode.NoFlush, flushMode); - bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); (status, output) = GetSinglePendingResult(completedOutputs); } ClassicAssert.AreEqual(expectedStatus, status, status.ToString()); - ClassicAssert.AreEqual(expectedStatus.Expired, status.Expired, status.ToString()); + ClassicAssert.AreEqual(expectedStatus.IsExpired, status.IsExpired, status.ToString()); return output; } @@ -637,9 +619,9 @@ private void InitialRead(FlushMode flushMode, bool afterIncrement) ClassicAssert.AreEqual(GetValue(ModifyKey) + (afterIncrement ? 1 : 0), output.retrievedValue); Funcs expectedFuncs = flushMode switch { - FlushMode.NoFlush => Funcs.ConcurrentReader, - FlushMode.ReadOnly => Funcs.SingleReader, - FlushMode.OnDisk => Funcs.SingleReader | Funcs.ReadCompletionCallback, + FlushMode.NoFlush => Funcs.Reader, + FlushMode.ReadOnly => Funcs.Reader, + FlushMode.OnDisk => Funcs.Reader | Funcs.ReadCompletionCallback, _ => Funcs.Invalid }; ClassicAssert.AreNotEqual(expectedFuncs, Funcs.Invalid, $"Unexpected flushmode {flushMode}"); @@ -710,7 +692,7 @@ public void PassiveExpireTest([Values] FlushMode flushMode, [Values(Phase.REST, MaybeEvict(flushMode); IncrementValue(TestOp.PassiveExpire, flushMode); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - GetRecord(ModifyKey, new(StatusCode.NotFound | StatusCode.Expired), flushMode); + _ = GetRecord(ModifyKey, new(StatusCode.NotFound | StatusCode.Expired), flushMode); } [Test] @@ -737,10 +719,9 @@ public void ExpireDeleteTest([Values] FlushMode flushMode, [Values(Phase.REST, P ClassicAssert.AreEqual(ExpirationResult.ExpireDelete, output.result); // Verify it's not there - if (flushMode == FlushMode.NoFlush) - GetRecord(key, new(StatusCode.NotFound), flushMode); // Expiration was IPU-deletion - else - GetRecord(key, new(StatusCode.NotFound | StatusCode.Expired), flushMode); + _ = flushMode == FlushMode.NoFlush + ? GetRecord(key, new(StatusCode.NotFound), flushMode) // Expiration was IPU-deletion + : GetRecord(key, new(StatusCode.NotFound | StatusCode.Expired), flushMode); } [Test] @@ -813,7 +794,7 @@ public void SetIfKeyExistsTest([Values] FlushMode flushMode, [Values(Phase.REST, ClassicAssert.AreEqual(expectedFuncs, output.functionsCalled); // Verify it's not there - GetRecord(key, new(StatusCode.NotFound), flushMode); + _ = GetRecord(key, new(StatusCode.NotFound), flushMode); } [Test] @@ -1068,7 +1049,7 @@ public void DeleteAndCancelIfValueEqualsTest([Values(Phase.REST, Phase.PREPARE)] ClassicAssert.AreEqual(ExpirationResult.Deleted, output.result); // Verify it's not there - GetRecord(key, new(StatusCode.NotFound), flushMode); + _ = GetRecord(key, new(StatusCode.NotFound), flushMode); // Value doesn't equal - no-op key += 1; // We deleted ModifyKey so get the next-higher key @@ -1095,7 +1076,7 @@ public void DeleteIfValueNotEqualsTest([Values] FlushMode flushMode, [Values(Pha // For this test, IPU will Cancel rather than go to the IPU path Status expectedFoundRmwStatus = flushMode == FlushMode.NoFlush ? new(StatusCode.InPlaceUpdatedRecord | StatusCode.Expired) : new(StatusCode.CreatedRecord | StatusCode.Expired); - ClassicAssert.IsTrue(expectedFoundRmwStatus.Expired, expectedFoundRmwStatus.ToString()); + ClassicAssert.IsTrue(expectedFoundRmwStatus.IsExpired, expectedFoundRmwStatus.ToString()); VerifyKeyNotCreated(testOp, flushMode); session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); @@ -1126,7 +1107,7 @@ public void DeleteIfValueNotEqualsTest([Values] FlushMode flushMode, [Values(Pha ClassicAssert.AreEqual(ExpirationResult.Deleted, output.result); // Verify it's not there - GetRecord(key, new(StatusCode.NotFound), flushMode); + _ = GetRecord(key, new(StatusCode.NotFound), flushMode); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/FixedLengthTransactionalKeyStruct.cs b/libs/storage/Tsavorite/cs/test/FixedLengthTransactionalKeyStruct.cs new file mode 100644 index 00000000000..1eb617b1a8a --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/FixedLengthTransactionalKeyStruct.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using Tsavorite.core; + +namespace Tsavorite.test +{ + /// + /// A utility class to carry a fixed-length key (blittable or object type) and its assciated info for Locking + /// + public struct FixedLengthTransactionalKeyStruct : ITransactionalKey + { + /// + /// The key that is acquiring or releasing a lock + /// + public TestSpanByteKey Key; + + #region ITransactionalKey + /// + public long KeyHash { get; set; } + + /// + public LockType LockType { get; set; } + #endregion ITransactionalKey + + /// + /// Constructor + /// + public FixedLengthTransactionalKeyStruct(ReadOnlySpan key, LockType lockType, ITsavoriteContext context) + { + Key = TestSpanByteKey.FromPinnedSpan(key); + LockType = lockType; + KeyHash = context.GetKeyHash(Key); + } + + /// + /// Constructor + /// + public FixedLengthTransactionalKeyStruct(ReadOnlySpan key, long keyHash, LockType lockType, ITransactionalContext context) + { + Key = TestSpanByteKey.FromPinnedSpan(key); + KeyHash = keyHash; + LockType = lockType; + } + + /// + /// Sort the passed key array for use in + /// and + /// + /// + /// + public static void Sort(FixedLengthTransactionalKeyStruct[] keys, ITransactionalContext context) => context.SortKeyHashes(keys); + + /// + public override string ToString() + { + var hashStr = Utility.GetHashString(KeyHash); + return $"key {Key}, hash {hashStr}, {LockType}"; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/FunctionPerSessionTests.cs b/libs/storage/Tsavorite/cs/test/FunctionPerSessionTests.cs deleted file mode 100644 index a52cc81348d..00000000000 --- a/libs/storage/Tsavorite/cs/test/FunctionPerSessionTests.cs +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using System.Threading; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Tsavorite.test -{ - // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - public struct RefCountedValueStruct - { - public int ReferenceCount; - public long Value; - } -} - -namespace Tsavorite.test -{ - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - public class RefCountedAdder : SessionFunctionsBase - { - public int InitialCount; - public int InPlaceCount; - public int CopyCount; - - public override bool InitialUpdater(ref int key, ref long input, ref RefCountedValueStruct value, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref InitialCount); - - value.Value = input; - value.ReferenceCount = 1; - return true; - } - - public override bool InPlaceUpdater(ref int key, ref long input, ref RefCountedValueStruct value, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref InPlaceCount); - - value.Value = input; - value.ReferenceCount++; - return true; - } - - public override bool CopyUpdater(ref int key, ref long input, ref RefCountedValueStruct oldValue, ref RefCountedValueStruct newValue, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref CopyCount); - - newValue.Value = input; - newValue.ReferenceCount = oldValue.ReferenceCount + 1; - return true; - } - } - - public class RefCountedRemover : SessionFunctionsBase - { - public int InitialCount; - public int InPlaceCount; - public int CopyCount; - - public override bool InitialUpdater(ref int key, ref Empty input, ref RefCountedValueStruct value, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref InitialCount); - - value.Value = 0; - value.ReferenceCount = 0; - return true; - } - - public override bool InPlaceUpdater(ref int key, ref Empty input, ref RefCountedValueStruct value, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref InPlaceCount); - - if (value.ReferenceCount > 0) - value.ReferenceCount--; - - return true; - } - - public override bool CopyUpdater(ref int key, ref Empty input, ref RefCountedValueStruct oldValue, ref RefCountedValueStruct newValue, ref Empty output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - _ = Interlocked.Increment(ref CopyCount); - - newValue.ReferenceCount = oldValue.ReferenceCount; - if (newValue.ReferenceCount > 0) - newValue.ReferenceCount--; - newValue.Value = oldValue.Value; - return true; - } - } - - public class RefCountedReader : SessionFunctionsBase - { - public override bool SingleReader(ref int key, ref Empty input, ref RefCountedValueStruct value, ref RefCountedValueStruct dst, ref ReadInfo readInfo) - { - dst = value; - return true; - } - - public override bool ConcurrentReader(ref int key, ref Empty input, ref RefCountedValueStruct value, ref RefCountedValueStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst = value; - return true; - } - } - - [AllureNUnit] - [TestFixture] - public class FunctionPerSessionTests : AllureTestBase - { - private IDevice _log; - private TsavoriteKV store; - private RefCountedAdder _adder; - private RefCountedRemover _remover; - private RefCountedReader _reader; - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - _log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "FunctionPerSessionTests1.log"), deleteOnClose: true); - - store = new(new() - { - IndexSize = 1L << 13, - LogDevice = _log, - }, StoreFunctions.Create(IntKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - _adder = new RefCountedAdder(); - _remover = new RefCountedRemover(); - _reader = new RefCountedReader(); - } - - [TearDown] - public void TearDown() - { - store?.Dispose(); - store = null; - _log?.Dispose(); - _log = null; - TestUtils.OnTearDown(); - } - - [Test] - [Category("TsavoriteKV")] - public void Should_create_multiple_sessions_with_different_callbacks() - { - using var adderSession = store.NewSession(_adder); - using var removerSession = store.NewSession(_remover); - using var readerSession = store.NewSession(_reader); - var key = 101; - var input = 1000L; - - _ = adderSession.BasicContext.RMW(ref key, ref input); - _ = adderSession.BasicContext.RMW(ref key, ref input); - _ = adderSession.BasicContext.RMW(ref key, ref input); - - ClassicAssert.AreEqual(1, _adder.InitialCount); - ClassicAssert.AreEqual(2, _adder.InPlaceCount); - - var empty = default(Empty); - _ = removerSession.BasicContext.RMW(ref key, ref empty); - - ClassicAssert.AreEqual(1, _remover.InPlaceCount); - - RefCountedValueStruct output = new(); - _ = readerSession.BasicContext.Read(ref key, ref output); - - ClassicAssert.AreEqual(2, output.ReferenceCount); - ClassicAssert.AreEqual(1000L, output.Value); - - store.Log.FlushAndEvict(true); - - _ = removerSession.BasicContext.RMW(ref key, ref empty); - _ = removerSession.BasicContext.CompletePending(wait: true); - _ = readerSession.BasicContext.Read(ref key, ref empty, ref output); - - ClassicAssert.AreEqual(1, output.ReferenceCount); - ClassicAssert.AreEqual(1000L, output.Value); - ClassicAssert.AreEqual(1, _remover.CopyCount); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/GenericByteArrayTests.cs b/libs/storage/Tsavorite/cs/test/GenericByteArrayTests.cs deleted file mode 100644 index 04da411fd6b..00000000000 --- a/libs/storage/Tsavorite/cs/test/GenericByteArrayTests.cs +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.IO; -using System.Linq; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Tsavorite.test -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class GenericByteArrayTests : AllureTestBase - { - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; - private IDevice log, objlog; - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "GenericStringTests.log"), deleteOnClose: true); - objlog = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "GenericStringTests.obj.log"), deleteOnClose: true); - - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - MemorySize = 1L << 14, - PageSize = 1L << 9 - }, StoreFunctions.Create(new ByteArrayEC(), () => new ByteArrayBinaryObjectSerializer(), () => new ByteArrayBinaryObjectSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - session = store.NewSession(new MyByteArrayFuncs()); - bContext = session.BasicContext; - } - - [TearDown] - public void TearDown() - { - session?.Dispose(); - session = null; - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - objlog?.Dispose(); - objlog = null; - - TestUtils.OnTearDown(); - } - - private static byte[] GetByteArray(int i) - { - return BitConverter.GetBytes(i); - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void ByteArrayBasicTest() - { - const int totalRecords = 2000; - for (int i = 0; i < totalRecords; i++) - { - var _key = GetByteArray(i); - var _value = GetByteArray(i); - _ = bContext.Upsert(ref _key, ref _value, Empty.Default); - } - _ = bContext.CompletePending(true); - - for (int i = 0; i < totalRecords; i++) - { - byte[] input = default; - byte[] output = default; - var key = GetByteArray(i); - var value = GetByteArray(i); - - if (bContext.Read(ref key, ref input, ref output, Empty.Default).IsPending) - _ = bContext.CompletePending(true); - else - ClassicAssert.IsTrue(output.SequenceEqual(value)); - } - } - - class MyByteArrayFuncs : SimpleSimpleFunctions - { - public override void ReadCompletionCallback(ref byte[] key, ref byte[] input, ref byte[] output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - ClassicAssert.IsTrue(output.SequenceEqual(key)); - } - } - } - - class ByteArrayEC : IKeyComparer - { - public bool Equals(ref byte[] k1, ref byte[] k2) - { - return k1.SequenceEqual(k2); - } - - public unsafe long GetHashCode64(ref byte[] k) - { - fixed (byte* b = k) - { - return Utility.HashBytes(b, k.Length); - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/GenericIterationTests.cs b/libs/storage/Tsavorite/cs/test/GenericIterationTests.cs deleted file mode 100644 index e1154a474ac..00000000000 --- a/libs/storage/Tsavorite/cs/test/GenericIterationTests.cs +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class GenericIterationTests : AllureTestBase - { - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; - private IDevice log, objlog; - - [SetUp] - public void Setup() - { - DeleteDirectory(MethodTestDir, wait: true); - // Tests call InternalSetup() - } - - private void InternalSetup(bool largeMemory) - { - // Broke this out as we have different requirements by test. - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericIterationTests.log"), deleteOnClose: true); - objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericIterationTests.obj.log"), deleteOnClose: true); - - store = new(new() - { - IndexSize = 1L << 13, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - MemorySize = 1L << (largeMemory ? 25 : 14), - PageSize = 1L << (largeMemory ? 20 : 9) - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer(), DefaultRecordDisposer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - session = store.NewSession(new MyFunctionsDelete()); - bContext = session.BasicContext; - } - - [TearDown] - public void TearDown() - { - session?.Dispose(); - session = null; - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - objlog?.Dispose(); - objlog = null; - - OnTearDown(); - } - - internal struct GenericPushIterationTestFunctions : IScanIteratorFunctions - { - internal int keyMultToValue; - internal long numRecords; - internal int stopAt; - - public bool SingleReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - { - cursorRecordResult = CursorRecordResult.Accept; // default; not used here - if (keyMultToValue > 0) - ClassicAssert.AreEqual(key.key * keyMultToValue, value.value); - return stopAt != ++numRecords; - } - - public bool ConcurrentReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public readonly bool OnStart(long beginAddress, long endAddress) => true; - public readonly void OnException(Exception exception, long numberOfRecords) { } - public readonly void OnStop(bool completed, long numberOfRecords) { } - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - - public void GenericIterationBasicTest([Values] ScanIteratorType scanIteratorType) - { - InternalSetup(largeMemory: false); - GenericPushIterationTestFunctions scanIteratorFunctions = new(); - - const int totalRecords = 2000; - - void iterateAndVerify(int keyMultToValue, int expectedRecs) - { - scanIteratorFunctions.keyMultToValue = keyMultToValue; - scanIteratorFunctions.numRecords = 0; - - if (scanIteratorType == ScanIteratorType.Pull) - { - using var iter = session.Iterate(); - while (iter.GetNext(out var recordInfo)) - _ = scanIteratorFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), default, default, out _); - } - else - ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); - - ClassicAssert.AreEqual(expectedRecs, scanIteratorFunctions.numRecords); - } - - // Initial population - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(1, totalRecords); - - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = 2 * i }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(2, totalRecords); - - for (int i = totalRecords / 2; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(0, totalRecords); - - for (int i = 0; i < totalRecords; i += 2) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(0, totalRecords); - - for (int i = 0; i < totalRecords; i += 2) - { - var key1 = new MyKey { key = i }; - _ = bContext.Delete(ref key1); - } - iterateAndVerify(0, totalRecords / 2); - - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = 3 * i }; - _ = bContext.Upsert(ref key1, ref value); - } - iterateAndVerify(3, totalRecords); - - store.Log.FlushAndEvict(wait: true); - iterateAndVerify(3, totalRecords); - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - - public void GenericIterationPushStopTest() - { - InternalSetup(largeMemory: false); - GenericPushIterationTestFunctions scanIteratorFunctions = new(); - - const int totalRecords = 2000; - var start = store.Log.TailAddress; - - void scanAndVerify(int stopAt, bool useScan) - { - scanIteratorFunctions.numRecords = 0; - scanIteratorFunctions.stopAt = stopAt; - if (useScan) - ClassicAssert.IsFalse(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); - else - ClassicAssert.IsFalse(session.Iterate(ref scanIteratorFunctions), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); - ClassicAssert.AreEqual(stopAt, scanIteratorFunctions.numRecords); - } - - // Initial population - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); - } - - scanAndVerify(42, useScan: true); - scanAndVerify(42, useScan: false); - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public unsafe void GenericIterationPushLockTest([Values(1, 4)] int scanThreads, [Values(1, 4)] int updateThreads, [Values] ScanMode scanMode) - { - InternalSetup(largeMemory: true); - - const int totalRecords = 2000; - var start = store.Log.TailAddress; - - void LocalScan(int i) - { - using var session = store.NewSession(new MyFunctionsDelete()); - GenericPushIterationTestFunctions scanIteratorFunctions = new(); - - if (scanMode == ScanMode.Scan) - ClassicAssert.IsTrue(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to complete push scan; numRecords = {scanIteratorFunctions.numRecords}"); - else - ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); - ClassicAssert.AreEqual(totalRecords, scanIteratorFunctions.numRecords); - } - - void LocalUpdate(int tid) - { - using var session = store.NewSession(new MyFunctionsDelete()); - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = (tid + 1) * i }; - _ = bContext.Upsert(ref key1, ref value); - } - } - - { // Initial population - for (int i = 0; i < totalRecords; i++) - { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); - } - } - - List tasks = []; // Task rather than Thread for propagation of exception. - var numThreads = scanThreads + updateThreads; - for (int t = 0; t < numThreads; t++) - { - var tid = t; - if (t < scanThreads) - tasks.Add(Task.Factory.StartNew(() => LocalScan(tid))); - else - tasks.Add(Task.Factory.StartNew(() => LocalUpdate(tid))); - } - Task.WaitAll([.. tasks]); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/GenericStringTests.cs b/libs/storage/Tsavorite/cs/test/GenericStringTests.cs deleted file mode 100644 index 036def9d369..00000000000 --- a/libs/storage/Tsavorite/cs/test/GenericStringTests.cs +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test -{ - using StringAllocator = GenericAllocator>>; - using StringStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class GenericStringTests : AllureTestBase - { - private TsavoriteKV store; - private ClientSession session; - private IDevice log, objlog; - - [SetUp] - public void Setup() - { - // Clean up log files from previous test runs in case they weren't cleaned up - DeleteDirectory(MethodTestDir, wait: true); - } - - [TearDown] - public void TearDown() - { - session?.Dispose(); - session = null; - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - objlog?.Dispose(); - objlog = null; - - OnTearDown(); - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void StringBasicTest([Values] TestDeviceType deviceType) - { - string logfilename = Path.Join(MethodTestDir, "GenericStringTests" + deviceType.ToString() + ".log"); - string objlogfilename = Path.Join(MethodTestDir, "GenericStringTests" + deviceType.ToString() + ".obj.log"); - - log = CreateTestDevice(deviceType, logfilename); - objlog = CreateTestDevice(deviceType, objlogfilename); - - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - MemorySize = 1L << 14, - PageSize = 1L << 9, - SegmentSize = 1L << 22 - }, StoreFunctions.Create(StringKeyComparer.Instance, () => new StringBinaryObjectSerializer(), () => new StringBinaryObjectSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - session = store.NewSession(new MyFuncs()); - var bContext = session.BasicContext; - - const int totalRecords = 200; - for (int i = 0; i < totalRecords; i++) - { - var _key = $"{i}"; - var _value = $"{i}"; ; - _ = bContext.Upsert(ref _key, ref _value, Empty.Default); - } - _ = bContext.CompletePending(true); - ClassicAssert.AreEqual(totalRecords, store.EntryCount); - - for (int i = 0; i < totalRecords; i++) - { - string input = default; - string output = default; - var key = $"{i}"; - var value = $"{i}"; - - var status = bContext.Read(ref key, ref input, ref output, Empty.Default); - if (status.IsPending) - { - _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); - (status, output) = GetSinglePendingResult(outputs); - } - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value, output); - } - } - - class MyFuncs : SimpleSimpleFunctions - { - public override void ReadCompletionCallback(ref string key, ref string input, ref string output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key, output); - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/InputOutputParameterTests.cs b/libs/storage/Tsavorite/cs/test/InputOutputParameterTests.cs index 21c6e469fda..b22d04f4086 100644 --- a/libs/storage/Tsavorite/cs/test/InputOutputParameterTests.cs +++ b/libs/storage/Tsavorite/cs/test/InputOutputParameterTests.cs @@ -1,8 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,76 +10,86 @@ namespace Tsavorite.test.InputOutputParameterTests { - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using IntAllocator = SpanByteAllocator>; + using IntStoreFunctions = StoreFunctions; [TestFixture] - class InputOutputParameterTests : AllureTestBase + class InputOutputParameterTests : TestBase { const int AddValue = 10_000; const int MultValue = 100; const int NumRecs = 10; - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; - internal class UpsertInputFunctions : SessionFunctionsBase + internal class UpsertInputFunctions : SessionFunctionsBase { internal long lastWriteAddress; - public override bool ConcurrentReader(ref int key, ref int input, ref int value, ref int output, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - lastWriteAddress = readInfo.Address; - return SingleReader(ref key, ref input, ref value, ref output, ref readInfo); - } - /// - public override bool SingleReader(ref int key, ref int input, ref int value, ref int output, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref int input, ref int output, ref ReadInfo readInfo) { - ClassicAssert.AreEqual(key * input, value); + ClassicAssert.AreEqual(srcLogRecord.Key.AsRef() * input, srcLogRecord.ValueSpan.AsRef()); lastWriteAddress = readInfo.Address; - output = value + AddValue; + output = srcLogRecord.ValueSpan.AsRef() + AddValue; return true; } /// - public override bool ConcurrentWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - => SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, WriteReason.Upsert, ref recordInfo); + public override bool InPlaceWriter(ref LogRecord logRecord, ref int input, ReadOnlySpan src, ref int output, ref UpsertInfo upsertInfo) + { + RecordSizeInfo sizeInfo = new(); // unused by InitialWriter + return InitialWriter(ref logRecord, in sizeInfo, ref input, src, ref output, ref upsertInfo); + } /// - public override bool SingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref int input, ReadOnlySpan src, ref int output, ref UpsertInfo upsertInfo) { lastWriteAddress = upsertInfo.Address; - dst = output = src * input; + ref var value = ref logRecord.ValueSpan.AsRef(); + value = output = src.AsRef() * input; return true; } /// - public override void PostSingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reasons) + public override void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ReadOnlySpan src, ref int output, ref UpsertInfo upsertInfo) { ClassicAssert.AreEqual(lastWriteAddress, upsertInfo.Address); - ClassicAssert.AreEqual(key * input, dst); - ClassicAssert.AreEqual(dst, output); + ref var value = ref dstLogRecord.ValueSpan.AsRef(); + ClassicAssert.AreEqual(dstLogRecord.Key.AsRef() * input, value); + ClassicAssert.AreEqual(value, output); } - public override bool InPlaceUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - => InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + public override bool InPlaceUpdater(ref LogRecord logRecord, ref int input, ref int output, ref RMWInfo rmwInfo) + { + RecordSizeInfo sizeInfo = new(); // unused by InitialUpdater + return InitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); + } - public override bool InitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) { lastWriteAddress = rmwInfo.Address; - value = output = key * input; + ref var value = ref logRecord.ValueSpan.AsRef(); + value = output = logRecord.Key.AsRef() * input; return true; } + /// - public override void PostInitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo) + public override void PostInitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) { ClassicAssert.AreEqual(lastWriteAddress, rmwInfo.Address); - ClassicAssert.AreEqual(key * input, value); + ref var value = ref dstLogRecord.ValueSpan.AsRef(); + ClassicAssert.AreEqual(dstLogRecord.Key.AsRef() * input, value); ClassicAssert.AreEqual(value, output); } + + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref int input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(int), ValueIsObject = false }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref int input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(int), ValueIsObject = false }; } [SetUp] @@ -92,13 +102,13 @@ public void Setup() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 22, + LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 - }, StoreFunctions.Create(IntKeyComparer.Instance) + }, StoreFunctions.Create(IntKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new UpsertInputFunctions()); + session = store.NewSession(new UpsertInputFunctions()); bContext = session.BasicContext; } @@ -130,9 +140,10 @@ void doWrites() for (int key = 0; key < NumRecs; ++key) { var tailAddress = store.Log.TailAddress; + var upsertOptions = new UpsertOptions(); status = useRMW - ? bContext.RMW(ref key, ref input, ref output, out var recordMetadata) - : bContext.Upsert(ref key, ref input, ref key, ref output, out recordMetadata); + ? bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input, ref output, out var recordMetadata) + : bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input, SpanByte.FromPinnedVariable(ref key), ref output, ref upsertOptions, out recordMetadata); if (loading) { if (useRMW) @@ -153,7 +164,7 @@ void doReads() { for (int key = 0; key < NumRecs; ++key) { - _ = bContext.Read(ref key, ref input, ref output); + _ = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input, ref output); ClassicAssert.AreEqual(key * input + AddValue, output); } } @@ -165,7 +176,7 @@ void doReads() loading = false; input *= input; - // ConcurrentWriter (update existing records) + // InPlaceWriter (update existing records) doWrites(); doReads(); } diff --git a/libs/storage/Tsavorite/cs/test/LargeObjectTests.cs b/libs/storage/Tsavorite/cs/test/LargeObjectTests.cs deleted file mode 100644 index 6f8e520ed0c..00000000000 --- a/libs/storage/Tsavorite/cs/test/LargeObjectTests.cs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.IO; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test.largeobjects -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class LargeObjectTests : AllureTestBase - { - [SetUp] - public void Setup() => RecreateDirectory(MethodTestDir); - - [TearDown] - public void TearDown() => OnTearDown(); - - [Test] - [Category("TsavoriteKV")] - public async ValueTask LargeObjectTest( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType - ) - { - int maxSize = 100; - int numOps = 5000; - - MyInput input = default; - MyLargeOutput output = new MyLargeOutput(); - Guid token = default; - - // Step 1: Create and populate store. - using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.log"))) - using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.obj.log"))) - using (var store = new TsavoriteKV( - new() - { - IndexSize = 1L << 13, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - PageSize = 1L << 21, - MemorySize = 1L << 26, - CheckpointDir = MethodTestDir - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyLargeValueSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) - using (var session = store.NewSession(new MyLargeFunctions())) - { - var bContext = session.BasicContext; - Random r = new Random(33); - - for (int key = 0; key < numOps; key++) - { - var mykey = new MyKey { key = key }; - var value = new MyLargeValue(1 + r.Next(maxSize)); - _ = bContext.Upsert(ref mykey, ref value, Empty.Default); - } - - _ = store.TryInitiateFullCheckpoint(out token, checkpointType); - await store.CompleteCheckpointAsync().ConfigureAwait(false); - } - - // Step 1: Create and recover store. - using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.log"))) - using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.obj.log"))) - using (var store = new TsavoriteKV( - new() - { - IndexSize = 1L << 13, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - PageSize = 1L << 21, - MemorySize = 1L << 26, - CheckpointDir = MethodTestDir - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyLargeValueSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) - { - _ = store.Recover(token); - - using (var session = store.NewSession(new MyLargeFunctions())) - { - var bContext = session.BasicContext; - - for (int keycnt = 0; keycnt < numOps; keycnt++) - { - var key = new MyKey { key = keycnt }; - var status = bContext.Read(ref key, ref input, ref output, Empty.Default); - - if (status.IsPending) - (status, output) = bContext.GetSinglePendingResult(); - - for (int i = 0; i < output.value.value.Length; i++) - ClassicAssert.AreEqual((byte)(output.value.value.Length + i), output.value.value[i]); - } - } - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/LowMemoryTests.cs b/libs/storage/Tsavorite/cs/test/LowMemoryTests.cs index 6c0d0d249d0..55077cd4ba6 100644 --- a/libs/storage/Tsavorite/cs/test/LowMemoryTests.cs +++ b/libs/storage/Tsavorite/cs/test/LowMemoryTests.cs @@ -1,42 +1,40 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; +using static Tsavorite.test.TestUtils; namespace Tsavorite.test.LowMemory { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; [TestFixture] - public class LowMemoryTests : AllureTestBase + public class LowMemoryTests : TestBase { IDevice log; - TsavoriteKV store1; + TsavoriteKV store1; const int NumOps = 2000; [SetUp] public void Setup() { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - log = new LocalMemoryDevice(1L << 28, 1L << 25, 1, latencyMs: 20, fileName: Path.Join(TestUtils.MethodTestDir, "test.log")); - _ = Directory.CreateDirectory(TestUtils.MethodTestDir); + DeleteDirectory(MethodTestDir, wait: true); + log = new LocalMemoryDevice(1L << 28, 1L << 25, 1, latencyMs: 20, fileName: Path.Join(MethodTestDir, "test.log")); + _ = Directory.CreateDirectory(MethodTestDir); store1 = new(new() { IndexSize = 1L << 16, LogDevice = log, MutableFraction = 1, PageSize = 1L << 10, - MemorySize = 1L << 12, + LogMemorySize = 1L << 12, SegmentSize = 1L << 26, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -51,19 +49,19 @@ public void TearDown() TestUtils.OnTearDown(); } - private static void Populate(ClientSession, LongStoreFunctions, LongAllocator> s1) + private static void Populate(ClientSession s1) { var bContext1 = s1.BasicContext; for (long key = 0; key < NumOps; key++) - _ = bContext1.Upsert(ref key, ref key); + _ = bContext1.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); } [Test] [Category("TsavoriteKV")] - [Category(TestUtils.StressTestCategory)] + [Category(StressTestCategory)] public void LowMemConcurrentUpsertReadTest() { - using var s1 = store1.NewSession>(new SimpleSimpleFunctions((a, b) => a + b)); + using var s1 = store1.NewSession(new SimpleLongSimpleFunctions((a, b) => a + b)); var bContext1 = s1.BasicContext; Populate(s1); @@ -72,12 +70,12 @@ public void LowMemConcurrentUpsertReadTest() var numCompleted = 0; for (long key = 0; key < NumOps; key++) { - var (status, output) = bContext1.Read(key); + var (status, output) = bContext1.Read(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key))); if (!status.IsPending) { ++numCompleted; - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key, output); + ClassicAssert.IsTrue(status.Found, $"key = {key}"); + ClassicAssert.AreEqual(key, output, $"key = {key}"); } } @@ -88,7 +86,7 @@ public void LowMemConcurrentUpsertReadTest() { ++numCompleted; ClassicAssert.IsTrue(completedOutputs.Current.Status.Found, $"{completedOutputs.Current.Status}"); - ClassicAssert.AreEqual(completedOutputs.Current.Key, completedOutputs.Current.Output); + ClassicAssert.AreEqual(completedOutputs.Current.Key.KeyBytes.AsRef(), completedOutputs.Current.Output); } } ClassicAssert.AreEqual(NumOps, numCompleted, "numCompleted"); @@ -96,10 +94,10 @@ public void LowMemConcurrentUpsertReadTest() [Test] [Category("TsavoriteKV")] - [Category(TestUtils.StressTestCategory)] - public void LowMemConcurrentUpsertRMWReadTest([Values] bool completeSync) + [Category(StressTestCategory)] + public void LowMemConcurrentUpsertRMWReadTest() { - using var s1 = store1.NewSession>(new SimpleSimpleFunctions((a, b) => a + b)); + using var s1 = store1.NewSession(new SimpleLongSimpleFunctions((a, b) => a + b)); var bContext1 = s1.BasicContext; Populate(s1); @@ -108,7 +106,7 @@ public void LowMemConcurrentUpsertRMWReadTest([Values] bool completeSync) int numPending = 0; for (long key = 0; key < NumOps; key++) { - var status = bContext1.RMW(ref key, ref key); + var status = bContext1.RMW(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key)), ref key); if (status.IsPending && (++numPending % 256) == 0) { _ = bContext1.CompletePending(wait: true); @@ -122,7 +120,7 @@ public void LowMemConcurrentUpsertRMWReadTest([Values] bool completeSync) var numCompleted = 0; for (long key = 0; key < NumOps; key++) { - var (status, output) = bContext1.Read(key); + var (status, output) = bContext1.Read(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key))); if (!status.IsPending) { ++numCompleted; @@ -138,7 +136,7 @@ public void LowMemConcurrentUpsertRMWReadTest([Values] bool completeSync) { ++numCompleted; ClassicAssert.IsTrue(completedOutputs.Current.Status.Found, $"{completedOutputs.Current.Status}"); - ClassicAssert.AreEqual(completedOutputs.Current.Key * 2, completedOutputs.Current.Output); + ClassicAssert.AreEqual(completedOutputs.Current.Key.KeyBytes.AsRef() * 2, completedOutputs.Current.Output); } } ClassicAssert.AreEqual(NumOps, numCompleted, "numCompleted"); diff --git a/libs/storage/Tsavorite/cs/test/MallocFixedPageSizeTests.cs b/libs/storage/Tsavorite/cs/test/MallocFixedPageSizeTests.cs index 90b5aae4e74..393fb38c691 100644 --- a/libs/storage/Tsavorite/cs/test/MallocFixedPageSizeTests.cs +++ b/libs/storage/Tsavorite/cs/test/MallocFixedPageSizeTests.cs @@ -1,7 +1,6 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,9 +9,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class MallocFixedPageSizeTests : AllureTestBase + internal class MallocFixedPageSizeTests : TestBase { public enum AllocMode { Single, Bulk }; diff --git a/libs/storage/Tsavorite/cs/test/MiscTests.cs b/libs/storage/Tsavorite/cs/test/MiscTests.cs index 04322a2e217..d1d85537882 100644 --- a/libs/storage/Tsavorite/cs/test/MiscTests.cs +++ b/libs/storage/Tsavorite/cs/test/MiscTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,16 +11,12 @@ namespace Tsavorite.test { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class MiscTests : AllureTestBase + internal class MiscTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log, objlog; [SetUp] @@ -30,18 +25,6 @@ public void Setup() DeleteDirectory(MethodTestDir, wait: true); log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MiscTests.log"), deleteOnClose: true); objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MiscTests.obj.log"), deleteOnClose: true); - - store = new(new() - { - IndexSize = 1L << 13, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - MemorySize = 1L << 15, - PageSize = 1L << 10 - }, StoreFunctions.Create(IntKeyComparer.Instance, null, () => new MyValueSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); } [TearDown] @@ -56,79 +39,12 @@ public void TearDown() OnTearDown(); } - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void MixedTest1() - { - using var session = store.NewSession(new MixedFunctions()); - var bContext = session.BasicContext; - - int key = 8999998; - var input1 = new MyInput { value = 23 }; - MyOutput output = new(); - - _ = bContext.RMW(ref key, ref input1, Empty.Default); - - int key2 = 8999999; - var input2 = new MyInput { value = 24 }; - _ = bContext.RMW(ref key2, ref input2, Empty.Default); - - _ = bContext.Read(ref key, ref input1, ref output, Empty.Default); - ClassicAssert.AreEqual(input1.value, output.value.value); - - _ = bContext.Read(ref key2, ref input2, ref output, Empty.Default); - ClassicAssert.AreEqual(input2.value, output.value.value); - } - - [Test] - [Category("TsavoriteKV")] - public void MixedTest2() - { - using var session = store.NewSession(new MixedFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < 2000; i++) - { - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref i, ref value, Empty.Default); - } - - var key2 = 23; - MyInput input = new(); - MyOutput g1 = new(); - var status = bContext.Read(ref key2, ref input, ref g1, Empty.Default); - - if (status.IsPending) - { - _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); - (status, _) = GetSinglePendingResult(outputs); - } - ClassicAssert.IsTrue(status.Found); - - ClassicAssert.AreEqual(23, g1.value.value); - - key2 = 99999; - status = bContext.Read(ref key2, ref input, ref g1, Empty.Default); - - if (status.IsPending) - { - _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); - (status, _) = GetSinglePendingResult(outputs); - } - ClassicAssert.IsFalse(status.Found); - } - [Test] [Category("TsavoriteKV")] public void ForceRCUAndRecover([Values(UpdateOp.Upsert, UpdateOp.Delete)] UpdateOp updateOp) { var copyOnWrite = new FunctionsCopyOnWrite(); - - // FunctionsCopyOnWrite - var log = default(IDevice); - TsavoriteKV store = default; - ClientSession session = default; + ClientSession session = default; try { @@ -138,38 +54,36 @@ public void ForceRCUAndRecover([Values(UpdateOp.Upsert, UpdateOp.Delete)] Update { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) + }, StoreFunctions.Create(KeyStruct.Comparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(copyOnWrite); + session = store.NewSession(copyOnWrite); var bContext = session.BasicContext; - var key = default(KeyStruct); - var value = default(ValueStruct); + var key = new KeyStruct() { kfield1 = 1, kfield2 = 2 }; + var value = new ValueStruct() { vfield1 = 1000, vfield2 = 2000 }; var input = default(InputStruct); var output = default(OutputStruct); - key = new KeyStruct() { kfield1 = 1, kfield2 = 2 }; - value = new ValueStruct() { vfield1 = 1000, vfield2 = 2000 }; - - var status = bContext.Upsert(ref key, ref input, ref value, ref output, out RecordMetadata recordMetadata1); + var upsertOptions = new UpsertOptions(); + var status = bContext.Upsert(key, ref input, SpanByte.FromPinnedVariable(ref value), ref output, ref upsertOptions, out RecordMetadata recordMetadata1); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); - // ConcurrentWriter and InPlaceUpater return false, so we create a new record. + // InPlaceWriter and InPlaceUpater return false, so we create a new record. RecordMetadata recordMetadata2; value = new ValueStruct() { vfield1 = 1001, vfield2 = 2002 }; if (updateOp == UpdateOp.Upsert) { - status = bContext.Upsert(ref key, ref input, ref value, ref output, out recordMetadata2); - ClassicAssert.AreEqual(1, copyOnWrite.ConcurrentWriterCallCount); + status = bContext.Upsert(key, ref input, SpanByte.FromPinnedVariable(ref value), ref output, ref upsertOptions, out recordMetadata2); + ClassicAssert.AreEqual(1, copyOnWrite.InPlaceWriterCallCount); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } else { - status = bContext.RMW(ref key, ref input, ref output, out recordMetadata2); + status = bContext.RMW(key, ref input, ref output, out recordMetadata2); ClassicAssert.AreEqual(1, copyOnWrite.InPlaceUpdaterCallCount); ClassicAssert.IsTrue(status.Found && status.Record.CopyUpdated, status.ToString()); } @@ -177,44 +91,45 @@ public void ForceRCUAndRecover([Values(UpdateOp.Upsert, UpdateOp.Delete)] Update using (var iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress)) { - ClassicAssert.True(iterator.GetNext(out var info)); // We should only get the new record... - ClassicAssert.False(iterator.GetNext(out info)); // ... the old record was elided, so was Sealed and invalidated. + ClassicAssert.True(iterator.GetNext()); // We should only get the new record... + ClassicAssert.False(iterator.GetNext()); // ... the old record was elided, so was Sealed and invalidated. } - status = bContext.Read(ref key, ref output); + status = bContext.Read(key, ref output); ClassicAssert.IsTrue(status.Found, status.ToString()); _ = store.TryInitiateFullCheckpoint(out Guid token, CheckpointType.Snapshot); store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); session.Dispose(); + session = null; store.Dispose(); + store = null; store = new(new() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) + }, StoreFunctions.Create(KeyStruct.Comparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); _ = store.Recover(token); - session = store.NewSession(copyOnWrite); + session = store.NewSession(copyOnWrite); + bContext = session.BasicContext; using (var iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress)) { - ClassicAssert.True(iterator.GetNext(out var info)); // We should only get one record... - ClassicAssert.False(iterator.GetNext(out info)); // ... the old record was Unsealed by Recovery, but remains invalid. + ClassicAssert.True(iterator.GetNext()); // We should only get one record... + ClassicAssert.False(iterator.GetNext()); // ... the old record was Unsealed by Recovery, but remains invalid. } - status = bContext.Read(ref key, ref output); + status = bContext.Read(key, ref output); ClassicAssert.IsTrue(status.Found, status.ToString()); } finally { session?.Dispose(); - store?.Dispose(); - log?.Dispose(); } } } diff --git a/libs/storage/Tsavorite/cs/test/NeedCopyUpdateTests.cs b/libs/storage/Tsavorite/cs/test/NeedCopyUpdateTests.cs index 57391362826..041599b6a94 100644 --- a/libs/storage/Tsavorite/cs/test/NeedCopyUpdateTests.cs +++ b/libs/storage/Tsavorite/cs/test/NeedCopyUpdateTests.cs @@ -1,28 +1,24 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; -using static Tsavorite.test.NeedCopyUpdateTests; using static Tsavorite.test.TestUtils; namespace Tsavorite.test { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; - using RMWValueAllocator = GenericAllocator>>; - using RMWValueStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using RMWValueAllocator = ObjectAllocator>; + using RMWValueStoreFunctions = StoreFunctions; [TestFixture] - internal class NeedCopyUpdateTests : AllureTestBase + internal class NeedCopyUpdateTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log, objlog; [SetUp] @@ -38,9 +34,9 @@ public void Setup() LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10 - }, StoreFunctions.Create(IntKeyComparer.Instance, keySerializerCreator: null, () => new RMWValueSerializer()) + }, StoreFunctions.Create(IntKeyComparer.Instance, () => new RMWValueSerializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -63,7 +59,7 @@ public void TearDown() public void TryAddTest() { TryAddTestFunctions functions = new(); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var bContext = session.BasicContext; Status status; @@ -72,24 +68,24 @@ public void TryAddTest() var value2 = new RMWValueObj { value = 2 }; functions.noNeedInitialUpdater = true; - status = bContext.RMW(ref key, ref value1); // needInitialUpdater false + NOTFOUND + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value1); // needInitialUpdater false + NOTFOUND ClassicAssert.IsFalse(status.Found, status.ToString()); ClassicAssert.IsFalse(value1.flag); // InitialUpdater is not called functions.noNeedInitialUpdater = false; - status = bContext.RMW(ref key, ref value1); // InitialUpdater + NotFound + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value1); // InitialUpdater + NotFound ClassicAssert.IsFalse(status.Found, status.ToString()); ClassicAssert.IsTrue(value1.flag); // InitialUpdater is called - status = bContext.RMW(ref key, ref value2); // InPlaceUpdater + Found + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value2); // InPlaceUpdater + Found ClassicAssert.IsTrue(status.Record.InPlaceUpdated, status.ToString()); store.Log.Flush(true); - status = bContext.RMW(ref key, ref value2); // NeedCopyUpdate returns false, so RMW returns simply Found + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value2); // NeedCopyUpdate returns false, so RMW returns simply Found ClassicAssert.IsTrue(status.Found, status.ToString()); store.Log.FlushAndEvict(true); - status = bContext.RMW(ref key, ref value2, new(StatusCode.Found)); // PENDING + NeedCopyUpdate + Found + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value2, new(StatusCode.Found)); // PENDING + NeedCopyUpdate + Found ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePendingWithOutputs(out var outputs, true); @@ -98,33 +94,38 @@ public void TryAddTest() ClassicAssert.IsTrue(status.Found, status.ToString()); // NeedCopyUpdate returns false, so RMW returns simply Found // Test stored value. Should be value1 - status = bContext.Read(ref key, ref value1, ref output, new(StatusCode.Found)); + status = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value1, ref output, new(StatusCode.Found)); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePending(true); - status = bContext.Delete(ref key); + status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); _ = bContext.CompletePending(true); store.Log.FlushAndEvict(true); - status = bContext.RMW(ref key, ref value2, new(StatusCode.NotFound | StatusCode.CreatedRecord)); // PENDING + InitialUpdater + NOTFOUND + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value2, new(StatusCode.NotFound | StatusCode.CreatedRecord)); // PENDING + InitialUpdater + NOTFOUND ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePending(true); } - internal class RMWValueObj + internal class RMWValueObj : HeapObjectBase { public int value; public bool flag; + + public override IHeapObject Clone() => throw new System.NotImplementedException(); + public override void Dispose() => throw new System.NotImplementedException(); + public override void DoSerialize(BinaryWriter writer) => throw new System.NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new System.NotImplementedException(); } - internal class RMWValueSerializer : BinaryObjectSerializer + internal class RMWValueSerializer : BinaryObjectSerializer { - public override void Serialize(ref RMWValueObj value) + public override void Serialize(IHeapObject value) { - writer.Write(value.value); + writer.Write(((RMWValueObj)value).value); } - public override void Deserialize(out RMWValueObj value) + public override void Deserialize(out IHeapObject value) { value = new RMWValueObj { @@ -133,29 +134,44 @@ public override void Deserialize(out RMWValueObj value) } } - internal class TryAddTestFunctions : TryAddFunctions + internal class TryAddTestFunctions : SessionFunctionsBase { internal bool noNeedInitialUpdater; - public override bool NeedInitialUpdate(ref int key, ref RMWValueObj input, ref RMWValueObj output, ref RMWInfo rmwInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref RMWValueObj input, ref RMWValueObj output, ref ReadInfo readInfo) { - return !noNeedInitialUpdater && base.NeedInitialUpdate(ref key, ref input, ref output, ref rmwInfo); + output = (RMWValueObj)srcLogRecord.ValueObject; + return true; } - public override bool InitialUpdater(ref int key, ref RMWValueObj input, ref RMWValueObj value, ref RMWValueObj output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool NeedInitialUpdate(TKey key, ref RMWValueObj input, ref RMWValueObj output, ref RMWInfo rmwInfo) + => !noNeedInitialUpdater; + + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref RMWValueObj input, ref RMWValueObj output, ref RMWInfo rmwInfo) { input.flag = true; - _ = base.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + Assert.That(dstLogRecord.TrySetValueObjectAndPrepareOptionals(input, in sizeInfo)); + output = input; return true; } - public override bool CopyUpdater(ref int key, ref RMWValueObj input, ref RMWValueObj oldValue, ref RMWValueObj newValue, ref RMWValueObj output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref RMWValueObj input, ref RMWValueObj output, ref RMWInfo rmwInfo) + => false; + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref RMWValueObj input, ref RMWValueObj output, ref RMWInfo rmwInfo) { - Assert.Fail("CopyUpdater"); + Assert.Fail("CopyUpdater should not be called here"); return false; } - public override void RMWCompletionCallback(ref int key, ref RMWValueObj input, ref RMWValueObj output, Status ctx, Status status, RecordMetadata recordMetadata) + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref RMWValueObj input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref RMWValueObj input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + /// + + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref RMWValueObj input, ref RMWValueObj output, Status ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.AreEqual(ctx, status); @@ -163,18 +179,16 @@ public override void RMWCompletionCallback(ref int key, ref RMWValueObj input, r ClassicAssert.IsTrue(input.flag); // InitialUpdater is called. } - public override void ReadCompletionCallback(ref int key, ref RMWValueObj input, ref RMWValueObj output, Status ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref RMWValueObj input, ref RMWValueObj output, Status ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.AreEqual(output.value, input.value); } } } - - [AllureNUnit] [TestFixture] - internal class NeedCopyUpdateTestsSinglePage : AllureTestBase + internal class NeedCopyUpdateTestsSinglePage : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; const int PageSizeBits = 16; @@ -191,9 +205,9 @@ public void Setup() IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << (PageSizeBits + 1), + LogMemorySize = 1L << (PageSizeBits + 1), PageSize = 1L << PageSizeBits - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -214,7 +228,7 @@ public void TearDown() public void CopyUpdateFromHeadReadOnlyPageTest() { RMWSinglePageFunctions functions = new(); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var bContext = session.BasicContext; // Two records is the most that can "fit" into the first Constants.kFirstValueAddress "range"; therefore when we close pages @@ -222,24 +236,26 @@ public void CopyUpdateFromHeadReadOnlyPageTest() // caused the HeadAddress to be moved above logicalAddress in CreateNewRecordRMW. const int padding = 2; - for (int key = 0; key < RecsPerPage - padding; key++) + for (long key = 0; key < RecsPerPage - padding; key++) { - var status = bContext.RMW(key, key << 32 + key); + long value = ((int)key << 32) + key; + var status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); ClassicAssert.IsTrue(status.IsCompletedSuccessfully, status.ToString()); } store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); // This should trigger CopyUpdater, after flushing the oldest page (closest to HeadAddress). - for (int key = 0; key < RecsPerPage - padding; key++) + for (long key = 0; key < RecsPerPage - padding; key++) { - var status = bContext.RMW(key, key << 32 + key); + long value = ((int)key << 32) + key; + var status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); if (status.IsPending) _ = bContext.CompletePending(wait: true); } } - internal class RMWSinglePageFunctions : SimpleSimpleFunctions + internal class RMWSinglePageFunctions : SimpleLongSimpleFunctions { } } diff --git a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest2.cs b/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest2.cs deleted file mode 100644 index e21964df06e..00000000000 --- a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest2.cs +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Tsavorite.test.recovery.objects -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - public class ObjectRecoveryTests2 : AllureTestBase - { - int iterations; - - [SetUp] - public void Setup() - { - TestUtils.RecreateDirectory(TestUtils.MethodTestDir); - } - - [TearDown] - public void TearDown() - { - TestUtils.OnTearDown(); - } - - [Test] - [Category("TsavoriteKV")] - [Category("CheckpointRestore")] - [Category("Smoke")] - - public async ValueTask ObjectRecoveryTest2( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Range(300, 700, 300)] int iterations, - [Values] bool isAsync) - { - this.iterations = iterations; - Prepare(out IDevice log, out IDevice objlog, out var store, out MyContext context); - - var session1 = store.NewSession(new MyFunctions()); - Write(session1, context, store, checkpointType); - Read(session1, context, false); - session1.Dispose(); - - _ = store.TryInitiateFullCheckpoint(out _, checkpointType); - store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); - - Destroy(log, objlog, store); - - Prepare(out log, out objlog, out store, out context); - - if (isAsync) - _ = await store.RecoverAsync().ConfigureAwait(false); - else - _ = store.Recover(); - - var session2 = store.NewSession(new MyFunctions()); - Read(session2, context, true); - session2.Dispose(); - - Destroy(log, objlog, store); - } - - private static void Prepare(out IDevice log, out IDevice objlog, out TsavoriteKV store, out MyContext context) - { - log = Devices.CreateLogDevice(Path.Combine(TestUtils.MethodTestDir, "RecoverTests.log")); - objlog = Devices.CreateLogDevice(Path.Combine(TestUtils.MethodTestDir, "RecoverTests_HEAP.log")); - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - ObjectLogDevice = objlog, - SegmentSize = 1L << 12, - MemorySize = 1L << 12, - PageSize = 1L << 9, - CheckpointDir = Path.Combine(TestUtils.MethodTestDir, "check-points") - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - context = new MyContext(); - } - - private static void Destroy(IDevice log, IDevice objlog, TsavoriteKV store) - { - // Dispose Tsavorite instance and log - store.Dispose(); - log.Dispose(); - objlog.Dispose(); - } - - private void Write(ClientSession session, MyContext context, - TsavoriteKV store, CheckpointType checkpointType) - { - var bContext = session.BasicContext; - - for (int i = 0; i < iterations; i++) - { - var _key = new MyKey { key = i, name = i.ToString() }; - var value = new MyValue { value = i.ToString() }; - _ = bContext.Upsert(ref _key, ref value, context); - - if (i % 100 == 0) - { - _ = store.TryInitiateFullCheckpoint(out _, checkpointType); - store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); - } - } - } - - private void Read(ClientSession session, MyContext context, bool delete) - { - var bContext = session.BasicContext; - - for (int i = 0; i < iterations; i++) - { - MyKey key = new() { key = i, name = i.ToString() }; - MyInput input = default; - MyOutput g1 = new(); - var status = bContext.Read(ref key, ref input, ref g1, context); - - if (status.IsPending) - { - _ = bContext.CompletePending(true); - context.FinalizeRead(ref status, ref g1); - } - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(i.ToString(), g1.value.value); - } - - if (delete) - { - MyKey key = new() { key = 1, name = "1" }; - MyInput input = default; - MyOutput output = new(); - _ = bContext.Delete(ref key, context); - var status = bContext.Read(ref key, ref input, ref output, context); - - if (status.IsPending) - { - _ = bContext.CompletePending(true); - context.FinalizeRead(ref status, ref output); - } - - ClassicAssert.IsFalse(status.Found); - } - } - } - - public class MyKeySerializer : BinaryObjectSerializer - { - public override void Serialize(ref MyKey key) - { - var bytes = System.Text.Encoding.UTF8.GetBytes(key.name); - writer.Write(4 + bytes.Length); - writer.Write(key.key); - writer.Write(bytes); - } - - public override void Deserialize(out MyKey key) - { - key = new MyKey(); - var size = reader.ReadInt32(); - key.key = reader.ReadInt32(); - var bytes = new byte[size - 4]; - _ = reader.Read(bytes, 0, size - 4); - key.name = System.Text.Encoding.UTF8.GetString(bytes); - - } - } - - public class MyValueSerializer : BinaryObjectSerializer - { - public override void Serialize(ref MyValue value) - { - var bytes = System.Text.Encoding.UTF8.GetBytes(value.value); - writer.Write(bytes.Length); - writer.Write(bytes); - } - - public override void Deserialize(out MyValue value) - { - value = new MyValue(); - var size = reader.ReadInt32(); - var bytes = new byte[size]; - _ = reader.Read(bytes, 0, size); - value.value = System.Text.Encoding.UTF8.GetString(bytes); - } - } - - public class MyKey - { - public int key; - public string name; - - public struct Comparer : IKeyComparer - { - public readonly long GetHashCode64(ref MyKey key) => Utility.GetHashCode(key.key); - public readonly bool Equals(ref MyKey key1, ref MyKey key2) => key1.key == key2.key && key1.name == key2.name; - } - } - - public class MyValue { public string value; } - public class MyInput { public string value; } - public class MyOutput { public MyValue value; } - - public class MyContext - { - private Status _status; - private MyOutput _g1; - - internal void Populate(ref Status status, ref MyOutput g1) - { - _status = status; - _g1 = g1; - } - internal void FinalizeRead(ref Status status, ref MyOutput g1) - { - status = _status; - g1 = _g1; - } - } - - - public class MyFunctions : SessionFunctionsBase - { - public override bool InitialUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { value.value = input.value; return true; } - public override bool NeedCopyUpdate(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyOutput output, ref RMWInfo rmwInfo) => true; - public override bool CopyUpdater(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { newValue = oldValue; return true; } - public override bool InPlaceUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - if (value.value.Length < input.value.Length) - return false; - value.value = input.value; - return true; - } - - - public override bool SingleReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo) - { - dst.value = value; - return true; - } - - public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) { dst = src; return true; } - - public override bool ConcurrentReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst.value = value; - return true; - } - - public override bool ConcurrentWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - if (src == null) - return false; - - if (dst.value.Length != src.value.Length) - return false; - - dst = src; - return true; - } - - public override void ReadCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, MyContext ctx, Status status, RecordMetadata recordMetadata) => ctx.Populate(ref status, ref output); - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest3.cs b/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest3.cs deleted file mode 100644 index 51ce0ad8dbf..00000000000 --- a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest3.cs +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Tsavorite.test.recovery.objects -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - public class ObjectRecoveryTests3 : AllureTestBase - { - int iterations; - - [SetUp] - public void Setup() - { - TestUtils.RecreateDirectory(TestUtils.MethodTestDir); - } - - [TearDown] - public void TearDown() - { - TestUtils.OnTearDown(); - } - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public async ValueTask ObjectRecoveryTest3( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values(1000)] int iterations, - [Values] bool isAsync) - { - this.iterations = iterations; - ObjectRecoveryTests3.Prepare(out IDevice log, out IDevice objlog, out var store, out MyContext context); - - var session1 = store.NewSession(new MyFunctions()); - var tokens = Write(session1, context, store, checkpointType); - Read(session1, context, false, iterations); - session1.Dispose(); - - _ = store.TryInitiateHybridLogCheckpoint(out Guid token, checkpointType); - store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); - tokens.Add((iterations, token)); - Destroy(log, objlog, store); - - foreach (var item in tokens) - { - ObjectRecoveryTests3.Prepare(out log, out objlog, out store, out context); - - if (isAsync) - _ = await store.RecoverAsync(default, item.Item2).ConfigureAwait(false); - else - _ = store.Recover(default, item.Item2); - - var session2 = store.NewSession(new MyFunctions()); - Read(session2, context, false, item.Item1); - session2.Dispose(); - - Destroy(log, objlog, store); - } - } - - private static void Prepare(out IDevice log, out IDevice objlog, out TsavoriteKV store, out MyContext context) - { - log = Devices.CreateLogDevice(Path.Combine(TestUtils.MethodTestDir, "RecoverTests.log")); - objlog = Devices.CreateLogDevice(Path.Combine(TestUtils.MethodTestDir, "RecoverTests_HEAP.log")); - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - ObjectLogDevice = objlog, - SegmentSize = 1L << 12, - MemorySize = 1L << 12, - PageSize = 1L << 9, - CheckpointDir = Path.Combine(TestUtils.MethodTestDir, "check-points") - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - context = new MyContext(); - } - - private static void Destroy(IDevice log, IDevice objlog, TsavoriteKV store) - { - // Dispose Tsavorite instance and log - store.Dispose(); - log.Dispose(); - objlog.Dispose(); - } - - private List<(int, Guid)> Write(ClientSession session, MyContext context, - TsavoriteKV store, CheckpointType checkpointType) - { - var bContext = session.BasicContext; - - var tokens = new List<(int, Guid)>(); - for (int i = 0; i < iterations; i++) - { - var _key = new MyKey { key = i, name = string.Concat(Enumerable.Repeat(i.ToString(), 100)) }; - var value = new MyValue { value = i.ToString() }; - _ = bContext.Upsert(ref _key, ref value, context); - - if (i % 1000 == 0 && i > 0) - { - _ = store.TryInitiateHybridLogCheckpoint(out Guid token, checkpointType); - store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); - tokens.Add((i, token)); - } - } - return tokens; - } - - private static void Read(ClientSession session, MyContext context, bool delete, int iter) - { - var bContext = session.BasicContext; - - for (int i = 0; i < iter; i++) - { - var key = new MyKey { key = i, name = string.Concat(Enumerable.Repeat(i.ToString(), 100)) }; - MyInput input = default; - MyOutput g1 = new(); - var status = bContext.Read(ref key, ref input, ref g1, context); - - if (status.IsPending) - { - _ = bContext.CompletePending(true); - context.FinalizeRead(ref status, ref g1); - } - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(i.ToString(), g1.value.value); - } - - if (delete) - { - var key = new MyKey { key = 1, name = "1" }; - var input = default(MyInput); - var output = new MyOutput(); - _ = bContext.Delete(ref key, context); - var status = bContext.Read(ref key, ref input, ref output, context); - - if (status.IsPending) - { - _ = bContext.CompletePending(true); - context.FinalizeRead(ref status, ref output); - } - - ClassicAssert.IsFalse(status.Found); - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTestTypes.cs b/libs/storage/Tsavorite/cs/test/ObjectRecoveryTestTypes.cs index 8521c66c8bc..640ded36819 100644 --- a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTestTypes.cs +++ b/libs/storage/Tsavorite/cs/test/ObjectRecoveryTestTypes.cs @@ -1,45 +1,42 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.IO; using System.Threading; using Tsavorite.core; +using Tsavorite.test.recovery.sumstore; namespace Tsavorite.test.recovery.objects { - public class AdIdObj + public class NumClicksObj : HeapObjectBase { - public long adId; + public long numClicks; - public partial struct Comparer : IKeyComparer - { - public readonly long GetHashCode64(ref AdIdObj key) => Utility.GetHashCode(key.adId); + public override string ToString() => numClicks.ToString(); - public readonly bool Equals(ref AdIdObj k1, ref AdIdObj k2) => k1.adId == k2.adId; - } + public override void Dispose() { } - public class Serializer : BinaryObjectSerializer - { - public override void Deserialize(out AdIdObj obj) => obj = new AdIdObj { adId = reader.ReadInt64() }; + public override HeapObjectBase Clone() => throw new NotImplementedException(); + public override void DoSerialize(BinaryWriter writer) => throw new NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new NotImplementedException(); - public override void Serialize(ref AdIdObj obj) => writer.Write(obj.adId); + public NumClicksObj() + { + HeapMemorySize = sizeof(long); } - } - public class NumClicksObj - { - public long numClicks; - - public class Serializer : BinaryObjectSerializer + public class Serializer : BinaryObjectSerializer { - public override void Deserialize(out NumClicksObj obj) => obj = new NumClicksObj { numClicks = reader.ReadInt64() }; + public override void Deserialize(out IHeapObject obj) => obj = new NumClicksObj { numClicks = reader.ReadInt64() }; - public override void Serialize(ref NumClicksObj obj) => writer.Write(obj.numClicks); + public override void Serialize(IHeapObject obj) => writer.Write(((NumClicksObj)obj).numClicks); } } public class Input { - public AdIdObj adId; + public AdId adId; public NumClicksObj numClicks; } @@ -48,40 +45,32 @@ public class Output public NumClicksObj value; } - public class Functions : SessionFunctionsBase + public class Functions : SessionFunctionsBase { // Read functions - public override bool SingleReader(ref AdIdObj key, ref Input input, ref NumClicksObj value, ref Output dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref Input input, ref Output output, ref ReadInfo readInfo) { - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref AdIdObj key, ref Input input, ref NumClicksObj value, ref Output dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst.value = value; + output.value = (NumClicksObj)srcLogRecord.ValueObject; return true; } // RMW functions - public override bool InitialUpdater(ref AdIdObj key, ref Input input, ref NumClicksObj value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value = input.numClicks; - return true; - } + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) + => dstLogRecord.TrySetValueObject(input.numClicks); - public override bool InPlaceUpdater(ref AdIdObj key, ref Input input, ref NumClicksObj value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref Input input, ref Output output, ref RMWInfo rmwInfo) { - _ = Interlocked.Add(ref value.numClicks, input.numClicks.numClicks); + _ = Interlocked.Add(ref ((NumClicksObj)logRecord.ValueObject).numClicks, input.numClicks.numClicks); return true; } - public override bool NeedCopyUpdate(ref AdIdObj key, ref Input input, ref NumClicksObj oldValue, ref Output output, ref RMWInfo rmwInfo) => true; + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Input input, ref Output output, ref RMWInfo rmwInfo) + => dstLogRecord.TrySetValueObject(new NumClicksObj { numClicks = ((NumClicksObj)srcLogRecord.ValueObject).numClicks + input.numClicks.numClicks }); - public override bool CopyUpdater(ref AdIdObj key, ref Input input, ref NumClicksObj oldValue, ref NumClicksObj newValue, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = new NumClicksObj { numClicks = oldValue.numClicks + input.numClicks.numClicks }; - return true; - } + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref Input input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref Input input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ObjectTestTypes.cs b/libs/storage/Tsavorite/cs/test/ObjectTestTypes.cs index 83d746e844a..0b9078947c7 100644 --- a/libs/storage/Tsavorite/cs/test/ObjectTestTypes.cs +++ b/libs/storage/Tsavorite/cs/test/ObjectTestTypes.cs @@ -1,234 +1,195 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Runtime.InteropServices; +using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; namespace Tsavorite.test { - public class MyKey + public enum TestValueStyle : byte { None, Inline, Overflow, Object }; + + public struct TestObjectKey : IKey { public int key; - public override string ToString() => key.ToString(); + // Not always pinned, so don't assume it is + public readonly bool IsPinned => false; - public struct Comparer : IKeyComparer - { - public long GetHashCode64(ref MyKey key) => Utility.GetHashCode(key.key); + [UnscopedRef] + public readonly ReadOnlySpan KeyBytes => MemoryMarshal.Cast(new ReadOnlySpan(in key)); - public bool Equals(ref MyKey k1, ref MyKey k2) => k1.key == k2.key; - } - } + /// + public bool HasNamespace => false; - public class MyKeySerializer : BinaryObjectSerializer - { - public override void Deserialize(out MyKey obj) => obj = new MyKey { key = reader.ReadInt32() }; + /// + public ReadOnlySpan NamespaceBytes => []; - public override void Serialize(ref MyKey obj) => writer.Write(obj.key); - } - - public class MyValue - { - public int value; + /// + public override readonly string ToString() => key.ToString(); - public override string ToString() => value.ToString(); - - public struct Comparer : IKeyComparer // This Value comparer is used by a test + public struct Comparer : IKeyComparer { - public long GetHashCode64(ref MyValue k) => Utility.GetHashCode(k.value); + public readonly long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(key.KeyBytes.AsRef().key); - public bool Equals(ref MyValue k1, ref MyValue k2) => k1.value == k2.value; + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef().key == k2.KeyBytes.AsRef().key; } } - public class MyValueSerializer : BinaryObjectSerializer - { - public override void Deserialize(out MyValue obj) => obj = new MyValue { value = reader.ReadInt32() }; - - public override void Serialize(ref MyValue obj) => writer.Write(obj.value); - } - - public class MyInput + public class TestObjectValue : HeapObjectBase { public int value; public override string ToString() => value.ToString(); - } - public class MyOutput - { - public MyValue value; + public override void Dispose() { } - public override string ToString() => value.ToString(); - } + public override HeapObjectBase Clone() => new TestObjectValue() { value = value }; + public override void DoSerialize(BinaryWriter writer) => throw new NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new NotImplementedException(); - public class MyFunctions : SessionFunctionsBase - { - public override bool InitialUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public TestObjectValue() { - value = new MyValue { value = input.value }; - return true; + HeapMemorySize = sizeof(int); } - public override bool InPlaceUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public class Serializer : BinaryObjectSerializer { - value.value += input.value; - return true; - } - - public override bool NeedCopyUpdate(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyOutput output, ref RMWInfo rmwInfo) => true; + public override void Deserialize(out IHeapObject obj) => obj = new TestObjectValue { value = reader.ReadInt32() }; - public override bool CopyUpdater(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = new MyValue { value = oldValue.value + input.value }; - return true; + public override void Serialize(IHeapObject obj) => writer.Write(((TestObjectValue)obj).value); } + } - public override bool ConcurrentReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - if (dst == default) - dst = new MyOutput(); - dst.value = value; - return true; - } + public struct TestObjectInput + { + public int value; - public override bool ConcurrentWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - dst.value = src.value; - return true; - } + public TestValueStyle wantValueStyle; - public override void ReadCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(output.value.value, key.key); - } + public override readonly string ToString() => $"value {value}, wantValStyle {wantValueStyle}"; + } - public override void RMWCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.IsTrue(status.Record.CopyUpdated); - } + public struct TestObjectOutput + { + public TestObjectValue value; - public override bool SingleReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo) - { - if (dst == default) - dst = new MyOutput(); - dst.value = value; - return true; - } + public TestValueStyle srcValueStyle; + public TestValueStyle destValueStyle; - public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - dst = src; - return true; - } + public override readonly string ToString() => $"value {value}, srcValStyle {srcValueStyle}, destValStyle {destValueStyle}"; } - public class MyFunctions2 : SessionFunctionsBase + public class TestObjectFunctions : SessionFunctionsBase { - public override bool InitialUpdater(ref MyValue key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) { - value = new MyValue { value = input.value }; - return true; + // (for debugging specific failures) Assert.That(input.value, Is.EqualTo(logRecord.Key.AsRef().key), $"Record logicalAddress: {rmwInfo.Address}"); + return logRecord.TrySetValueObjectAndPrepareOptionals(new TestObjectValue { value = input.value }, in sizeInfo); } - public override bool InPlaceUpdater(ref MyValue key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) { - value.value += input.value; + ((TestObjectValue)logRecord.ValueObject).value += input.value; return true; } - public override bool NeedCopyUpdate(ref MyValue key, ref MyInput input, ref MyValue oldValue, ref MyOutput output, ref RMWInfo rmwInfo) => true; + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) => true; - public override bool CopyUpdater(ref MyValue key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = new MyValue { value = oldValue.value + input.value }; - return true; - } - - public override bool ConcurrentReader(ref MyValue key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - if (dst == default) - dst = new MyOutput(); - dst.value = value; - return true; - } + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + => dstLogRecord.TrySetValueObjectAndPrepareOptionals(new TestObjectValue { value = ((TestObjectValue)srcLogRecord.ValueObject).value + input.value }, in sizeInfo); - public override bool ConcurrentWriter(ref MyValue key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) { - dst.value = src.value; + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + if (!logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + output.value = (TestObjectValue)logRecord.ValueObject; return true; } - public override void ReadCompletionCallback(ref MyValue key, ref MyInput input, ref MyOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key.value, output.value.value); + Assert.That(output.value.value, Is.EqualTo(srcLogRecord.Key.AsRef().key), $"Record logicalAddress: {recordMetadata.Address}"); } - public override void RMWCompletionCallback(ref MyValue key, ref MyInput input, ref MyOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); ClassicAssert.IsTrue(status.Record.CopyUpdated); } - public override bool SingleReader(ref MyValue key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref ReadInfo readInfo) { - if (dst == default) - dst = new MyOutput(); - dst.value = value; + output.value = (TestObjectValue)srcLogRecord.ValueObject; + // (for debugging specific failures) Assert.That(output.value.value, Is.EqualTo(srcLogRecord.Key.AsRef().key), $"Record logicalAddress: {readInfo.Address}"); return true; } - public override bool SingleWriter(ref MyValue key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) { - dst = src; - return true; + // (for debugging specific failures) Assert.That(((TestObjectValue)srcValue).value, Is.EqualTo(logRecord.Key.AsRef().key), $"Record logicalAddress: {upsertInfo.Address}"); + return logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); } + + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TestObjectInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TestObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TestObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; } - public class MyFunctionsDelete : SessionFunctionsBase + public class TestObjectFunctionsDelete : SessionFunctionsBase { - public override bool InitialUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value = new MyValue { value = input.value }; - return true; - } + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + => logRecord.TrySetValueObjectAndPrepareOptionals(new TestObjectValue { value = input.value }, in sizeInfo); - public override bool InPlaceUpdater(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) { - value.value += input.value; + ((TestObjectValue)logRecord.ValueObject).value += input.value; return true; } - public override bool NeedCopyUpdate(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyOutput output, ref RMWInfo rmwInfo) => true; - - public override bool CopyUpdater(ref MyKey key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = new MyValue { value = oldValue.value + input.value }; - return true; - } + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) => true; - public override bool ConcurrentReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst ??= new MyOutput(); - dst.value = value; - return true; - } + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + => dstLogRecord.TrySetValueObjectAndPrepareOptionals(new TestObjectValue { value = ((TestObjectValue)srcLogRecord.ValueObject).value + input.value }, in sizeInfo); - public override bool ConcurrentWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) { - dst = src; - return true; + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); } - public override void ReadCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, int ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, int ctx, Status status, RecordMetadata recordMetadata) { if (ctx == 0) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key.key, output.value.value); + ClassicAssert.AreEqual(srcLogRecord.Key.AsRef().key, output.value.value); } else if (ctx == 1) { @@ -236,7 +197,7 @@ public override void ReadCompletionCallback(ref MyKey key, ref MyInput input, re } } - public override void RMWCompletionCallback(ref MyKey key, ref MyInput input, ref MyOutput output, int ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, int ctx, Status status, RecordMetadata recordMetadata) { if (ctx == 0) { @@ -247,139 +208,284 @@ public override void RMWCompletionCallback(ref MyKey key, ref MyInput input, ref ClassicAssert.IsFalse(status.Found); } - public override bool SingleReader(ref MyKey key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref ReadInfo readInfo) { - dst ??= new MyOutput(); - dst.value = value; + output.value = (TestObjectValue)srcLogRecord.ValueObject; return true; } - public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - dst = src; - return true; - } + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) + => logRecord.TrySetValueObjectAndPrepareOptionals(srcValue, in sizeInfo); + + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TestObjectInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TestObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TestObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; } - public class MixedFunctions : SessionFunctionsBase + public class TestLargeObjectValue : HeapObjectBase { - public override bool InitialUpdater(ref int key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public byte[] value; + + public override HeapObjectBase Clone() => throw new NotImplementedException(); + public override void DoSerialize(BinaryWriter writer) => throw new NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new NotImplementedException(); + + public override void Dispose() { } + + public TestLargeObjectValue() { } + + public TestLargeObjectValue(int size) { - value = new MyValue { value = input.value }; - return true; + value = new byte[size]; + for (int i = 0; i < size; i++) + value[i] = (byte)(size + i); } - public override bool InPlaceUpdater(ref int key, ref MyInput input, ref MyValue value, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public class Serializer : BinaryObjectSerializer { - value.value += input.value; - return true; - } + public override void Deserialize(out IHeapObject obj) + { + var value = new TestLargeObjectValue(); + obj = value; + int size = reader.ReadInt32(); + Assert.That(size, Is.Not.EqualTo(0)); - public override bool NeedCopyUpdate(ref int key, ref MyInput input, ref MyValue oldValue, ref MyOutput output, ref RMWInfo rmwInfo) => true; + value.value = reader.ReadBytes(size); + Assert.That(value.value.Length, Is.EqualTo(size)); + } - public override bool CopyUpdater(ref int key, ref MyInput input, ref MyValue oldValue, ref MyValue newValue, ref MyOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = new MyValue { value = oldValue.value + input.value }; - return true; + public override void Serialize(IHeapObject obj) + { + var value = (TestLargeObjectValue)obj; + writer.Write(value.value.Length); + writer.Write(value.value); + } } + } + + public struct TestLargeObjectInput + { + public int value; + public TestValueStyle wantValueStyle; + public int expectedSpanLength; + + public override readonly string ToString() => $"value {value}, wantValStyle {wantValueStyle}"; + } + + public class TestLargeObjectOutput + { + public TestLargeObjectValue valueObject; + public byte[] valueArray; + } - public override bool ConcurrentReader(ref int key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public class TestLargeObjectFunctions : SessionFunctionsBase + { + public int expectedRecordLength = -1; + + public override void ReadCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestLargeObjectInput input, ref TestLargeObjectOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { - dst.value = value; - return true; + Assert.That(status.Found, Is.True); } - public override bool ConcurrentWriter(ref int key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo updateInfo, ref RecordInfo recordInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TestLargeObjectInput input, ref TestLargeObjectOutput output, ref ReadInfo readInfo) { - dst.value = src.value; + Assert.That(expectedRecordLength < 0 || srcLogRecord.AllocatedSize == expectedRecordLength); + switch (input.wantValueStyle) + { + case TestValueStyle.None: + Assert.Fail("wantValueStyle should not be None"); + break; + case TestValueStyle.Inline: + Assert.That(srcLogRecord.Info.ValueIsInline, Is.True); + Assert.That(srcLogRecord.ValueSpan.Length, Is.EqualTo(input.expectedSpanLength)); + output.valueArray = srcLogRecord.ValueSpan.ToArray(); + break; + case TestValueStyle.Overflow: + Assert.That(srcLogRecord.Info.ValueIsOverflow, Is.True); + Assert.That(srcLogRecord.ValueSpan.Length, Is.EqualTo(input.expectedSpanLength)); + output.valueArray = srcLogRecord.ValueSpan.ToArray(); + break; + case TestValueStyle.Object: + Assert.That(srcLogRecord.Info.ValueIsObject, Is.True); + break; + } + output.valueObject = srcLogRecord.Info.ValueIsObject ? (TestLargeObjectValue)srcLogRecord.ValueObject : default; return true; } - public override bool SingleReader(ref int key, ref MyInput input, ref MyValue value, ref MyOutput dst, ref ReadInfo readInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref TestLargeObjectInput input, IHeapObject srcValue, ref TestLargeObjectOutput output, ref UpsertInfo updateInfo) { - dst.value = value; + Assert.That(expectedRecordLength < 0 || logRecord.AllocatedSize == expectedRecordLength); + if (!logRecord.TrySetValueObject(srcValue)) // We should always be non-inline + return false; + output.valueObject = logRecord.Info.ValueIsObject ? (TestLargeObjectValue)logRecord.ValueObject : default; return true; } - public override bool SingleWriter(ref int key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo updateInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestLargeObjectInput input, IHeapObject srcValue, ref TestLargeObjectOutput output, ref UpsertInfo updateInfo) { - dst = src; + Assert.That(expectedRecordLength < 0 || logRecord.AllocatedSize == expectedRecordLength); + if (!logRecord.TrySetValueObject(srcValue)) // We should always be non-inline + return false; + if (output is not null) + output.valueObject = logRecord.Info.ValueIsObject ? (TestLargeObjectValue)logRecord.ValueObject : default; return true; } + + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TestLargeObjectInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TestLargeObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TestLargeObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref TestLargeObjectInput input) + => new() + { + KeySize = key.KeyBytes.Length, + ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, + ValueIsObject = inputLogRecord.Info.ValueIsObject, + HasETag = inputLogRecord.Info.HasETag, + HasExpiration = inputLogRecord.Info.HasExpiration + }; } - public class MyLargeValue + public class TestMultiListObjectValue : HeapObjectBase { - public byte[] value; + public List[] lists; + int objectIndex; + + public override HeapObjectBase Clone() => throw new NotImplementedException(); + public override void DoSerialize(BinaryWriter writer) => throw new NotImplementedException(); + public override void WriteType(BinaryWriter writer, bool isNull) => throw new NotImplementedException(); - public MyLargeValue() + public override void Dispose() { } + + public TestMultiListObjectValue() { } + + public static long CreateValue(int objectIndex, int listIndex, int itemIndex) => (long)(((ulong)objectIndex << 48) + ((ulong)listIndex << 32) + (ulong)itemIndex); + + public TestMultiListObjectValue(int objectIndex, int numLists, int numItems, Random rng = null) { + this.objectIndex = objectIndex; + lists = new List[numLists]; + for (int ii = 0; ii < numLists; ii++) + { + var numElements = rng is not null ? 1 + rng.Next(numItems) : numItems; + lists[ii] = new List(numElements); + for (int jj = 0; jj < numElements; jj++) + lists[ii].Add(CreateValue(objectIndex, ii, jj)); + } } - public MyLargeValue(int size) + public class Serializer : BinaryObjectSerializer { - value = new byte[size]; - for (int i = 0; i < size; i++) + public override void Deserialize(out IHeapObject obj) { - value[i] = (byte)(size + i); + var value = new TestMultiListObjectValue(); + obj = value; + + value.objectIndex = reader.ReadInt32(); + int numLists = reader.ReadInt32(); + Assert.That(numLists, Is.Not.EqualTo(0)); + value.lists = new List[numLists]; + for (var ii = 0; ii < numLists; ii++) + { + int numItems = reader.ReadInt32(); + Assert.That(numItems, Is.Not.EqualTo(0)); + var list = new List(numItems); + value.lists[ii] = list; + for (int jj = 0; jj < numItems; jj++) + list.Add(reader.ReadInt64()); + } + } + + public override void Serialize(IHeapObject obj) + { + var value = (TestMultiListObjectValue)obj; + writer.Write(value.objectIndex); + writer.Write(value.lists.Length); + var numLists = value.lists.Length; + Assert.That(numLists, Is.Not.EqualTo(0)); + for (var ii = 0; ii < numLists; ii++) + { + var list = value.lists[ii]; + var numItems = list.Count; + Assert.That(numItems, Is.Not.EqualTo(0)); + writer.Write(numItems); + for (int jj = 0; jj < numItems; jj++) + writer.Write(list[jj]); + } } } } - public class MyLargeValueSerializer : BinaryObjectSerializer + public struct TestMultiListObjectInput { - public override void Deserialize(out MyLargeValue obj) - { - obj = new MyLargeValue(); - int size = reader.ReadInt32(); - obj.value = reader.ReadBytes(size); - } + public int objectIndex, listIndex, itemIndex; + public long updateValue; - public override void Serialize(ref MyLargeValue obj) - { - writer.Write(obj.value.Length); - writer.Write(obj.value); - } + public readonly long ExpectedOutputValue => TestMultiListObjectValue.CreateValue(objectIndex, listIndex, itemIndex); + + public override readonly string ToString() => $"objectIndex {objectIndex}, listIndex {listIndex}, itemIndex {itemIndex}, updateValue {updateValue}"; } - public class MyLargeOutput + public class TestMultiListObjectOutput { - public MyLargeValue value; + public TestMultiListObjectValue valueObject; + public long oldValue, newValue; + + public override string ToString() => $"oldValue {oldValue}, newValue {newValue}"; } - public class MyLargeFunctions : SessionFunctionsBase + public class TestMultiListObjectFunctions : SessionFunctionsBase { - public override void ReadCompletionCallback(ref MyKey key, ref MyInput input, ref MyLargeOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord srcLogRecord, ref TestMultiListObjectInput input, ref TestMultiListObjectOutput output, Empty ctx, Status status, RecordMetadata recordMetadata) { - ClassicAssert.IsTrue(status.Found); - for (int i = 0; i < output.value.value.Length; i++) - { - ClassicAssert.AreEqual((byte)(output.value.value.Length + i), output.value.value[i]); - } + Assert.That(status.Found, Is.True); } - public override bool SingleReader(ref MyKey key, ref MyInput input, ref MyLargeValue value, ref MyLargeOutput dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TestMultiListObjectInput input, ref TestMultiListObjectOutput output, ref ReadInfo readInfo) { - dst.value = value; + output.valueObject = (TestMultiListObjectValue)srcLogRecord.ValueObject; + output.oldValue = output.valueObject.lists[input.listIndex][input.itemIndex]; return true; } - public override bool ConcurrentReader(ref MyKey key, ref MyInput input, ref MyLargeValue value, ref MyLargeOutput dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref TestMultiListObjectInput input, IHeapObject srcValue, ref TestMultiListObjectOutput output, ref UpsertInfo updateInfo) { - dst.value = value; + output.valueObject = (TestMultiListObjectValue)logRecord.ValueObject; + output.oldValue = output.valueObject.lists[input.listIndex][input.itemIndex]; + output.valueObject.lists[input.listIndex][input.itemIndex] = input.updateValue; + output.newValue = output.valueObject.lists[input.listIndex][input.itemIndex]; return true; } - public override bool ConcurrentWriter(ref MyKey key, ref MyInput input, ref MyLargeValue src, ref MyLargeValue dst, ref MyLargeOutput output, ref UpsertInfo updateInfo, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestMultiListObjectInput input, IHeapObject srcValue, ref TestMultiListObjectOutput output, ref UpsertInfo updateInfo) { - dst = src; + if (!logRecord.TrySetValueObject(srcValue)) // We should always be non-inline + return false; + if (output is not null) + output.valueObject = (TestMultiListObjectValue)srcValue; return true; } - public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyLargeValue src, ref MyLargeValue dst, ref MyLargeOutput output, ref UpsertInfo updateInfo, WriteReason reason, ref RecordInfo recordInfo) - { - dst = src; - return true; - } + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TestMultiListObjectInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TestMultiListObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TestMultiListObjectInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, in TSourceLogRecord inputLogRecord, ref TestMultiListObjectInput input) + => new() + { + KeySize = key.KeyBytes.Length, + ValueSize = inputLogRecord.Info.ValueIsObject ? ObjectIdMap.ObjectIdSize : inputLogRecord.ValueSpan.Length, + ValueIsObject = inputLogRecord.Info.ValueIsObject, + HasETag = inputLogRecord.Info.HasETag, + HasExpiration = inputLogRecord.Info.HasExpiration + }; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ObjectTests.cs b/libs/storage/Tsavorite/cs/test/ObjectTests.cs deleted file mode 100644 index 2f16a8ac18c..00000000000 --- a/libs/storage/Tsavorite/cs/test/ObjectTests.cs +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - internal class ObjectTests : AllureTestBase - { - private TsavoriteKV store; - private IDevice log, objlog; - - [SetUp] - public void Setup() - { - DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.log"), deleteOnClose: true); - objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.obj.log"), deleteOnClose: true); - - store = new(new() - { - IndexSize = 1L << 13, - LogDevice = log, - ObjectLogDevice = objlog, - MutableFraction = 0.1, - MemorySize = 1L << 15, - PageSize = 1L << 10 - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer(), DefaultRecordDisposer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - } - - [TearDown] - public void TearDown() - { - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - objlog?.Dispose(); - objlog = null; - OnTearDown(); - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void ObjectInMemWriteRead() - { - using var session = store.NewSession(new MyFunctions()); - var bContext = session.BasicContext; - - MyKey key1 = new() { key = 9999999 }; - MyValue value = new() { value = 23 }; - - MyInput input = null; - MyOutput output = new(); - - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - _ = bContext.Read(ref key1, ref input, ref output, Empty.Default); - ClassicAssert.AreEqual(value.value, output.value.value); - } - - [Test] - [Category("TsavoriteKV")] - public void ObjectInMemWriteRead2() - { - using var session = store.NewSession(new MyFunctions()); - var bContext = session.BasicContext; - - MyKey key1 = new() { key = 8999998 }; - MyInput input1 = new() { value = 23 }; - MyOutput output = new(); - - _ = bContext.RMW(ref key1, ref input1, Empty.Default); - - MyKey key2 = new() { key = 8999999 }; - MyInput input2 = new() { value = 24 }; - _ = bContext.RMW(ref key2, ref input2, Empty.Default); - - _ = bContext.Read(ref key1, ref input1, ref output, Empty.Default); - - ClassicAssert.AreEqual(input1.value, output.value.value); - - _ = bContext.Read(ref key2, ref input2, ref output, Empty.Default); - ClassicAssert.AreEqual(input2.value, output.value.value); - - } - - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public void ObjectDiskWriteRead() - { - using var session = store.NewSession(new MyFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < 2000; i++) - { - var key = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key, ref value, Empty.Default); - // store.ShiftReadOnlyAddress(store.LogTailAddress); - } - - MyKey key2 = new() { key = 23 }; - MyInput input = new(); - MyOutput g1 = new(); - var status = bContext.Read(ref key2, ref input, ref g1, Empty.Default); - - if (status.IsPending) - { - _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); - (status, g1) = GetSinglePendingResult(outputs); - } - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(23, g1.value.value); - - key2 = new MyKey { key = 99999 }; - status = bContext.Read(ref key2, ref input, ref g1, Empty.Default); - - if (status.IsPending) - (status, _) = bContext.GetSinglePendingResult(); - ClassicAssert.IsFalse(status.Found); - - // Update last 100 using RMW in memory - for (int i = 1900; i < 2000; i++) - { - var key = new MyKey { key = i }; - input = new MyInput { value = 1 }; - status = bContext.RMW(ref key, ref input, Empty.Default); - ClassicAssert.IsFalse(status.IsPending, "Expected RMW to complete in-memory"); - } - - // Update first 100 using RMW from storage - for (int i = 0; i < 100; i++) - { - var key1 = new MyKey { key = i }; - input = new MyInput { value = 1 }; - status = bContext.RMW(ref key1, ref input, Empty.Default); - if (status.IsPending) - _ = bContext.CompletePending(true); - } - - for (int i = 0; i < 2000; i++) - { - var output = new MyOutput(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - - status = bContext.Read(ref key1, ref input, ref output, Empty.Default); - if (status.IsPending) - (status, output) = bContext.GetSinglePendingResult(); - else - { - if (i < 100 || i >= 1900) - ClassicAssert.AreEqual(value.value + 1, output.value.value); - else - ClassicAssert.AreEqual(value.value, output.value.value); - } - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/OverflowBucketLockTableTests.cs b/libs/storage/Tsavorite/cs/test/OverflowBucketLockTableTests.cs index 4b629e7f652..f5a52115091 100644 --- a/libs/storage/Tsavorite/cs/test/OverflowBucketLockTableTests.cs +++ b/libs/storage/Tsavorite/cs/test/OverflowBucketLockTableTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; @@ -6,7 +6,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,27 +14,40 @@ namespace Tsavorite.test.LockTable { - using LongStoreFunctions = StoreFunctions>; + using LongStoreFunctions = StoreFunctions; - internal class SingleBucketComparer : IKeyComparer + internal class SingleBucketComparer : IKeyComparer { - public bool Equals(ref long k1, ref long k2) => k1 == k2; - - public long GetHashCode64(ref long k) => 42L; + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef() == k2.KeyBytes.AsRef(); + + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + + => 42L; } // Used to signal Setup to use the SingleBucketComparer public enum UseSingleBucketComparer { UseSingleBucket } - - [AllureNUnit] [TestFixture] - internal class OverflowBucketLockTableTests : AllureTestBase + internal class OverflowBucketLockTableTests : TestBase { - IKeyComparer comparer = new LongKeyComparer(); - long SingleBucketKey = 1; // We use a single bucket here for most tests so this lets us use 'ref' easily + IKeyComparer comparer = new LongKeyComparer(); + long singleBucketKey = 1; // We use a single bucket here for most tests so this lets us use 'ref' easily // For OverflowBucketLockTable, we need an instance of TsavoriteKV - private TsavoriteKV> store; + private TsavoriteKV> store; private IDevice log; [SetUp] @@ -60,8 +72,8 @@ public void Setup() IndexSize = 1L << 26, LogDevice = log, PageSize = 1L << 12, - MemorySize = 1L << 22 - }, StoreFunctions.Create(LongKeyComparer.Instance) + LogMemorySize = 1L << 22 + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -79,7 +91,7 @@ public void TearDown() void TryLock(long key, LockType lockType, int expectedCurrentReadLocks, bool expectedLockResult) { - HashEntryInfo hei = new(comparer.GetHashCode64(ref key)); + HashEntryInfo hei = new(comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)))); PopulateHei(ref hei); // Check for existing lock @@ -94,7 +106,7 @@ void TryLock(long key, LockType lockType, int expectedCurrentReadLocks, bool exp void Unlock(long key, LockType lockType) { - HashEntryInfo hei = new(comparer.GetHashCode64(ref key)); + HashEntryInfo hei = new(comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)))); PopulateHei(ref hei); if (lockType == LockType.Shared) store.LockTable.UnlockShared(ref hei); @@ -104,9 +116,9 @@ void Unlock(long key, LockType lockType) internal void PopulateHei(ref HashEntryInfo hei) => PopulateHei(store, ref hei); - internal static void PopulateHei(TsavoriteKV store, ref HashEntryInfo hei) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void PopulateHei(TsavoriteKV store, ref HashEntryInfo hei) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => store.FindOrCreateTag(ref hei, store.Log.BeginAddress); internal void AssertLockCounts(ref HashEntryInfo hei, bool expectedX, long expectedS) @@ -116,42 +128,33 @@ internal void AssertLockCounts(ref HashEntryInfo hei, bool expectedX, long expec ClassicAssert.AreEqual(expectedS, lockState.NumLockedShared); } - internal static void AssertLockCounts(TsavoriteKV store, TKey key, bool expectedX, int expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => AssertLockCounts(store, ref key, expectedX, expectedS); - - internal static void AssertLockCounts(TsavoriteKV store, ref TKey key, bool expectedX, int expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void AssertLockCounts(TsavoriteKV store, ReadOnlySpan key, bool expectedX, int expectedS) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(ref key)); + HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(TestSpanByteKey.FromPinnedSpan(key))); PopulateHei(store, ref hei); var lockState = store.LockTable.GetLockState(ref hei); ClassicAssert.AreEqual(expectedX, lockState.IsLockedExclusive, "XLock mismatch"); ClassicAssert.AreEqual(expectedS, lockState.NumLockedShared, "SLock mismatch"); } - internal static void AssertLockCounts(TsavoriteKV store, ref TKey key, bool expectedX, bool expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void AssertLockCounts(TsavoriteKV store, ReadOnlySpan key, bool expectedX, bool expectedS) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - FixedLengthLockableKeyStruct keyStruct = new() - { - Key = key, - KeyHash = store.storeFunctions.GetKeyHashCode64(ref key), - LockType = LockType.None, // Not used for this call - }; - keyStruct.KeyHash = store.GetKeyHash(ref key); - AssertLockCounts(store, ref keyStruct, expectedX, expectedS); + HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(TestSpanByteKey.FromPinnedSpan(key))); + PopulateHei(store, ref hei); + var lockState = store.LockTable.GetLockState(ref hei); + ClassicAssert.AreEqual(expectedX, lockState.IsLockedExclusive, "XLock mismatch"); + ClassicAssert.AreEqual(expectedS, lockState.NumLockedShared > 0, "SLock mismatch"); } - - internal static void AssertLockCounts(TsavoriteKV store, ref FixedLengthLockableKeyStruct key, bool expectedX, bool expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void AssertLockCounts(TsavoriteKV store, ref FixedLengthTransactionalKeyStruct keyStruct, bool expectedX, bool expectedS) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - HashEntryInfo hei = new(key.KeyHash); + HashEntryInfo hei = new(keyStruct.KeyHash); PopulateHei(store, ref hei); var lockState = store.LockTable.GetLockState(ref hei); ClassicAssert.AreEqual(expectedX, lockState.IsLockedExclusive, "XLock mismatch"); @@ -161,9 +164,9 @@ internal static void AssertLockCounts internal unsafe void AssertTotalLockCounts(long expectedX, long expectedS) => AssertTotalLockCounts(store, expectedX, expectedS); - internal static unsafe void AssertTotalLockCounts(TsavoriteKV store, long expectedX, long expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static unsafe void AssertTotalLockCounts(TsavoriteKV store, long expectedX, long expectedS) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { HashBucket* buckets = store.state[store.resizeInfo.version].tableAligned; var count = store.LockTable.NumBuckets; @@ -178,11 +181,11 @@ internal static unsafe void AssertTotalLockCounts(T ClassicAssert.AreEqual(expectedS, scount); } - internal void AssertBucketLockCount(ref FixedLengthLockableKeyStruct key, long expectedX, long expectedS) => AssertBucketLockCount(store, ref key, expectedX, expectedS); + internal void AssertBucketLockCount(ref FixedLengthTransactionalKeyStruct keyStruct, long expectedX, long expectedS) => AssertBucketLockCount(store, ref keyStruct, expectedX, expectedS); - internal static unsafe void AssertBucketLockCount(TsavoriteKV store, ref FixedLengthLockableKeyStruct key, long expectedX, long expectedS) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static unsafe void AssertBucketLockCount(TsavoriteKV store, ref FixedLengthTransactionalKeyStruct key, long expectedX, long expectedS) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { var bucketIndex = store.LockTable.GetBucketIndex(key.KeyHash); var bucket = store.state[store.resizeInfo.version].tableAligned + bucketIndex; @@ -194,7 +197,7 @@ internal static unsafe void AssertBucketLockCount(T [Category(LockTestCategory), Category(LockTableTestCategory), Category(SmokeTestCategory)] public void SingleKeyTest([Values] UseSingleBucketComparer /* justToSignalSetup */ _) { - HashEntryInfo hei = new(comparer.GetHashCode64(ref SingleBucketKey)); + HashEntryInfo hei = new(comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref singleBucketKey)))); PopulateHei(ref hei); AssertLockCounts(ref hei, false, 0); @@ -230,7 +233,7 @@ public void SingleKeyTest([Values] UseSingleBucketComparer /* justToSignalSetup [Category(LockTestCategory), Category(LockTableTestCategory), Category(SmokeTestCategory)] public void ThreeKeyTest([Values] UseSingleBucketComparer /* justToSignalSetup */ _) { - HashEntryInfo hei = new(comparer.GetHashCode64(ref SingleBucketKey)); + HashEntryInfo hei = new(comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref singleBucketKey)))); PopulateHei(ref hei); AssertLockCounts(ref hei, false, 0); @@ -306,24 +309,28 @@ public void ThreadedLockStressTestMultiThreadsRandomContention([Values(3, 8)] in AssertTotalLockCounts(0, 0); } - FixedLengthLockableKeyStruct[] CreateKeys(Random rng, int numKeys, int numRecords) + // Creates numRecords random keys in the range [0,numKeys) and returns both the array of key structs and the pinned array of longs used as storage for the keys. + (FixedLengthTransactionalKeyStruct[], long[]) CreateKeys(Random rng, int numKeys, int numRecords) { - FixedLengthLockableKeyStruct createKey() + // This needs to return the pinned source of the key longs so its lifetime is at least that of the output KeyStruct[]. + var keyNums = GC.AllocateArray(numRecords, pinned: true); + FixedLengthTransactionalKeyStruct createKey(int recordNum) { - long key = rng.Next(numKeys); - var keyHash = store.GetKeyHash(ref key); + keyNums[recordNum] = rng.Next(numKeys); + var key = SpanByte.FromPinnedVariable(ref keyNums[recordNum]); + var keyHash = store.GetKeyHash(TestSpanByteKey.FromPinnedSpan(key)); return new() { - Key = key, + Key = TestSpanByteKey.FromPinnedSpan(PinnedSpanByte.FromPinnedSpan(key)), // LockType.None means split randomly between Shared and Exclusive LockType = rng.Next(0, 100) < 25 ? LockType.Exclusive : LockType.Shared, KeyHash = keyHash, }; } - return [.. Enumerable.Range(0, numRecords).Select(ii => createKey())]; + return ([.. Enumerable.Range(0, numRecords).Select(createKey)], keyNums); } - void AssertSorted(FixedLengthLockableKeyStruct[] keys, int count) + void AssertSorted(FixedLengthTransactionalKeyStruct[] keys, int count) { long prevCode = default; long lastXcode = default; @@ -363,8 +370,10 @@ void AssertSorted(FixedLengthLockableKeyStruct[] keys, int count) [Category(LockTestCategory), Category(LockTableTestCategory), Category(SmokeTestCategory)] public void FullArraySortTest() { - var keys = CreateKeys(new Random(101), 100, 1000); - store.LockTable.SortKeyHashes>(keys); + const int numRecords = 1000; + var (keys, keyNums) = CreateKeys(new Random(101), 100, numRecords); + Assert.That(keyNums.Length, Is.EqualTo(numRecords)); + store.LockTable.SortKeyHashes(keys); AssertSorted(keys, keys.Length); } @@ -373,7 +382,8 @@ public void FullArraySortTest() public void PartialArraySortTest() { var numRecords = 1000; - var keys = CreateKeys(new Random(101), 100, numRecords); + var (keys, keyNums) = CreateKeys(new Random(101), 100, numRecords); + Assert.That(keyNums.Length, Is.EqualTo(numRecords)); const int count = 800; // Make the later elements invalid. @@ -397,16 +407,17 @@ void runThread(int tid) { Random rng = new(101 * tid); - // maxNumKeys < 0 means use random number of keys + // maxNumKeys < 0 means use random number of keys. SpanByte requires persistent storage so we need the threadKeyNums vector in parallel with threadStructs. int numKeys = maxNumKeys < 0 ? rng.Next(1, -maxNumKeys) : maxNumKeys; - var threadStructs = new FixedLengthLockableKeyStruct[numKeys]; + var threadKeyNums = GC.AllocateArray(numKeys, pinned: true); + var threadStructs = new FixedLengthTransactionalKeyStruct[numKeys]; long getNextKey() { while (true) { var key = rng.Next(lowKey, highKey + 1); // +1 because the end # is not included - if (!Array.Exists(threadStructs, it => it.Key == key)) + if (!Array.Exists(threadStructs, it => it.Key.KeyBytes.Length > 0 && it.Key.KeyBytes.AsRef() == key)) return key; } } @@ -416,19 +427,20 @@ long getNextKey() // Create key structs for (var ii = 0; ii < numKeys; ++ii) { - var key = getNextKey(); - threadStructs[ii] = new() // local var for debugging + threadKeyNums[ii] = getNextKey(); + var key = SpanByte.FromPinnedVariable(ref threadKeyNums[ii]); // storage for the SpanByte in the pinned array + threadStructs[ii] = new() { - Key = key, + Key = TestSpanByteKey.FromPinnedSpan(PinnedSpanByte.FromPinnedSpan(key)), // LockType.None means split randomly between Shared and Exclusive LockType = lockType == LockType.None ? (rng.Next(0, 100) > 50 ? LockType.Shared : LockType.Exclusive) : lockType, - KeyHash = comparer.GetHashCode64(ref key), + KeyHash = comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(key)), }; - threadStructs[ii].KeyHash = store.GetKeyHash(ref key); + threadStructs[ii].KeyHash = store.GetKeyHash(TestSpanByteKey.FromPinnedSpan(key)); } // Sort and lock - store.LockTable.SortKeyHashes>(threadStructs); + store.LockTable.SortKeyHashes(threadStructs); for (var ii = 0; ii < numKeys; ++ii) { HashEntryInfo hei = new(threadStructs[ii].KeyHash); @@ -454,7 +466,6 @@ long getNextKey() } Array.Clear(threadStructs); } - } for (int t = 1; t <= numThreads; t++) diff --git a/libs/storage/Tsavorite/cs/test/PostOperationsTests.cs b/libs/storage/Tsavorite/cs/test/PostOperationsTests.cs index 86806bb6673..afa35ab10e2 100644 --- a/libs/storage/Tsavorite/cs/test/PostOperationsTests.cs +++ b/libs/storage/Tsavorite/cs/test/PostOperationsTests.cs @@ -1,8 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,14 +11,12 @@ namespace Tsavorite.test { - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using IntAllocator = SpanByteAllocator>; + using IntStoreFunctions = StoreFunctions; [TestFixture] - internal class PostOperationsTests : AllureTestBase + internal class PostOperationsTests : TestBase { - class PostFunctions : SimpleSimpleFunctions + class PostFunctions : SimpleIntSimpleFunctions { internal long pswAddress; internal long piuAddress; @@ -28,26 +26,39 @@ class PostFunctions : SimpleSimpleFunctions internal void Clear() { - pswAddress = Constants.kInvalidAddress; - piuAddress = Constants.kInvalidAddress; - pcuAddress = Constants.kInvalidAddress; - psdAddress = Constants.kInvalidAddress; + pswAddress = LogAddress.kInvalidAddress; + piuAddress = LogAddress.kInvalidAddress; + pcuAddress = LogAddress.kInvalidAddress; + psdAddress = LogAddress.kInvalidAddress; } internal PostFunctions() : base() { } - public override void PostSingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason) { pswAddress = upsertInfo.Address; } + public override void PostInitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ReadOnlySpan src, ref int output, ref UpsertInfo upsertInfo) + => pswAddress = upsertInfo.Address; + + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) + { + dstLogRecord.ValueSpan.AsRef() = input; + return true; + } - public override bool InitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { value = input; return true; } /// - public override void PostInitialUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo) { piuAddress = rmwInfo.Address; } + public override void PostInitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int value, ref int output, ref RMWInfo rmwInfo) + => piuAddress = rmwInfo.Address; - public override bool InPlaceUpdater(ref int key, ref int input, ref int value, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => false; // For this test, we want this to fail and lead to InitialUpdater + public override bool InPlaceUpdater(ref LogRecord logRecord, ref int input, ref int output, ref RMWInfo rmwInfo) + => false; // For this test, we want this to fail and lead to InitialUpdater /// - public override bool CopyUpdater(ref int key, ref int input, ref int oldValue, ref int newValue, ref int output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { newValue = oldValue; return true; } + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) + { + dstLogRecord.ValueSpan.AsRef() = srcLogRecord.ValueSpan.AsRef(); + return true; + } + /// - public override bool PostCopyUpdater(ref int key, ref int input, ref int oldValue, ref int newValue, ref int output, ref RMWInfo rmwInfo) + public override bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) { pcuAddress = rmwInfo.Address; if (returnFalseFromPCU) @@ -55,13 +66,16 @@ public override bool PostCopyUpdater(ref int key, ref int input, ref int oldValu return !returnFalseFromPCU; } - public override void PostSingleDeleter(ref int key, ref DeleteInfo deleteInfo) { psdAddress = deleteInfo.Address; } - public override bool ConcurrentDeleter(ref int key, ref int value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) => false; + public override void PostInitialDeleter(ref LogRecord dstLogRecord, ref DeleteInfo deleteInfo) + => psdAddress = deleteInfo.Address; + + public override bool InPlaceDeleter(ref LogRecord dstLogRecord, ref DeleteInfo deleteInfo) + => false; } - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; const int NumRecords = 100; @@ -79,12 +93,12 @@ public void Setup() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10 - }, StoreFunctions.Create(IntKeyComparer.Instance) + }, StoreFunctions.Create(IntKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new PostFunctions()); + session = store.NewSession(new PostFunctions()); bContext = session.BasicContext; Populate(); } @@ -106,7 +120,10 @@ void Populate() for (var key = 0; key < NumRecords; ++key) { expectedAddress = store.Log.TailAddress; - _ = bContext.Upsert(key, key * 100); + if ((expectedAddress % store.hlogBase.PageSize) == 0) + expectedAddress += PageHeader.Size; + var value = key * 100; + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); ClassicAssert.AreEqual(expectedAddress, session.functions.pswAddress); } @@ -125,13 +142,14 @@ internal void CompletePendingAndVerifyInsertedAddress() [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public void PostSingleWriterTest() + public void PostInitialWriterTest() { // Populate has already executed the not-found test (InternalInsert) as part of its normal insert. // Execute the ReadOnly (InternalInsert) test store.Log.FlushAndEvict(wait: true); - _ = bContext.Upsert(TargetKey, TargetKey * 1000); + int key = TargetKey, value = TargetKey * 100; + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); _ = bContext.CompletePending(wait: true); ClassicAssert.AreEqual(expectedAddress, session.functions.pswAddress); } @@ -142,21 +160,25 @@ public void PostSingleWriterTest() public void PostInitialUpdaterTest() { // Execute the not-found test (InternalRMW). - _ = bContext.RMW(NumRecords + 1, (NumRecords + 1) * 1000); + int key = NumRecords + 1, value = (NumRecords + 1) * 1000; + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); ClassicAssert.AreEqual(expectedAddress, session.functions.piuAddress); session.functions.Clear(); // Now cause an attempt at InPlaceUpdater, which we've set to fail, so CopyUpdater is done (InternalInsert). expectedAddress = store.Log.TailAddress; - _ = bContext.RMW(TargetKey, TargetKey * 1000); + key = TargetKey; + value = TargetKey * 1000; + + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); ClassicAssert.AreEqual(expectedAddress, session.functions.pcuAddress); // Execute the not-in-memory test (InternalContinuePendingRMW). First delete the record so it has a tombstone; this will go to InitialUpdater. - _ = bContext.Delete(TargetKey); + _ = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); store.Log.FlushAndEvict(wait: true); expectedAddress = store.Log.TailAddress; - _ = bContext.RMW(TargetKey, TargetKey * 1000); + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); CompletePendingAndVerifyInsertedAddress(); ClassicAssert.AreEqual(expectedAddress, session.functions.piuAddress); } @@ -167,14 +189,16 @@ public void PostInitialUpdaterTest() public void PostCopyUpdaterTest() { // First try to modify in-memory, readonly (InternalRMW). + var key = TargetKey; + var value = TargetKey * 1000; store.Log.ShiftReadOnlyAddress(store.Log.ReadOnlyAddress, wait: true); - _ = bContext.RMW(TargetKey, TargetKey * 1000); + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); ClassicAssert.AreEqual(expectedAddress, session.functions.pcuAddress); // Execute the not-in-memory test (InternalContinuePendingRMW). store.Log.FlushAndEvict(wait: true); expectedAddress = store.Log.TailAddress; - _ = bContext.RMW(TargetKey, TargetKey * 1000); + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); CompletePendingAndVerifyInsertedAddress(); ClassicAssert.AreEqual(expectedAddress, session.functions.pcuAddress); } @@ -185,7 +209,9 @@ public void PostCopyUpdaterTest() public void PostCopyUpdaterFalseTest([Values(FlushMode.ReadOnly, FlushMode.OnDisk)] FlushMode flushMode) { // Verify the key exists - var (status, output) = bContext.Read(TargetKey); + var key = TargetKey; + var value = TargetKey * 1000; + var (status, _ /*output*/) = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.IsTrue(status.Found, "Expected the record to exist"); session.functions.returnFalseFromPCU = true; @@ -196,26 +222,28 @@ public void PostCopyUpdaterFalseTest([Values(FlushMode.ReadOnly, FlushMode.OnDis store.Log.FlushAndEvict(wait: true); // Call RMW - _ = bContext.RMW(TargetKey, TargetKey * 1000); + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); // Verify the key no longer exists. - (status, output) = bContext.Read(TargetKey); + (status, _ /*output*/) = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.IsFalse(status.Found, "Expected the record to no longer exist"); } [Test] [Category("TsavoriteKV")] [Category("Smoke")] - public void PostSingleDeleterTest() + public void PostInitialDeleterTest() { - // Execute the not-in-memory test (InternalDelete); ConcurrentDeleter returns false to force a new record to be added. - _ = bContext.Delete(TargetKey); + // Execute the not-in-memory test (InternalDelete); InPlaceDeleter returns false to force a new record to be added. + var key = TargetKey; + _ = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(expectedAddress, session.functions.psdAddress); // Execute the not-in-memory test (InternalDelete). store.Log.FlushAndEvict(wait: true); expectedAddress = store.Log.TailAddress; - _ = bContext.Delete(TargetKey + 1); + key++; + _ = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(expectedAddress, session.functions.psdAddress); } } diff --git a/libs/storage/Tsavorite/cs/test/RecordTriggersExtTests.cs b/libs/storage/Tsavorite/cs/test/RecordTriggersExtTests.cs new file mode 100644 index 00000000000..9dc2248b717 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/RecordTriggersExtTests.cs @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Concurrent; +using System.IO; +using System.Threading; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +#pragma warning disable IDE0060 // Unused parameter + +namespace Tsavorite.test +{ + using ExtAllocator = ObjectAllocator>; + using ExtStoreFunctions = StoreFunctions; + + /// + /// Tests for the trigger extensions added for BfTree (RangeIndex) lifecycle integration: + /// + /// receives the correct logical address. + /// fires AFTER device truncation completes. + /// + /// fires from TryCopyToTail with valid source/destination addresses. + /// + /// All tests use because that is the allocator that fires + /// (and is what Garnet's unified store uses for + /// the RangeIndex stub records). + /// + [TestFixture] + public class RecordTriggersExtTests : TestBase + { + /// Per-event log entries recorded by the test trigger. + internal sealed class TriggerEvents + { + public readonly ConcurrentBag FlushAddresses = new(); + public readonly ConcurrentBag TruncateAddresses = new(); + public readonly ConcurrentBag<(long SrcAddr, long DstAddr)> PostCopyToTailEvents = new(); + + public bool CallOnFlushFlag; + public bool CallOnTruncateFlag; + public bool CallPostCopyToTailFlag; + public bool CallOnDiskReadFlag; + + public int FlushCount => FlushAddresses.Count; + public int TruncateCount => TruncateAddresses.Count; + public int PostCopyCount => PostCopyToTailEvents.Count; + } + + internal struct ExtRecordTriggers : IRecordTriggers + { + internal readonly TriggerEvents events; + public ExtRecordTriggers(TriggerEvents events) { this.events = events; } + + public readonly bool CallOnFlush => events?.CallOnFlushFlag ?? false; + public readonly bool CallOnEvict => false; + public readonly bool CallOnDiskRead => events?.CallOnDiskReadFlag ?? false; + public readonly bool CallPostCopyToTail => events?.CallPostCopyToTailFlag ?? false; + public readonly bool CallOnTruncate => events?.CallOnTruncateFlag ?? false; + + public readonly void OnFlush(ref LogRecord logRecord, long logicalAddress) + { + events?.FlushAddresses.Add(logicalAddress); + } + + public readonly void OnTruncate(long newBeginAddress) + { + events?.TruncateAddresses.Add(newBeginAddress); + } + + public readonly void PostCopyToTail(in TSourceLogRecord srcLogRecord, long srcLogicalAddress, + ref LogRecord dstLogRecord, long dstLogicalAddress) + where TSourceLogRecord : ISourceLogRecord +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + events?.PostCopyToTailEvents.Add((srcLogicalAddress, dstLogicalAddress)); + } + } + + private TsavoriteKV store; + private IDevice log, objlog; + private TriggerEvents events; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "RecordTriggersExtTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "RecordTriggersExtTests.obj.log"), deleteOnClose: true); + events = new TriggerEvents(); + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << 15, + PageSize = 1L << 10, + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), + new ExtRecordTriggers(events)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); store = null; + log?.Dispose(); log = null; + objlog?.Dispose(); objlog = null; + OnTearDown(); + } + + private void InsertN(int n) + { + using var session = store.NewSession(new TestObjectFunctionsDelete()); + var bContext = session.BasicContext; + for (int i = 0; i < n; i++) + _ = bContext.Upsert(new TestObjectKey { key = i }, new TestObjectValue { value = i }, 0); + } + + // -- OnFlush(addr) tests -- + + /// + /// Verifies that fires on + /// FlushAndEvict with logical addresses that fall in [BeginAddress, TailAddress). + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void OnFlushReceivesCorrectLogicalAddress() + { + events.CallOnFlushFlag = true; + InsertN(200); + + store.Log.FlushAndEvict(wait: true); + + ClassicAssert.Greater(events.FlushCount, 0, + $"OnFlush should fire at least once after FlushAndEvict (got {events.FlushCount})"); + + var tail = store.Log.TailAddress; + var ba = store.Log.BeginAddress; + foreach (var addr in events.FlushAddresses) + { + ClassicAssert.GreaterOrEqual(addr, ba, $"flush addr {addr} below BeginAddress {ba}"); + ClassicAssert.Less(addr, tail, $"flush addr {addr} >= TailAddress {tail}"); + } + } + + /// + /// Verifies that false skips the OnFlush callback. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void CallOnFlushFalseSkipsCallback() + { + events.CallOnFlushFlag = false; + InsertN(200); + store.Log.FlushAndEvict(wait: true); + + ClassicAssert.AreEqual(0, events.FlushCount, "OnFlush should not fire when CallOnFlush=false"); + } + + // -- OnTruncate tests -- + + /// + /// Verifies that fires from + /// ShiftBeginAddress(_, truncateLog: true) AFTER the device truncation completes, + /// receiving the new BeginAddress. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void OnTruncateFiresWithNewBeginAddress() + { + events.CallOnTruncateFlag = true; + InsertN(200); + store.Log.FlushAndEvict(wait: true); + + var newBA = store.Log.HeadAddress; + store.Log.ShiftBeginAddress(newBA, truncateLog: true); + + // TruncateUntilAddress runs on a Task.Run; wait up to 5s for fire. + for (int wait = 0; wait < 100 && events.TruncateCount == 0; wait++) + Thread.Sleep(50); + + ClassicAssert.GreaterOrEqual(events.TruncateCount, 1, "OnTruncate should fire at least once"); + ClassicAssert.Contains(newBA, events.TruncateAddresses, + $"OnTruncate should receive the new BeginAddress {newBA}"); + } + + /// + /// Verifies that false skips the callback. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void CallOnTruncateFalseSkipsCallback() + { + events.CallOnTruncateFlag = false; + InsertN(200); + store.Log.FlushAndEvict(wait: true); + store.Log.ShiftBeginAddress(store.Log.HeadAddress, truncateLog: true); + Thread.Sleep(500); + + ClassicAssert.AreEqual(0, events.TruncateCount, "OnTruncate should not fire when CallOnTruncate=false"); + } + + // -- PostCopyToTail tests -- + + /// + /// Verifies that fires from compaction with + /// valid source and destination logical addresses (src in [oldBA, compactUntil], + /// dst at/above pre-compact tail). + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers"), Category("Compaction")] + public void PostCopyToTailFiresFromCompaction([Values] CompactionType compactionType) + { + events.CallPostCopyToTailFlag = true; + events.CallOnDiskReadFlag = true; + + using var session = store.NewSession(new TestObjectFunctionsDelete()); + var bContext = session.BasicContext; + + const int N = 800; + long compactUntil = 0; + for (int i = 0; i < N; i++) + { + if (i == N / 2) + compactUntil = store.Log.TailAddress; + _ = bContext.Upsert(new TestObjectKey { key = i }, new TestObjectValue { value = i }, 0); + } + + // Snapshot pre-compact state. + var oldBA = store.Log.BeginAddress; + var preCompactTail = store.Log.TailAddress; + store.Log.FlushAndEvict(wait: true); + + session.Compact(compactUntil, compactionType); + + ClassicAssert.Greater(events.PostCopyCount, 0, + $"PostCopyToTail should fire at least once for compacted records (got {events.PostCopyCount})"); + + // Every PostCopyToTail event should have: + // - srcAddr in [oldBA, compactUntil]: source was below compactUntil at compaction time. + // - dstAddr >= preCompactTail: record copied to the tail. + foreach (var (srcAddr, dstAddr) in events.PostCopyToTailEvents) + { + ClassicAssert.GreaterOrEqual(srcAddr, oldBA, + $"src addr {srcAddr} below pre-compact BeginAddress {oldBA}"); + ClassicAssert.LessOrEqual(srcAddr, compactUntil, + $"src addr {srcAddr} above compactUntil {compactUntil}"); + ClassicAssert.GreaterOrEqual(dstAddr, preCompactTail, + $"dst addr {dstAddr} should be at/after pre-compact tail {preCompactTail}"); + } + } + + /// + /// Verifies that false skips the callback. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers"), Category("Compaction")] + public void CallPostCopyToTailFalseSkipsCallback() + { + events.CallPostCopyToTailFlag = false; + + using var session = store.NewSession(new TestObjectFunctionsDelete()); + var bContext = session.BasicContext; + for (int i = 0; i < 800; i++) + _ = bContext.Upsert(new TestObjectKey { key = i }, new TestObjectValue { value = i }, 0); + var compactUntil = store.Log.TailAddress; + store.Log.FlushAndEvict(wait: true); + session.Compact(compactUntil, CompactionType.Scan); + + ClassicAssert.AreEqual(0, events.PostCopyCount, + "PostCopyToTail should not fire when CallPostCopyToTail=false"); + } + + // -- Default trigger flags -- + + /// + /// Default no-op trigger struct must have all the new flags returning false. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void DefaultRecordTriggersHasAllNewFlagsFalse() + { + IRecordTriggers def = DefaultRecordTriggers.Instance; + ClassicAssert.IsFalse(def.CallOnFlush); + ClassicAssert.IsFalse(def.CallOnEvict); + ClassicAssert.IsFalse(def.CallOnDiskRead); + ClassicAssert.IsFalse(def.CallPostCopyToTail); + ClassicAssert.IsFalse(def.CallOnTruncate); + } + + /// + /// SpanByteRecordTriggers (legacy no-op) must also have all flags false. + /// + [Test, Category("TsavoriteKV"), Category("RecordTriggers")] + public void SpanByteRecordTriggersHasAllNewFlagsFalse() + { + IRecordTriggers def = SpanByteRecordTriggers.Instance; + ClassicAssert.IsFalse(def.CallOnFlush); + ClassicAssert.IsFalse(def.CallOnEvict); + ClassicAssert.IsFalse(def.CallOnDiskRead); + ClassicAssert.IsFalse(def.CallPostCopyToTail); + ClassicAssert.IsFalse(def.CallOnTruncate); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/RecoveryChecks.cs b/libs/storage/Tsavorite/cs/test/RecoveryChecks.cs deleted file mode 100644 index 29712ca6464..00000000000 --- a/libs/storage/Tsavorite/cs/test/RecoveryChecks.cs +++ /dev/null @@ -1,1095 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.IO; -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using Tsavorite.devices; -using Tsavorite.test.recovery.sumstore; - -namespace Tsavorite.test.recovery -{ - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - - public enum DeviceMode - { - Local, - Cloud - } - - public class RecoveryCheckBase : AllureTestBase - { - protected IDevice log; - protected const int NumOps = 5000; - protected AdId[] inputArray; - - protected void BaseSetup() - { - inputArray = new AdId[NumOps]; - for (int i = 0; i < NumOps; i++) - { - inputArray[i].adId = i; - } - - log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "hlog.log"), deleteOnClose: false); - TestUtils.RecreateDirectory(TestUtils.MethodTestDir); - } - - protected void BaseTearDown() - { - log?.Dispose(); - log = null; - TestUtils.OnTearDown(); - } - - public class MyFunctions : SimpleSimpleFunctions - { - public override void ReadCompletionCallback(ref long key, ref long input, ref long output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - - public class MyFunctions2 : SimpleSimpleFunctions - { - public override void ReadCompletionCallback(ref long key, ref long input, ref long output, Empty ctx, Status status, RecordMetadata recordMetadata) - { - Verify(status, key, output); - } - - internal static void Verify(Status status, long key, long output) - { - ClassicAssert.IsTrue(status.Found); - if (key < 950) - ClassicAssert.AreEqual(key, output); - else - ClassicAssert.AreEqual(key + 1, output); - } - } - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheck1Tests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV")] - [Category("CheckpointRestore")] - [Category("Smoke")] - - public async ValueTask RecoveryCheck1( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession(new MyFunctions()); - var bc1 = s1.BasicContext; - - for (long key = 0; key < 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - var task = store1.TakeFullCheckpointAsync(checkpointType); - - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - _ = await store2.RecoverAsync(default, token).ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - _ = store2.Recover(default, token); - } - - ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); - ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); - ClassicAssert.AreEqual(store1.Log.TailAddress, store2.Log.TailAddress); - - using var s2 = store2.NewSession(new MyFunctions()); - var bc2 = s2.BasicContext; - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheck2Tests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public async ValueTask RecoveryCheck2( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession>(new SimpleSimpleFunctions()); - var bc1 = s1.BasicContext; - - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - for (int i = 0; i < 5; i++) - { - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - var task = store1.TakeHybridLogCheckpointAsync(checkpointType); - - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - _ = await store2.RecoverAsync(default, token).ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - _ = store2.Recover(default, token); - } - - ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); - ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); - ClassicAssert.AreEqual(store1.Log.TailAddress, store2.Log.TailAddress); - - using var s2 = store2.NewSession>(new SimpleSimpleFunctions()); - var bc2 = s2.BasicContext; - for (long key = 0; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public void RecoveryCheck2Repeated( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType - ) - { - Guid token = default; - - for (int i = 0; i < 6; i++) - { - using var store = new TsavoriteKV(new() - { - IndexSize = 1L << 13, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - if (i > 0) - _ = store.Recover(default, token); - - using var s1 = store.NewSession>(new SimpleSimpleFunctions()); - var bc1 = s1.BasicContext; - - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - var task = store.TakeHybridLogCheckpointAsync(checkpointType); - bool success; - (success, token) = task.AsTask().GetAwaiter().GetResult(); - ClassicAssert.IsTrue(success); - - using var s2 = store.NewSession>(new SimpleSimpleFunctions()); - var bc2 = s2.BasicContext; - - for (long key = 0; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public void RecoveryRollback( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType - ) - { - using var store = new TsavoriteKV(new() - { - IndexSize = 1L << 13, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 11, - SegmentSize = 1L << 11, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store.NewSession>(new SimpleSimpleFunctions()); - var bc1 = s1.BasicContext; - - for (long key = 0; key < 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - var task = store.TakeHybridLogCheckpointAsync(checkpointType); - (bool success, Guid token) = task.AsTask().GetAwaiter().GetResult(); - ClassicAssert.IsTrue(success); - - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePendingWithOutputs(out var completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found); - ClassicAssert.AreEqual(completedOutputs.Current.Key, completedOutputs.Current.Output, $"output = {completedOutputs.Current.Output}"); - } - completedOutputs.Dispose(); - - for (long key = 1000; key < 2000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - // Reset store to empty state - store.Reset(); - - for (long key = 0; key < 2000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.NotFound, $"status = {status}"); - } - } - _ = bc1.CompletePendingWithOutputs(out completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.NotFound); - } - completedOutputs.Dispose(); - - // Rollback to previous checkpoint - _ = store.Recover(default, token); - - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePendingWithOutputs(out completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found); - ClassicAssert.AreEqual(completedOutputs.Current.Key, completedOutputs.Current.Output, $"output = {completedOutputs.Current.Output}"); - } - completedOutputs.Dispose(); - - for (long key = 1000; key < 2000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.NotFound, $"status = {status}"); - } - } - _ = bc1.CompletePendingWithOutputs(out completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.NotFound); - } - completedOutputs.Dispose(); - - for (long key = 1000; key < 2000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - for (long key = 0; key < 2000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - else - { - _ = bc1.CompletePendingWithOutputs(out completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found); - ClassicAssert.AreEqual(completedOutputs.Current.Key, completedOutputs.Current.Output, $"output = {completedOutputs.Current.Output}"); - } - completedOutputs.Dispose(); - } - } - _ = bc1.CompletePendingWithOutputs(out completedOutputs, true); - while (completedOutputs.Next()) - { - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found); - ClassicAssert.AreEqual(completedOutputs.Current.Key, completedOutputs.Current.Output, $"output = {completedOutputs.Current.Output}"); - } - completedOutputs.Dispose(); - } - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheck3Tests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public async ValueTask RecoveryCheck3( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession>(new SimpleSimpleFunctions()); - var bc1 = s1.BasicContext; - - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - for (int i = 0; i < 5; i++) - { - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - var task = store1.TakeFullCheckpointAsync(checkpointType); - - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - _ = await store2.RecoverAsync(default, token).ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - _ = store2.Recover(default, token); - } - - ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); - ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); - ClassicAssert.AreEqual(store1.Log.TailAddress, store2.Log.TailAddress); - - using var s2 = store2.NewSession>(new SimpleSimpleFunctions()); - var bc2 = s2.BasicContext; - for (long key = 0; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } - - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheck4Tests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV"), Category("CheckpointRestore")] - public async ValueTask RecoveryCheck4( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession>(new SimpleSimpleFunctions()); - var bc1 = s1.BasicContext; - - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - for (int i = 0; i < 5; i++) - { - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 1000 * i; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - if (i == 0) - _ = store1.TakeIndexCheckpointAsync().AsTask().GetAwaiter().GetResult(); - - var task = store1.TakeHybridLogCheckpointAsync(checkpointType); - - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - _ = await store2.RecoverAsync(default, token).ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - _ = store2.Recover(default, token); - } - - ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); - ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); - ClassicAssert.AreEqual(store1.Log.TailAddress, store2.Log.TailAddress); - - using var s2 = store2.NewSession>(new SimpleSimpleFunctions()); - var bc2 = s2.BasicContext; - for (long key = 0; key < 1000 * i + 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } - - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheck5Tests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV")] - [Category("CheckpointRestore")] - public async ValueTask RecoveryCheck5( - [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, - [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession(new MyFunctions()); - var bc1 = s1.BasicContext; - for (long key = 0; key < 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - var result = await store1.GrowIndexAsync().ConfigureAwait(false); - ClassicAssert.IsTrue(result); - - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - - var task = store1.TakeFullCheckpointAsync(checkpointType); - - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - _ = await store2.RecoverAsync(default, token).ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - _ = store2.Recover(default, token); - } - - ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); - ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); - ClassicAssert.AreEqual(store1.Log.TailAddress, store2.Log.TailAddress); - - using var s2 = store2.NewSession(new MyFunctions()); - var bc2 = s2.BasicContext; - - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheckSnapshotTests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - [Test] - [Category("TsavoriteKV")] - [Category("CheckpointRestore")] - [Category("Smoke")] - public async ValueTask IncrSnapshotRecoveryCheck([Values] DeviceMode deviceMode) - { - ICheckpointManager checkpointManager; - if (deviceMode == DeviceMode.Local) - { - checkpointManager = new DeviceLogCommitCheckpointManager( - new LocalStorageNamedDeviceFactoryCreator(), - new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir + "/checkpoints/")); // PurgeAll deletes this directory - } - else - { - TestUtils.IgnoreIfNotRunningAzureTests(); - checkpointManager = new DeviceLogCommitCheckpointManager( - TestUtils.AzureStorageNamedDeviceFactoryCreator, - new AzureCheckpointNamingScheme($"{TestUtils.AzureTestContainer}/{TestUtils.AzureTestDirectory}")); - } - - await IncrSnapshotRecoveryCheck(checkpointManager).ConfigureAwait(false); - checkpointManager.PurgeAll(); - checkpointManager.Dispose(); - } - - private async ValueTask IncrSnapshotRecoveryCheck(ICheckpointManager checkpointManager) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = 1L << 16, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - CheckpointManager = checkpointManager - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession(new MyFunctions2()); - var bc1 = s1.BasicContext; - for (long key = 0; key < 1000; key++) - _ = bc1.Upsert(ref key, ref key); - - var task = store1.TakeHybridLogCheckpointAsync(CheckpointType.Snapshot); - var (success, token) = await task.ConfigureAwait(false); - - for (long key = 950; key < 1000; key++) - _ = bc1.Upsert(key, key + 1); - - var version1 = store1.CurrentVersion; - var _result1 = store1.TryInitiateHybridLogCheckpoint(out var _token1, CheckpointType.Snapshot, true); - await store1.CompleteCheckpointAsync().ConfigureAwait(false); - - ClassicAssert.IsTrue(_result1); - ClassicAssert.AreEqual(token, _token1); - - for (long key = 1000; key < 2000; key++) - _ = bc1.Upsert(key, key + 1); - - var version2 = store1.CurrentVersion; - var _result2 = store1.TryInitiateHybridLogCheckpoint(out var _token2, CheckpointType.Snapshot, true); - await store1.CompleteCheckpointAsync().ConfigureAwait(false); - - ClassicAssert.IsTrue(_result2); - ClassicAssert.AreEqual(token, _token2); - - // Test that we can recover to latest version - using var store2 = new TsavoriteKV(new() - { - IndexSize = 1L << 16, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 14, - CheckpointManager = checkpointManager - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - await store2.RecoverAsync(default, _token2).ConfigureAwait(false); - - ClassicAssert.AreEqual(store2.Log.TailAddress, store1.Log.TailAddress); - - using var s2 = store2.NewSession(new MyFunctions2()); - var bc2 = s2.BasicContext; - - for (long key = 0; key < 2000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - MyFunctions2.Verify(status, key, output); - } - } - _ = bc2.CompletePending(true); - - // Test that we can recover to earlier version - using var store3 = new TsavoriteKV(new() - { - IndexSize = 1L << 16, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 14, - CheckpointManager = checkpointManager - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - _ = await store3.RecoverAsync(recoverTo: version1).ConfigureAwait(false); - - ClassicAssert.IsTrue(store3.EntryCount == 1000); - using var s3 = store3.NewSession(new MyFunctions2()); - var bc3 = s3.BasicContext; - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc3.Read(ref key, ref output); - if (!status.IsPending) - { - MyFunctions2.Verify(status, key, output); - } - } - _ = bc3.CompletePending(true); - } - } - - [AllureNUnit] - [TestFixture] - public class RecoveryCheckStreamingSnapshotTests : RecoveryCheckBase - { - [SetUp] - public void Setup() => BaseSetup(); - - [TearDown] - public void TearDown() => BaseTearDown(); - - public class SnapshotIterator : IStreamingSnapshotIteratorFunctions - { - readonly TsavoriteKV store2; - readonly long expectedCount; - - ClientSession session2; - BasicContext bc2; - - public SnapshotIterator(TsavoriteKV store2, long expectedCount) - { - this.store2 = store2; - this.expectedCount = expectedCount; - } - - public bool OnStart(Guid checkpointToken, long currentVersion, long nextVersion) - { - store2.SetVersion(nextVersion); - session2 = store2.NewSession(new MyFunctions()); - bc2 = session2.BasicContext; - return true; - } - - public bool Reader(ref long key, ref long value, RecordMetadata recordMetadata, long numberOfRecords) - { - _ = bc2.Upsert(ref key, ref value); - return true; - } - - public void OnException(Exception exception, long numberOfRecords) - => Assert.Fail(exception.Message); - - public void OnStop(bool completed, long numberOfRecords) - { - Assert.That(numberOfRecords, Is.EqualTo(expectedCount)); - session2.Dispose(); - } - } - - [Test] - [Category("TsavoriteKV")] - [Category("CheckpointRestore")] - [Category("Smoke")] - - public async ValueTask StreamingSnapshotBasicTest([Values] bool isAsync, [Values] bool useReadCache, [Values] bool reInsert, [Values(1L << 13, 1L << 16)] long indexSize) - { - using var store1 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - using var s1 = store1.NewSession(new MyFunctions()); - var bc1 = s1.BasicContext; - - for (long key = 0; key < (reInsert ? 800 : 1000); key++) - { - // If reInsert, we insert the wrong key during the first pass for the first 500 keys - long value = reInsert && key < 500 ? key + 1 : key; - _ = bc1.Upsert(ref key, ref value); - } - - if (reInsert) - { - store1.Log.FlushAndEvict(true); - for (long key = 0; key < 500; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - for (long key = 800; key < 1000; key++) - { - _ = bc1.Upsert(ref key, ref key); - } - } - - if (useReadCache) - { - store1.Log.FlushAndEvict(true); - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc1.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc1.CompletePending(true); - } - - // First create the new store, we will insert into this store as part of the iterator functions on the old store - using var store2 = new TsavoriteKV(new() - { - IndexSize = indexSize, - LogDevice = log, - MutableFraction = 1, - PageSize = 1L << 10, - MemorySize = 1L << 20, - ReadCacheEnabled = useReadCache, - CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - // Take a streaming snapshot checkpoint of the old store - var iterator = new SnapshotIterator(store2, 1000); - var task = store1.TakeFullCheckpointAsync(CheckpointType.StreamingSnapshot, streamingSnapshotIteratorFunctions: iterator); - if (isAsync) - { - var (status, token) = await task.ConfigureAwait(false); - } - else - { - var (status, token) = task.AsTask().GetAwaiter().GetResult(); - } - - // Verify that the new store has all the records - using var s2 = store2.NewSession(new MyFunctions()); - var bc2 = s2.BasicContext; - for (long key = 0; key < 1000; key++) - { - long output = default; - var status = bc2.Read(ref key, ref output); - if (!status.IsPending) - { - ClassicAssert.IsTrue(status.Found, $"status = {status}"); - ClassicAssert.AreEqual(key, output, $"output = {output}"); - } - } - _ = bc2.CompletePending(true); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/RecoveryTestTypes.cs b/libs/storage/Tsavorite/cs/test/RecoveryTestTypes.cs index f5f9da583bd..8ab06b7e141 100644 --- a/libs/storage/Tsavorite/cs/test/RecoveryTestTypes.cs +++ b/libs/storage/Tsavorite/cs/test/RecoveryTestTypes.cs @@ -1,22 +1,53 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; using System.Threading; using Tsavorite.core; namespace Tsavorite.test.recovery.sumstore { - public struct AdId + public struct AdId : IKey { + public const int Size = sizeof(long); + public long adId; + // Not always pinned, so don't act like it is. + public readonly bool IsPinned => false; + + [UnscopedRef] + public readonly ReadOnlySpan KeyBytes => MemoryMarshal.AsBytes(new(in adId)); + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + public override string ToString() => adId.ToString(); - public struct Comparer : IKeyComparer + public struct Comparer : IKeyComparer { - public long GetHashCode64(ref AdId key) => Utility.GetHashCode(key.adId); - - public bool Equals(ref AdId k1, ref AdId k2) => k1.adId == k2.adId; + public long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(key.KeyBytes.AsRef().adId); + + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef().adId == k2.KeyBytes.AsRef().adId; } } @@ -30,6 +61,7 @@ public struct AdInput public struct NumClicks { + public const int Size = sizeof(long); public long numClicks; public override string ToString() => numClicks.ToString(); @@ -42,40 +74,36 @@ public struct Output public override string ToString() => value.ToString(); } - public class Functions : SessionFunctionsBase + public class Functions : SessionFunctionsBase { // Read functions - public override bool SingleReader(ref AdId key, ref AdInput input, ref NumClicks value, ref Output dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref AdInput input, ref Output output, ref ReadInfo readInfo) { - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref AdId key, ref AdInput input, ref NumClicks value, ref Output dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst.value = value; + output.value = srcLogRecord.ValueSpan.AsRef(); return true; } // RMW functions - public override bool InitialUpdater(ref AdId key, ref AdInput input, ref NumClicks value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value = input.numClicks; - return true; - } + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) + => dstLogRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref input.numClicks), in sizeInfo); - public override bool InPlaceUpdater(ref AdId key, ref AdInput input, ref NumClicks value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { - Interlocked.Add(ref value.numClicks, input.numClicks.numClicks); + _ = Interlocked.Add(ref logRecord.ValueSpan.AsRef().numClicks, input.numClicks.numClicks); return true; } - public override bool NeedCopyUpdate(ref AdId key, ref AdInput input, ref NumClicks oldValue, ref Output output, ref RMWInfo rmwInfo) => true; - - public override bool CopyUpdater(ref AdId key, ref AdInput input, ref NumClicks oldValue, ref NumClicks newValue, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { - newValue.numClicks += oldValue.numClicks + input.numClicks.numClicks; + dstLogRecord.ValueSpan.AsRef().numClicks += srcLogRecord.ValueSpan.AsRef().numClicks + input.numClicks.numClicks; return true; } + + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref AdInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = NumClicks.Size, ValueIsObject = false }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref AdInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = NumClicks.Size, ValueIsObject = false }; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SessionContextTestUtils.cs b/libs/storage/Tsavorite/cs/test/SessionContextTestUtils.cs new file mode 100644 index 00000000000..7de24d091b8 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/SessionContextTestUtils.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using NUnit.Framework.Legacy; +using Tsavorite.core; + +namespace Tsavorite.test.ReadCacheTests +{ + internal enum RecordRegion { Immutable, OnDisk, Mutable } + + internal static class ReadCacheChainTestUtils + { + internal static (long logicalAddress, long physicalAddress) SkipReadCacheChain(TsavoriteKV store, TestSpanByteKey key) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + var (la, pa) = GetHashChain(store, key, out _, out _, out bool isReadCache); + while (isReadCache) + (la, pa) = NextInChain(store, pa, out _, out _, ref isReadCache); + return (la, pa); + } + + static (long logicalAddress, long physicalAddress) GetHashChain(TsavoriteKV store, TestSpanByteKey key, out PinnedSpanByte recordKey, out bool invalid, out bool isReadCache) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + var tagExists = store.FindHashBucketEntryForKey(key, out var entry); + ClassicAssert.IsTrue(tagExists); + + isReadCache = entry.IsReadCache; + var log = isReadCache ? store.readcacheBase : store.hlogBase; + var pa = log.GetPhysicalAddress(entry.Address); + recordKey = PinnedSpanByte.FromPinnedSpan(LogRecord.GetInlineKey(pa)); + invalid = LogRecord.GetInfo(pa).Invalid; + + return (entry.Address, pa); + } + + static (long logicalAddress, long physicalAddress) NextInChain(TsavoriteKV store, long physicalAddress, out PinnedSpanByte recordKey, out bool invalid, ref bool isReadCache) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + { + var log = isReadCache ? store.readcacheBase : store.hlogBase; + var info = LogRecord.GetInfo(physicalAddress); + var la = info.PreviousAddress; + + isReadCache = LogAddress.IsReadCache(la); + log = isReadCache ? store.readcacheBase : store.hlogBase; + var pa = log.GetPhysicalAddress(la); + recordKey = PinnedSpanByte.FromPinnedSpan(LogRecord.GetInlineKey(pa)); + invalid = LogRecord.GetInfo(pa).Invalid; + return (la, pa); + } + } +} + +namespace Tsavorite.test.TransactionalUnsafeContext +{ + internal enum LockOperationType { Lock, Unlock } + + internal static class TransactionalUnsafeContextTestUtils + { + internal static IEnumerable EnumActionKeyIndices(FixedLengthTransactionalKeyStruct[] keys, LockOperationType lockOpType) + { + if (lockOpType == LockOperationType.Lock) + { + for (int ii = 0; ii < keys.Length; ++ii) + { + if (ii == 0 || keys[ii].KeyHash != keys[ii - 1].KeyHash) + yield return ii; + } + yield break; + } + + for (int ii = keys.Length - 1; ii >= 0; --ii) + { + if (ii == 0 || keys[ii].KeyHash != keys[ii - 1].KeyHash) + yield return ii; + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SharedDirectoryTests.cs b/libs/storage/Tsavorite/cs/test/SharedDirectoryTests.cs index 8873cab0f8d..4d20b12fbe3 100644 --- a/libs/storage/Tsavorite/cs/test/SharedDirectoryTests.cs +++ b/libs/storage/Tsavorite/cs/test/SharedDirectoryTests.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using Microsoft.Win32.SafeHandles; using NUnit.Framework; @@ -16,16 +15,14 @@ namespace Tsavorite.test.recovery.sumstore { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class SharedDirectoryTests : AllureTestBase + internal class SharedDirectoryTests : TestBase { - const long NumUniqueKeys = 1L << 5; + const int NumUniqueKeys = 1 << 5; const long KeySpace = 1L << 11; - const long NumOps = 1L << 10; + const int NumOps = 1 << 10; const long CompletePendingInterval = 1L << 10; private string sharedLogDirectory; TsavoriteTestInstance original; @@ -57,16 +54,16 @@ public void TearDown() public async ValueTask SharedLogDirectory([Values] bool isAsync) { original.Initialize(Path.Join(TestUtils.MethodTestDir, "OriginalCheckpoint"), sharedLogDirectory); - ClassicAssert.IsTrue(SharedDirectoryTests.IsDirectoryEmpty(sharedLogDirectory)); // sanity check - SharedDirectoryTests.Populate(original.Store); + ClassicAssert.IsTrue(IsDirectoryEmpty(sharedLogDirectory)); // sanity check + Populate(original.Store); // Take checkpoint from original to start the clone from ClassicAssert.IsTrue(original.Store.TryInitiateFullCheckpoint(out var checkpointGuid, CheckpointType.FoldOver)); original.Store.CompleteCheckpointAsync().GetAwaiter().GetResult(); // Sanity check against original - ClassicAssert.IsFalse(SharedDirectoryTests.IsDirectoryEmpty(sharedLogDirectory)); - SharedDirectoryTests.Test(original, checkpointGuid); + ClassicAssert.IsFalse(IsDirectoryEmpty(sharedLogDirectory)); + Test(original, checkpointGuid); // Copy checkpoint directory var cloneCheckpointDirectory = Path.Join(TestUtils.MethodTestDir, "CloneCheckpoint"); @@ -76,40 +73,40 @@ public async ValueTask SharedLogDirectory([Values] bool isAsync) clone.Initialize(cloneCheckpointDirectory, sharedLogDirectory, populateLogHandles: true); if (isAsync) - await clone.Store.RecoverAsync(checkpointGuid).ConfigureAwait(false); + _ = await clone.Store.RecoverAsync(checkpointGuid).ConfigureAwait(false); else - clone.Store.Recover(checkpointGuid); + _ = clone.Store.Recover(checkpointGuid); // Both sessions should work concurrently - SharedDirectoryTests.Test(original, checkpointGuid); - SharedDirectoryTests.Test(clone, checkpointGuid); + Test(original, checkpointGuid); + Test(clone, checkpointGuid); // Dispose original, files should not be deleted on Windows original.TearDown(); - if (RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { // Clone should still work on Windows - ClassicAssert.IsFalse(SharedDirectoryTests.IsDirectoryEmpty(sharedLogDirectory)); - SharedDirectoryTests.Test(clone, checkpointGuid); + ClassicAssert.IsFalse(IsDirectoryEmpty(sharedLogDirectory)); + Test(clone, checkpointGuid); } clone.TearDown(); // Files should be deleted after both instances are closed - ClassicAssert.IsTrue(SharedDirectoryTests.IsDirectoryEmpty(sharedLogDirectory)); + ClassicAssert.IsTrue(IsDirectoryEmpty(sharedLogDirectory)); } private struct TsavoriteTestInstance { public string CheckpointDirectory { get; private set; } public string LogDirectory { get; private set; } - public TsavoriteKV Store { get; private set; } + public TsavoriteKV Store { get; private set; } public IDevice LogDevice { get; private set; } public void Initialize(string checkpointDirectory, string logDirectory, bool populateLogHandles = false) { - if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) populateLogHandles = false; CheckpointDirectory = checkpointDirectory; @@ -137,21 +134,16 @@ public void Initialize(string checkpointDirectory, string logDirectory, bool pop } } - if (!RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows)) - { - LogDevice = new ManagedLocalStorageDevice(deviceFileName, deleteOnClose: true); - } - else - { - LogDevice = new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles); - } + LogDevice = !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? new ManagedLocalStorageDevice(deviceFileName, deleteOnClose: true) + : new LocalStorageDevice(deviceFileName, deleteOnClose: true, disableFileBuffering: false, initialLogFileHandles: initialHandles); Store = new(new() { IndexSize = KeySpace, LogDevice = LogDevice, CheckpointDir = CheckpointDirectory - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -165,13 +157,13 @@ public void TearDown() } } - private static void Populate(TsavoriteKV store) + private static void Populate(TsavoriteKV store) { - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; // Prepare the dataset - var inputArray = new AdInput[NumOps]; + var inputArray = GC.AllocateArray(NumOps, pinned: true); for (int i = 0; i < NumOps; i++) { inputArray[i].adId.adId = i % NumUniqueKeys; @@ -181,12 +173,10 @@ private static void Populate(TsavoriteKV(NumUniqueKeys, pinned: true); for (int i = 0; i < NumUniqueKeys; i++) { inputArray[i].adId.adId = i; @@ -213,13 +203,13 @@ private static void Test(TsavoriteTestInstance tsavoriteInstance, Guid checkpoin var input = default(AdInput); var output = default(Output); - using var session = tsavoriteInstance.Store.NewSession(new Functions()); + using var session = tsavoriteInstance.Store.NewSession(new Functions()); var bContext = session.BasicContext; // Issue read requests for (var i = 0; i < NumUniqueKeys; i++) { - var status = bContext.Read(ref inputArray[i].adId, ref input, ref output, Empty.Default); + var status = bContext.Read(inputArray[i].adId, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); inputArray[i].numClicks = output.value; } @@ -235,9 +225,7 @@ private static void CopyDirectory(DirectoryInfo source, DirectoryInfo target) { // Copy each file foreach (var file in source.GetFiles()) - { _ = file.CopyTo(Path.Combine(target.FullName, file.Name), true); - } // Copy each subdirectory foreach (var sourceSubDirectory in source.GetDirectories()) diff --git a/libs/storage/Tsavorite/cs/test/SimpleTests.cs b/libs/storage/Tsavorite/cs/test/SimpleTests.cs index b448aee658f..349023f58a7 100644 --- a/libs/storage/Tsavorite/cs/test/SimpleTests.cs +++ b/libs/storage/Tsavorite/cs/test/SimpleTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,9 +9,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class SimpleTests : AllureTestBase + internal class SimpleTests : TestBase { [Test] [Category("TsavoriteKV")] diff --git a/libs/storage/Tsavorite/cs/test/SimpleVersionSchemeTest.cs b/libs/storage/Tsavorite/cs/test/SimpleVersionSchemeTest.cs index b123e81099d..4906b719a25 100644 --- a/libs/storage/Tsavorite/cs/test/SimpleVersionSchemeTest.cs +++ b/libs/storage/Tsavorite/cs/test/SimpleVersionSchemeTest.cs @@ -4,16 +4,14 @@ using System; using System.Collections.Generic; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class SimpleVersionSchemeTest : AllureTestBase + internal class SimpleVersionSchemeTest : TestBase { [Test] [Category("TsavoriteLog")] diff --git a/libs/storage/Tsavorite/cs/test/SingleWriterTests.cs b/libs/storage/Tsavorite/cs/test/SingleWriterTests.cs deleted file mode 100644 index 3ad982abe34..00000000000 --- a/libs/storage/Tsavorite/cs/test/SingleWriterTests.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test.SingleWriter -{ - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; - - internal class SingleWriterTestFunctions : SimpleSimpleFunctions - { - internal WriteReason actualReason; - - public override bool SingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - ClassicAssert.AreEqual((WriteReason)input, reason); - actualReason = reason; - return true; - } - - public override void PostSingleWriter(ref int key, ref int input, ref int src, ref int dst, ref int output, ref UpsertInfo upsertInfo, WriteReason reason) - { - ClassicAssert.AreEqual((WriteReason)input, reason); - actualReason = reason; - } - } - - [AllureNUnit] - [TestFixture] - class SingleWriterTests : AllureTestBase - { - const int NumRecords = 1000; - const int ValueMult = 1_000_000; - const WriteReason NoReason = (WriteReason)255; - - SingleWriterTestFunctions functions; - - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; - private IDevice log; - - [SetUp] - public void Setup() - { - DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.log"), deleteOnClose: false); - - functions = new(); - KVSettings kvSettings = new() - { - IndexSize = 1L << 26, - LogDevice = log, - PageSize = 1L << 12, - MemorySize = 1L << 22, - ReadCopyOptions = new(ReadCopyFrom.Device, ReadCopyTo.MainLog), - CheckpointDir = MethodTestDir - }; - foreach (var arg in TestContext.CurrentContext.Test.Arguments) - { - if (arg is ReadCopyDestination dest) - { - if (dest == ReadCopyDestination.ReadCache) - { - kvSettings.ReadCachePageSize = 1L << 12; - kvSettings.ReadCacheMemorySize = 1L << 22; - kvSettings.ReadCacheEnabled = true; - kvSettings.ReadCopyOptions = default; - } - break; - } - } - - store = new(kvSettings - , StoreFunctions.Create(IntKeyComparer.Instance) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - session = store.NewSession(functions); - bContext = session.BasicContext; - } - - [TearDown] - public void TearDown() - { - session?.Dispose(); - session = null; - store?.Dispose(); - store = null; - log?.Dispose(); - log = null; - OnTearDown(); - } - - void Populate() - { - int input = (int)WriteReason.Upsert; - int output = 0; - for (int key = 0; key < NumRecords; key++) - ClassicAssert.False(bContext.Upsert(key, input, key * ValueMult, ref output).IsPending); - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public void SingleWriterReasonsTest([Values] ReadCopyDestination readCopyDestination) - { - functions.actualReason = NoReason; - Populate(); - ClassicAssert.AreEqual(WriteReason.Upsert, functions.actualReason); - - store.Log.FlushAndEvict(wait: true); - - functions.actualReason = NoReason; - int key = 42; - WriteReason expectedReason = readCopyDestination == ReadCopyDestination.ReadCache ? WriteReason.CopyToReadCache : WriteReason.CopyToTail; - int input = (int)expectedReason; - var status = bContext.Read(key, input, out int output); - ClassicAssert.IsTrue(status.IsPending); - _ = bContext.CompletePending(wait: true); - ClassicAssert.AreEqual(expectedReason, functions.actualReason); - - functions.actualReason = NoReason; - key = 64; - expectedReason = WriteReason.CopyToTail; - input = (int)expectedReason; - ReadOptions readOptions = new() { CopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog) }; - status = bContext.Read(ref key, ref input, ref output, ref readOptions, out _); - ClassicAssert.IsTrue(status.IsPending && !status.IsCompleted); - _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); - (status, output) = GetSinglePendingResult(outputs); - ClassicAssert.IsTrue(!status.IsPending && status.IsCompleted && status.IsCompletedSuccessfully); - ClassicAssert.IsTrue(status.Found && !status.NotFound && status.Record.Copied); - ClassicAssert.AreEqual(expectedReason, functions.actualReason); - - functions.actualReason = NoReason; - expectedReason = WriteReason.Compaction; - input = (int)expectedReason; - _ = store.Log.Compact(functions, ref input, ref output, store.Log.SafeReadOnlyAddress, CompactionType.Scan); - ClassicAssert.AreEqual(expectedReason, functions.actualReason); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SpanByteTests.cs b/libs/storage/Tsavorite/cs/test/SpanByteTests.cs deleted file mode 100644 index 7fc33216f76..00000000000 --- a/libs/storage/Tsavorite/cs/test/SpanByteTests.cs +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Runtime.InteropServices; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.core.Utility; - -namespace Tsavorite.test.spanbyte -{ - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] - [TestFixture] - internal class SpanByteTests : AllureTestBase - { - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public unsafe void SpanByteTest1() - { - Span output = stackalloc byte[20]; - SpanByte input = default; - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - - try - { - using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "hlog1.log"), deleteOnClose: true); - using var store = new TsavoriteKV>( - new() - { - IndexSize = 1L << 13, - LogDevice = log, - MemorySize = 1L << 17, - PageSize = 1L << 12 - }, StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - using var session = store.NewSession>(new SpanByteFunctions()); - var bContext = session.BasicContext; - - var key1 = MemoryMarshal.Cast("key1".AsSpan()); - var value1 = MemoryMarshal.Cast("value1".AsSpan()); - var output1 = SpanByteAndMemory.FromPinnedSpan(output); - - fixed (byte* key1Ptr = key1) - fixed (byte* value1Ptr = value1) - { - var key1SpanByte = SpanByte.FromPinnedPointer(key1Ptr, key1.Length); - var value1SpanByte = SpanByte.FromPinnedPointer(value1Ptr, value1.Length); - - _ = bContext.Upsert(key1SpanByte, value1SpanByte); - _ = bContext.Read(ref key1SpanByte, ref input, ref output1); - } - - ClassicAssert.IsTrue(output1.IsSpanByte); - ClassicAssert.IsTrue(output1.SpanByte.AsReadOnlySpan().SequenceEqual(value1)); - - var key2 = MemoryMarshal.Cast("key2".AsSpan()); - var value2 = MemoryMarshal.Cast("value2value2value2".AsSpan()); - var output2 = SpanByteAndMemory.FromPinnedSpan(output); - - fixed (byte* key2Ptr = key2) - fixed (byte* value2Ptr = value2) - { - var key2SpanByte = SpanByte.FromPinnedPointer(key2Ptr, key2.Length); - var value2SpanByte = SpanByte.FromPinnedPointer(value2Ptr, value2.Length); - - _ = bContext.Upsert(key2SpanByte, value2SpanByte); - _ = bContext.Read(ref key2SpanByte, ref input, ref output2); - } - - ClassicAssert.IsTrue(!output2.IsSpanByte); - ClassicAssert.IsTrue(output2.Memory.Memory.Span.Slice(0, output2.Length).SequenceEqual(value2)); - output2.Memory.Dispose(); - } - finally - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir); - } - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public unsafe void MultiRead_SpanByte_Test() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - - try - { - using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "test.log"), deleteOnClose: true); - using var store = new TsavoriteKV>( - new() - { - IndexSize = 1L << 16, - LogDevice = log, - MemorySize = 1L << 15, - PageSize = 1L << 12 - }, StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - using var session = store.NewSession>(new SpanByteFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < 200; i++) - { - var key = MemoryMarshal.Cast($"{i}".AsSpan()); - var value = MemoryMarshal.Cast($"{i + 1000}".AsSpan()); - fixed (byte* k = key, v = value) - _ = bContext.Upsert(SpanByte.FromPinnedSpan(key), SpanByte.FromPinnedSpan(value)); - } - - // Read, evict all records to disk, read again - MultiRead(evicted: false); - store.Log.FlushAndEvict(true); - MultiRead(evicted: true); - - void MultiRead(bool evicted) - { - for (long key = 0; key < 50; key++) - { - // read each key multiple times - for (int i = 0; i < 10; i++) - ReadKey(key, key + 1000, evicted); - } - } - - void ReadKey(long key, long value, bool evicted) - { - Status status; - SpanByteAndMemory output = default; - - var keyBytes = MemoryMarshal.Cast($"{key}".AsSpan()); - fixed (byte* _ = keyBytes) - status = bContext.Read(key: SpanByte.FromPinnedSpan(keyBytes), out output); - ClassicAssert.AreEqual(evicted, status.IsPending, "evicted/pending mismatch"); - - if (evicted) - (status, output) = bContext.GetSinglePendingResult(); - ClassicAssert.IsTrue(status.Found, $"expected to find key; status = {status}, pending = {evicted}"); - - ClassicAssert.IsFalse(output.IsSpanByte, "Output should not have a valid SpanByte"); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - ClassicAssert.AreEqual(value, long.Parse(outputString), $"outputString mismatch; pending = {evicted}"); - output.Memory.Dispose(); - } - } - finally - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir); - } - } - - [Test] - [Category("TsavoriteKV")] - [Category("Smoke")] - public unsafe void SpanByteUnitTest1() - { - Span payload = stackalloc byte[20]; - Span serialized = stackalloc byte[24]; - - SpanByte sb = SpanByte.FromPinnedSpan(payload); - ClassicAssert.IsFalse(sb.Serialized); - ClassicAssert.AreEqual(20, sb.Length); - ClassicAssert.AreEqual(24, sb.TotalSize); - ClassicAssert.AreEqual(20, sb.AsSpan().Length); - ClassicAssert.AreEqual(20, sb.AsReadOnlySpan().Length); - - fixed (byte* ptr = serialized) - sb.CopyTo(ptr); - ref SpanByte ssb = ref SpanByte.ReinterpretWithoutLength(serialized); - ClassicAssert.IsTrue(ssb.Serialized); - ClassicAssert.AreEqual(0, ssb.MetadataSize); - ClassicAssert.AreEqual(20, ssb.Length); - ClassicAssert.AreEqual(24, ssb.TotalSize); - ClassicAssert.AreEqual(20, ssb.AsSpan().Length); - ClassicAssert.AreEqual(20, ssb.AsReadOnlySpan().Length); - - ssb.MarkExtraMetadata(); - ClassicAssert.IsTrue(ssb.Serialized); - ClassicAssert.AreEqual(8, ssb.MetadataSize); - ClassicAssert.AreEqual(20, ssb.Length); - ClassicAssert.AreEqual(24, ssb.TotalSize); - ClassicAssert.AreEqual(20 - 8, ssb.AsSpan().Length); - ClassicAssert.AreEqual(20 - 8, ssb.AsReadOnlySpan().Length); - ssb.ExtraMetadata = 31337; - ClassicAssert.AreEqual(31337, ssb.ExtraMetadata); - - sb.MarkExtraMetadata(); - ClassicAssert.AreEqual(20, sb.Length); - ClassicAssert.AreEqual(24, sb.TotalSize); - ClassicAssert.AreEqual(20 - 8, sb.AsSpan().Length); - ClassicAssert.AreEqual(20 - 8, sb.AsReadOnlySpan().Length); - sb.ExtraMetadata = 31337; - ClassicAssert.AreEqual(31337, sb.ExtraMetadata); - - fixed (byte* ptr = serialized) - sb.CopyTo(ptr); - ClassicAssert.IsTrue(ssb.Serialized); - ClassicAssert.AreEqual(8, ssb.MetadataSize); - ClassicAssert.AreEqual(20, ssb.Length); - ClassicAssert.AreEqual(24, ssb.TotalSize); - ClassicAssert.AreEqual(20 - 8, ssb.AsSpan().Length); - ClassicAssert.AreEqual(20 - 8, ssb.AsReadOnlySpan().Length); - ClassicAssert.AreEqual(31337, ssb.ExtraMetadata); - } - - [Test] - [Category("TsavoriteKV")] - public unsafe void ShouldSkipEmptySpaceAtEndOfPage() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - - using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "vl-iter.log"), deleteOnClose: true); - using var store = new TsavoriteKV>( - new() - { - IndexSize = 1L << 13, - LogDevice = log, - MemorySize = 1L << 17, - PageSize = 1L << 10 // 1KB page - }, StoreFunctions.Create() - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - using var session = store.NewSession(new VLVectorFunctions()); - var bContext = session.BasicContext; - - const int PageSize = 1024; - Span keySpan = stackalloc long[1]; - var key = keySpan.AsSpanByte(); - Span valueSpan = stackalloc byte[PageSize]; - var value = valueSpan.AsSpanByte(); // We'll adjust the length below - - Set(ref keySpan, 1L, ref valueSpan, 800, 1); // Inserted on page#0 and leaves empty space - Set(ref keySpan, 2L, ref valueSpan, 800, 2); // Inserted on page#1 because there is not enough space in page#0, and leaves empty space - - // Add a second record on page#1 to fill it exactly. Page#1 starts at offset 0 on the page (unlike page#0, which starts at 24 or 64, - // depending on data). Subtract the RecordInfo and key space for both the first record and the second record we're about to insert, - // the value space for the first record, and the length header for the second record. This is the space available for the second record's value. - var p2value2len = PageSize - - 2 * RecordInfo.GetLength() - - 2 * RoundUp(key.TotalSize, Constants.kRecordAlignment) - - RoundUp(value.TotalSize, Constants.kRecordAlignment) - - sizeof(int); - Set(ref keySpan, 3L, ref valueSpan, p2value2len, 3); // Inserted on page#1 - ClassicAssert.AreEqual(PageSize * 2, store.Log.TailAddress, "TailAddress should be at the end of page#2"); - - Set(ref keySpan, 4L, ref valueSpan, 64, 4); // Inserted on page#2 - - var data = new List<(long, int, int)>(); - using (var iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress)) - { - while (iterator.GetNext(out var info)) - { - var scanKey = iterator.GetKey().AsSpan(); - var scanValue = iterator.GetValue().AsSpan(); - - data.Add((scanKey[0], scanValue.Length, scanValue[0])); - } - } - - ClassicAssert.AreEqual(4, data.Count); - - ClassicAssert.AreEqual((1L, 800, 1), data[0]); - ClassicAssert.AreEqual((2L, 800, 2), data[1]); - ClassicAssert.AreEqual((3L, p2value2len, 3), data[2]); - ClassicAssert.AreEqual((4L, 64, 4), data[3]); - - TestUtils.DeleteDirectory(TestUtils.MethodTestDir); - - void Set(ref Span keySpan, long keyValue, ref Span valueSpan, int valueLength, byte tag) - { - keySpan[0] = keyValue; - value.Length = valueLength; - valueSpan[0] = tag; - _ = bContext.Upsert(ref key, ref value, Empty.Default); - } - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs b/libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs index 51c205b76b2..13c62255e1f 100644 --- a/libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs +++ b/libs/storage/Tsavorite/cs/test/StateMachineDriverTests.cs @@ -5,7 +5,6 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,10 +13,12 @@ namespace Tsavorite.test.recovery { using static Tsavorite.test.TestUtils; - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; - public abstract class StateMachineDriverTestsBase : AllureTestBase + public enum TimeFuzzMode { TimeFuzz, NoTimeFuzz }; + + public abstract class StateMachineDriverTestsBase : TestBase { readonly int numOpThreads = 2; protected readonly int numKeys = 4; @@ -44,19 +45,19 @@ protected void BaseTearDown() OnTearDown(waitForDelete: true); } - protected abstract void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store); + protected abstract void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store); public async ValueTask DoCheckpointVersionSwitchEquivalenceCheck(CheckpointType checkpointType, long indexSize, bool useTimingFuzzing) { // Create the original store - using var store1 = new TsavoriteKV(new() + using var store1 = new TsavoriteKV(new() { IndexSize = indexSize, LogDevice = log, PageSize = 1L << 10, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, CheckpointDir = MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -89,12 +90,12 @@ public async ValueTask DoCheckpointVersionSwitchEquivalenceCheck(CheckpointType await Task.WhenAll(opTasks).ConfigureAwait(false); // Verify the final state of the old store - using var s1 = store1.NewSession(new SumFunctions(0, false)); + using var s1 = store1.NewSession(new SumFunctions(0, false)); var bc1 = s1.BasicContext; for (long key = 0; key < numKeys; key++) { long output = default; - var status = bc1.Read(ref key, ref output); + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); if (status.IsPending) { var completed = bc1.CompletePendingWithOutputs(out var completedOutputs, true); @@ -113,26 +114,26 @@ public async ValueTask DoCheckpointVersionSwitchEquivalenceCheck(CheckpointType } // Recover new store from the checkpoint - using var store2 = new TsavoriteKV(new() + using var store2 = new TsavoriteKV(new() { IndexSize = indexSize, LogDevice = log, MutableFraction = 1, PageSize = 1L << 10, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, CheckpointDir = MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); _ = await store2.RecoverAsync(default, checkpointToken).ConfigureAwait(false); // Verify the state of the new store - using var s2 = store2.NewSession(new SumFunctions(0, false)); + using var s2 = store2.NewSession(new SumFunctions(0, false)); var bc2 = s2.BasicContext; for (long key = 0; key < numKeys; key++) { long output = default; - var status = bc2.Read(ref key, ref output); + var status = bc2.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); if (status.IsPending) { var completed = bc2.CompletePendingWithOutputs(out var completedOutputs, true); @@ -161,14 +162,14 @@ public async ValueTask DoCheckpointVersionSwitchEquivalenceCheck(CheckpointType public async ValueTask DoGrowIndexVersionSwitchEquivalenceCheck(long indexSize, bool useTimingFuzzing) { // Create the original store - using var store1 = new TsavoriteKV(new() + using var store1 = new TsavoriteKV(new() { IndexSize = indexSize, LogDevice = log, PageSize = 1L << 10, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, CheckpointDir = MethodTestDir - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -199,12 +200,12 @@ public async ValueTask DoGrowIndexVersionSwitchEquivalenceCheck(long indexSize, await Task.WhenAll(opTasks).ConfigureAwait(false); // Verify the final state of the store - using var s1 = store1.NewSession(new SumFunctions(0, false)); + using var s1 = store1.NewSession(new SumFunctions(0, false)); var bc1 = s1.BasicContext; for (long key = 0; key < numKeys; key++) { long output = default; - var status = bc1.Read(ref key, ref output); + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); if (status.IsPending) { var completed = bc1.CompletePendingWithOutputs(out var completedOutputs, true); @@ -224,7 +225,7 @@ public async ValueTask DoGrowIndexVersionSwitchEquivalenceCheck(long indexSize, } } - public class SumFunctions : SimpleSimpleFunctions + public class SumFunctions : SimpleLongSimpleFunctions { readonly Random fuzzer; @@ -233,30 +234,29 @@ public SumFunctions(int thread_id, bool useTimingFuzzing) : base((l, r) => l + r if (useTimingFuzzing) fuzzer = new Random(thread_id); } - public override bool InPlaceUpdater(ref long key, ref long input, ref long value, ref long output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref long input, ref long output, ref RMWInfo rmwInfo) { Fuzz(); - var ret = base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + var ret = base.InPlaceUpdater(ref logRecord, ref input, ref output, ref rmwInfo); Fuzz(); return ret; } - public override bool CopyUpdater(ref long key, ref long input, ref long oldValue, ref long newValue, ref long output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref long input, ref long output, ref RMWInfo rmwInfo) { Fuzz(); - var ret = base.CopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo, ref recordInfo); + var ret = base.CopyUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); Fuzz(); return ret; } void Fuzz() { - if (fuzzer != null) Thread.Sleep(fuzzer.Next(30)); + if (fuzzer != null) + Thread.Sleep(fuzzer.Next(30)); } } } - - [AllureNUnit] [TestFixture] public class CheckpointVersionSwitchRmw : StateMachineDriverTestsBase { @@ -266,9 +266,9 @@ public class CheckpointVersionSwitchRmw : StateMachineDriverTestsBase [TearDown] public void TearDown() => BaseTearDown(); - protected override void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store) + protected override void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store) { - using var s = store.NewSession(new SumFunctions(thread_id, useTimingFuzzing)); + using var s = store.NewSession(new SumFunctions(thread_id, useTimingFuzzing)); var bc = s.BasicContext; var r = new Random(thread_id); @@ -282,7 +282,7 @@ protected override void OperationThread(int thread_id, bool useTimingFuzzing, Ts key = r.Next(numKeys); // Run the RMW operation - _ = bc.RMW(ref key, ref input); + _ = bc.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input); // Update expected counts for the old and new version of store if (bc.Session.Version == currentIteration + 1) @@ -299,20 +299,20 @@ protected override void OperationThread(int thread_id, bool useTimingFuzzing, Ts } [Test] + //[Repeat(1000)] public async ValueTask CheckpointVersionSwitchRmwTest( [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, [Values(1L << 13, 1L << 16)] long indexSize, - [Values] bool useTimingFuzzing) - => await DoCheckpointVersionSwitchEquivalenceCheck(checkpointType, indexSize, useTimingFuzzing).ConfigureAwait(false); + [Values] TimeFuzzMode timeFuzzMode) + => await DoCheckpointVersionSwitchEquivalenceCheck(checkpointType, indexSize, timeFuzzMode == TimeFuzzMode.TimeFuzz).ConfigureAwait(false); [Test] + //[Repeat(1000)] public async ValueTask GrowIndexVersionSwitchRmwTest( [Values(1L << 13, 1L << 16)] long indexSize, - [Values] bool useTimingFuzzing) - => await DoGrowIndexVersionSwitchEquivalenceCheck(indexSize, useTimingFuzzing).ConfigureAwait(false); + [Values] TimeFuzzMode timeFuzzMode) + => await DoGrowIndexVersionSwitchEquivalenceCheck(indexSize, timeFuzzMode == TimeFuzzMode.TimeFuzz).ConfigureAwait(false); } - - [AllureNUnit] [TestFixture] public class CheckpointVersionSwitchTxn : StateMachineDriverTestsBase { @@ -322,10 +322,10 @@ public class CheckpointVersionSwitchTxn : StateMachineDriverTestsBase [TearDown] public void TearDown() => BaseTearDown(); - protected override void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store) + protected override void OperationThread(int thread_id, bool useTimingFuzzing, TsavoriteKV store) { - using var s = store.NewSession(new SumFunctions(thread_id, useTimingFuzzing)); - var lc = s.LockableContext; + using var s = store.NewSession(new SumFunctions(thread_id, useTimingFuzzing)); + var lc = s.TransactionalContext; var r = new Random(thread_id); ClassicAssert.IsTrue(numKeys > 1); @@ -342,31 +342,31 @@ protected override void OperationThread(int thread_id, bool useTimingFuzzing, Ts key2 = r.Next(numKeys); } while (key2 == key1); - var exclusiveVec = new FixedLengthLockableKeyStruct[] { - new(key1, LockType.Exclusive, lc), - new(key2, LockType.Exclusive, lc) + var exclusiveVec = new FixedLengthTransactionalKeyStruct[] { + new(SpanByte.FromPinnedVariable(ref key1), LockType.Exclusive, lc), + new(SpanByte.FromPinnedVariable(ref key2), LockType.Exclusive, lc) }; var txnVersion = store.stateMachineDriver.AcquireTransactionVersion(); // Start transaction, session does not acquire version in this call - lc.BeginLockable(); + lc.BeginTransaction(); // Lock keys, session acquires version in this call - lc.Lock>(exclusiveVec); + lc.Lock(exclusiveVec); txnVersion = store.stateMachineDriver.VerifyTransactionVersion(txnVersion); lc.LocksAcquired(txnVersion); // Run transaction - _ = lc.RMW(ref key1, ref input); - _ = lc.RMW(ref key2, ref input); + _ = lc.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key1)), ref input); + _ = lc.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key2)), ref input); // Unlock keys - lc.Unlock>(exclusiveVec); + lc.Unlock(exclusiveVec); // End transaction - lc.EndLockable(); + lc.EndTransaction(); store.stateMachineDriver.EndTransaction(txnVersion); @@ -392,13 +392,13 @@ protected override void OperationThread(int thread_id, bool useTimingFuzzing, Ts public async ValueTask CheckpointVersionSwitchTxnTest( [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, [Values(1L << 13, 1L << 16)] long indexSize, - [Values] bool useTimingFuzzing) - => await DoCheckpointVersionSwitchEquivalenceCheck(checkpointType, indexSize, useTimingFuzzing).ConfigureAwait(false); + [Values] TimeFuzzMode timeFuzzMode) + => await DoCheckpointVersionSwitchEquivalenceCheck(checkpointType, indexSize, timeFuzzMode == TimeFuzzMode.TimeFuzz).ConfigureAwait(false); [Test] public async ValueTask GrowIndexVersionSwitchTxnTest( [Values(1L << 13, 1L << 16)] long indexSize, - [Values] bool useTimingFuzzing) - => await DoGrowIndexVersionSwitchEquivalenceCheck(indexSize, useTimingFuzzing).ConfigureAwait(false); + [Values] TimeFuzzMode timeFuzzMode) + => await DoGrowIndexVersionSwitchEquivalenceCheck(indexSize, timeFuzzMode == TimeFuzzMode.TimeFuzz).ConfigureAwait(false); } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/StructWithStringTests.cs b/libs/storage/Tsavorite/cs/test/StructWithStringTests.cs deleted file mode 100644 index 27ed5f726e4..00000000000 --- a/libs/storage/Tsavorite/cs/test/StructWithStringTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.IO; -using Allure.NUnit; -using Garnet.test; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; -using static Tsavorite.test.TestUtils; - -namespace Tsavorite.test.StructWithString -{ - // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - public struct StructWithString(int intValue, string prefix) - { - public int intField = intValue; - public string stringField = prefix + intValue.ToString(); - - public override readonly string ToString() => stringField; - - public class Comparer : IKeyComparer - { - public long GetHashCode64(ref StructWithString k) => Utility.GetHashCode(k.intField); - - public bool Equals(ref StructWithString k1, ref StructWithString k2) => k1.intField == k2.intField && k1.stringField == k2.stringField; - } - - public class Serializer : BinaryObjectSerializer - { - public override void Deserialize(out StructWithString obj) - { - var intField = reader.ReadInt32(); - var stringField = reader.ReadString(); - obj = new() { intField = intField, stringField = stringField }; - } - - public override void Serialize(ref StructWithString obj) - { - writer.Write(obj.intField); - writer.Write(obj.stringField); - } - } - } -} - -namespace Tsavorite.test.StructWithString -{ - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - public class StructWithStringTests : AllureTestBase - { - internal class StructWithStringTestFunctions : SimpleSimpleFunctions - { - } - - const int NumRecords = 1_000; - const string KeyPrefix = "key_"; - string valuePrefix = "value_"; - - StructWithStringTestFunctions functions; - - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; - private IDevice log, objlog; - - [SetUp] - public void Setup() - { - // create a string of size 1024 bytes - valuePrefix = new string('a', 1024); - - DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.log"), deleteOnClose: false); - objlog = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.obj.log"), deleteOnClose: false); - - store = new(new() - { - IndexSize = 1L << 26, - LogDevice = log, - ObjectLogDevice = objlog, - PageSize = 1L << 10, - MemorySize = 1L << 22, - SegmentSize = 1L << 16, - CheckpointDir = MethodTestDir - }, StoreFunctions.Create(new StructWithString.Comparer(), () => new StructWithString.Serializer(), () => new StructWithString.Serializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) - ); - - functions = new(); - session = store.NewSession(functions); - bContext = session.BasicContext; - } - - [TearDown] - public void TearDown() - { - session?.Dispose(); - session = null; - store?.Dispose(); - store = null; - objlog?.Dispose(); - objlog = null; - log?.Dispose(); - log = null; - OnTearDown(); - } - - void Populate() - { - for (int ii = 0; ii < NumRecords; ii++) - { - StructWithString key = new(ii, KeyPrefix); - StructWithString value = new(ii, valuePrefix); - bContext.Upsert(ref key, ref value); - if (ii % 3_000 == 0) - { - store.TakeHybridLogCheckpointAsync(CheckpointType.FoldOver).GetAwaiter().GetResult(); - store.Recover(); - } - } - } - - [Test] - [Category(TsavoriteKVTestCategory)] - [Category(SmokeTestCategory)] - public void StructWithStringCompactTest([Values] CompactionType compactionType, [Values] bool flush) - { - void readKey(int keyInt) - { - StructWithString key = new(keyInt, KeyPrefix); - var (status, output) = bContext.Read(key); - bool wasPending = status.IsPending; - if (status.IsPending) - { - bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); - using (completedOutputs) - (status, output) = GetSinglePendingResult(completedOutputs); - } - - ClassicAssert.IsTrue(status.Found, $"{status.ToString()}; wasPending = {wasPending}"); - ClassicAssert.AreEqual(key.intField, output.intField); - } - - Populate(); - readKey(12); - if (flush) - { - store.Log.FlushAndEvict(wait: true); - readKey(24); - } - int count = 0; - using var iter = store.Log.Scan(0, store.Log.TailAddress); - while (iter.GetNext(out var _)) - count++; - ClassicAssert.AreEqual(count, NumRecords); - - readKey(48); - store.Log.Compact(functions, store.Log.SafeReadOnlyAddress, compactionType); - readKey(48); - } - } -} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/TestSpanByteKey.cs b/libs/storage/Tsavorite/cs/test/TestSpanByteKey.cs new file mode 100644 index 00000000000..bc3b6f0ff89 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/TestSpanByteKey.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using Tsavorite.core; + +namespace Tsavorite.test +{ + public readonly unsafe struct TestSpanByteKey : IKey + { + private readonly byte[] arr; + private readonly void* ptr; + private readonly int len; + + public readonly bool IsPinned => arr == null; + + public readonly ReadOnlySpan KeyBytes => arr == null ? new(ptr, len) : arr.AsSpan(); + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + + private TestSpanByteKey(byte[] arr, void* ptr, int len) + { + this.arr = arr; + this.ptr = ptr; + this.len = len; + } + + public static TestSpanByteKey FromPinnedSpan(ReadOnlySpan key) + { + var ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(key)); + var len = key.Length; + return new(null, ptr, len); + } + + public static TestSpanByteKey CopySpan(ReadOnlySpan key) + { + var arr = key.ToArray(); + return new(arr, null, arr.Length); + } + + public static TestSpanByteKey FromPointer(byte* ptr, int len) => new(null, ptr, len); + + public static TestSpanByteKey FromArray(byte[] array) => new(array, null, array.Length); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/TestTypes.cs b/libs/storage/Tsavorite/cs/test/TestTypes.cs index 4d63a059f34..c915c65ea19 100644 --- a/libs/storage/Tsavorite/cs/test/TestTypes.cs +++ b/libs/storage/Tsavorite/cs/test/TestTypes.cs @@ -2,40 +2,83 @@ // Licensed under the MIT license. using System; +using System.Runtime.InteropServices; using System.Threading; +using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; +#pragma warning disable CA2211 // Non-constant fields should not be visible (This is for the .Instance members) + namespace Tsavorite.test { - public struct KeyStruct + [StructLayout(LayoutKind.Explicit)] + public struct KeyStruct : IKey { + [FieldOffset(0)] public long kfield1; + [FieldOffset(8)] public long kfield2; - public override string ToString() => $"kfield1 {kfield1}, kfield2 {kfield2}"; + // Not always pinned, so don't assume it is + public readonly bool IsPinned => false; + + public ReadOnlySpan KeyBytes => MemoryMarshal.Cast(MemoryMarshal.CreateReadOnlySpan(ref kfield1, 2)); + + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + + public override readonly string ToString() => $"kfield1 {kfield1}, kfield2 {kfield2}"; - public struct Comparer : IKeyComparer + public struct Comparer : IKeyComparer { - public long GetHashCode64(ref KeyStruct key) => Utility.GetHashCode(key.kfield1); - public bool Equals(ref KeyStruct k1, ref KeyStruct k2) => k1.kfield1 == k2.kfield1 && k1.kfield2 == k2.kfield2; + public readonly long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(key.KeyBytes.AsRef().kfield1); + + public readonly bool Equals(TFirstKey key1, TSecondKey key2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + var k1 = key1.KeyBytes.AsRef(); + var k2 = key2.KeyBytes.AsRef(); + return k1.kfield1 == k2.kfield1 && k1.kfield2 == k2.kfield2; + } public static Comparer Instance = new(); } } - public struct ValueStruct + [StructLayout(LayoutKind.Explicit)] + public unsafe struct ValueStruct { + [FieldOffset(0)] public long vfield1; + [FieldOffset(8)] public long vfield2; - public override string ToString() => $"vfield1 {vfield1}, vfield2 {vfield2}"; + + public static int AsSpanByteDataSize => sizeof(ValueStruct); + + public override readonly string ToString() => $"vfield1 {vfield1}, vfield2 {vfield2}"; } public struct InputStruct { public long ifield1; public long ifield2; - public override string ToString() => $"ifield1 {ifield1}, ifield2 {ifield2}"; + public override readonly string ToString() => $"ifield1 {ifield1}, ifield2 {ifield2}"; } public struct OutputStruct @@ -47,93 +90,98 @@ public struct ContextStruct { public long cfield1; public long cfield2; - public override string ToString() => $"cfield1 {cfield1}, cfield2 {cfield2}"; + public override readonly string ToString() => $"cfield1 {cfield1}, cfield2 {cfield2}"; } public class Functions : FunctionsWithContext { } - public class FunctionsWithContext : SessionFunctionsBase + public class FunctionsWithContext : SessionFunctionsBase { - public override void RMWCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, TContext ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, TContext ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); ClassicAssert.IsTrue(status.Record.CopyUpdated); - ClassicAssert.AreEqual(key.kfield1 + input.ifield1, output.value.vfield1); - ClassicAssert.AreEqual(key.kfield2 + input.ifield2, output.value.vfield2); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().kfield1 + input.ifield1, output.value.vfield1); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().kfield2 + input.ifield2, output.value.vfield2); } - public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, TContext ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, TContext ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key.kfield1, output.value.vfield1); - ClassicAssert.AreEqual(key.kfield2, output.value.vfield2); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().kfield1, output.value.vfield1); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().kfield2, output.value.vfield2); } // Read functions - public override bool SingleReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref ReadInfo readInfo) { - ClassicAssert.IsFalse(readInfo.RecordInfo.IsNull()); - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - ClassicAssert.IsFalse(readInfo.RecordInfo.IsNull()); - dst.value = value; + output.value = srcLogRecord.ValueSpan.AsRef(); return true; } // RMW functions - public override bool InitialUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + ref var value = ref logRecord.ValueSpan.AsRef(); value.vfield1 = input.ifield1; value.vfield2 = input.ifield2; output.value = value; return true; } - public override bool InPlaceUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + ref var value = ref logRecord.ValueSpan.AsRef(); value.vfield1 += input.ifield1; value.vfield2 += input.ifield2; output.value = value; return true; } - public override bool NeedCopyUpdate(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref OutputStruct output, ref RMWInfo rmwInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + ClassicAssert.IsTrue(srcLogRecord.IsSet); return true; } - public override bool CopyUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref ValueStruct newValue, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + var oldValue = srcLogRecord.ValueSpan.AsRef(); + ref var newValue = ref dstLogRecord.ValueSpan.AsRef(); + newValue.vfield1 = oldValue.vfield1 + input.ifield1; newValue.vfield2 = oldValue.vfield2 + input.ifield2; output.value = newValue; return true; } + + /// + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref InputStruct input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length }; } - public class FunctionsCompaction : SessionFunctionsBase + public class FunctionsCompaction : SessionFunctionsBase { - public override void RMWCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, int ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, int ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); ClassicAssert.IsTrue(status.Record.CopyUpdated); } - public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, int ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, int ctx, Status status, RecordMetadata recordMetadata) { if (ctx == 0) { ClassicAssert.IsTrue(status.Found); + var key = diskLogRecord.Key.AsRef(); ClassicAssert.AreEqual(key.kfield1, output.value.vfield1); ClassicAssert.AreEqual(key.kfield2, output.value.vfield2); } @@ -144,150 +192,225 @@ public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct i } // Read functions - public override bool SingleReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref ReadInfo readInfo) { - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - dst.value = value; + output.value = srcLogRecord.ValueSpan.AsRef(); return true; } // RMW functions - public override bool InitialUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { + ref var value = ref logRecord.ValueSpan.AsRef(); value.vfield1 = input.ifield1; value.vfield2 = input.ifield2; return true; } - public override bool InPlaceUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { + ref var value = ref logRecord.ValueSpan.AsRef(); value.vfield1 += input.ifield1; value.vfield2 += input.ifield2; return true; } - public override bool NeedCopyUpdate(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref OutputStruct output, ref RMWInfo rmwInfo) => true; + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) => true; - public override bool CopyUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref ValueStruct newValue, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { + var oldValue = srcLogRecord.ValueSpan.AsRef(); + ref var newValue = ref dstLogRecord.ValueSpan.AsRef(); + newValue.vfield1 = oldValue.vfield1 + input.ifield1; newValue.vfield2 = oldValue.vfield2 + input.ifield2; return true; } + + /// + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref InputStruct input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length }; } - public class FunctionsCopyOnWrite : SessionFunctionsBase + public class FunctionsCopyOnWrite : SessionFunctionsBase { - private int _concurrentWriterCallCount; - private int _inPlaceUpdaterCallCount; + private int inPlaceWriterCallCount; + private int inPlaceUpdaterCallCount; - public int ConcurrentWriterCallCount => _concurrentWriterCallCount; - public int InPlaceUpdaterCallCount => _inPlaceUpdaterCallCount; + public int InPlaceWriterCallCount => inPlaceWriterCallCount; + public int InPlaceUpdaterCallCount => inPlaceUpdaterCallCount; - public override void RMWCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); ClassicAssert.IsTrue(status.Record.CopyUpdated); } - public override void ReadCompletionCallback(ref KeyStruct key, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref InputStruct input, ref OutputStruct output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); + var key = diskLogRecord.Key.AsRef(); ClassicAssert.AreEqual(key.kfield1, output.value.vfield1); ClassicAssert.AreEqual(key.kfield2, output.value.vfield2); } // Read functions - public override bool SingleReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord logRecord, ref InputStruct input, ref OutputStruct output, ref ReadInfo readInfo) { - ClassicAssert.IsFalse(readInfo.RecordInfo.IsNull()); - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - { - ClassicAssert.IsFalse(readInfo.RecordInfo.IsNull()); - dst.value = value; + output.value = logRecord.ValueSpan.AsRef(); return true; } // Upsert functions - public override bool SingleWriter(ref KeyStruct key, ref InputStruct input, ref ValueStruct src, ref ValueStruct dst, ref OutputStruct output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ReadOnlySpan srcValue, ref OutputStruct output, ref UpsertInfo upsertInfo) { - ClassicAssert.IsFalse(upsertInfo.RecordInfo.IsNull()); - dst = src; + logRecord.ValueSpan.AsRef() = srcValue.AsRef(); return true; } - public override bool ConcurrentWriter(ref KeyStruct key, ref InputStruct input, ref ValueStruct src, ref ValueStruct dst, ref OutputStruct output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref InputStruct input, ReadOnlySpan srcValue, ref OutputStruct output, ref UpsertInfo upsertInfo) { - ClassicAssert.IsFalse(upsertInfo.RecordInfo.IsNull()); - Interlocked.Increment(ref _concurrentWriterCallCount); + _ = Interlocked.Increment(ref inPlaceWriterCallCount); return false; } // RMW functions - public override bool InitialUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + ref var value = ref logRecord.ValueSpan.AsRef(); value.vfield1 = input.ifield1; value.vfield2 = input.ifield2; return true; } - public override bool InPlaceUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct value, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); - Interlocked.Increment(ref _inPlaceUpdaterCallCount); + _ = Interlocked.Increment(ref inPlaceUpdaterCallCount); return false; } - public override bool NeedCopyUpdate(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref OutputStruct output, ref RMWInfo rmwInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + ClassicAssert.IsTrue(srcLogRecord.IsSet); return true; } - public override bool CopyUpdater(ref KeyStruct key, ref InputStruct input, ref ValueStruct oldValue, ref ValueStruct newValue, ref OutputStruct output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref InputStruct input, ref OutputStruct output, ref RMWInfo rmwInfo) { - ClassicAssert.IsFalse(rmwInfo.RecordInfo.IsNull()); + var oldValue = srcLogRecord.ValueSpan.AsRef(); + ref var newValue = ref dstLogRecord.ValueSpan.AsRef(); + newValue.vfield1 = oldValue.vfield1 + input.ifield1; newValue.vfield2 = oldValue.vfield2 + input.ifield2; return true; } + + /// + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref InputStruct input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(ValueStruct) }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref InputStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length }; + } + + public class SimpleLongSimpleFunctions : SimpleIntegerFunctionsBase + { + public SimpleLongSimpleFunctions() : base() { } + public SimpleLongSimpleFunctions(Func merger) : base(merger) { } + } + + public class SimpleIntSimpleFunctions : SimpleIntegerFunctionsBase + { + public SimpleIntSimpleFunctions() : base() { } + public SimpleIntSimpleFunctions(Func merger) : base(merger) { } } - class RMWSimpleFunctions : SimpleSimpleFunctions + public class SimpleIntegerFunctionsBase : SessionFunctionsBase + where TInteger : unmanaged { - public RMWSimpleFunctions(Func merger) : base(merger) { } + private readonly Func merger; - public override bool InitialUpdater(ref TKey key, ref TValue input, ref TValue value, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public SimpleIntegerFunctionsBase() : base() => merger = (input, oldValue) => input; + + public SimpleIntegerFunctionsBase(Func merger) => this.merger = merger; + + /// + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TInteger input, ref TInteger output, ref ReadInfo readInfo) { - base.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); - output = input; + output = srcLogRecord.ValueSpan.AsRef(); return true; } + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInteger input, ReadOnlySpan srcValue, ref TInteger output, ref UpsertInfo upsertInfo) + { + var result = base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + if (result) + output = srcValue.AsRef(); + return result; + } + + public override bool InPlaceWriter(ref LogRecord logRecord, ref TInteger input, ReadOnlySpan srcValue, ref TInteger output, ref UpsertInfo upsertInfo) + { + var result = base.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo); + if (result) + output = srcValue.AsRef(); + return result; + } + /// - public override bool CopyUpdater(ref TKey key, ref TValue input, ref TValue oldValue, ref TValue newValue, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInteger input, ref TInteger output, ref RMWInfo rmwInfo) { - base.CopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo, ref recordInfo); - output = newValue; - return true; + var ok = dstLogRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref input), in sizeInfo); + if (ok) + output = input; + return ok; } /// - public override bool InPlaceUpdater(ref TKey key, ref TValue input, ref TValue value, ref TValue output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TInteger input, ref TInteger output, ref RMWInfo rmwInfo) { - base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); - output = value; - return true; + ClassicAssert.IsTrue(dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo), "Failed TryCopyRecordValues"); + var result = output = merger(input, srcLogRecord.ValueSpan.AsRef()); // 'result' must be local for SpanByte.From; 'output' may be on the heap + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref result), in sizeInfo); + } + + /// + public override bool InPlaceUpdater(ref LogRecord logRecord, ref TInteger input, ref TInteger output, ref RMWInfo rmwInfo) + { + var result = output = merger(input, logRecord.ValueSpan.AsRef()); // 'result' must be local for SpanByte.From; 'output' may be on the heap + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref result), in sizeInfo); + } + + /// + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TInteger input) + { + Assert.That(srcLogRecord.ValueSpan.Length, Is.EqualTo(sizeof(TInteger))); + return new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(TInteger) }; + } + + /// + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TInteger input) + { + Assert.That(key.KeyBytes.Length, Is.EqualTo(sizeof(TInteger))); + return new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(TInteger) }; + } + + /// + public override unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref TInteger input) + { + Assert.That(value.Length, Is.EqualTo(sizeof(TInteger))); + return new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(TInteger) }; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/TestUtils.cs b/libs/storage/Tsavorite/cs/test/TestUtils.cs index 82407e411cc..ab303a355c9 100644 --- a/libs/storage/Tsavorite/cs/test/TestUtils.cs +++ b/libs/storage/Tsavorite/cs/test/TestUtils.cs @@ -2,7 +2,10 @@ // Licensed under the MIT license. using System; +using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -21,7 +24,7 @@ public static class TestUtils internal const string StressTestCategory = "Stress"; internal const string TsavoriteKVTestCategory = "TsavoriteKV"; internal const string ReadTestCategory = "Read"; - internal const string LockableUnsafeContextTestCategory = "LockableUnsafeContext"; + internal const string TransactionalUnsafeContextTestCategory = "TransactionalUnsafeContext"; internal const string ReadCacheTestCategory = "ReadCache"; internal const string LockTestCategory = "Locking"; internal const string LockTableTestCategory = "LockTable"; @@ -31,6 +34,10 @@ public static class TestUtils internal const string IteratorCategory = "Iterator"; internal const string ModifiedBitTestCategory = "ModifiedBitTest"; internal const string RevivificationCategory = "Revivification"; + internal const string MultiLevelPageArrayCategory = "MultiLevelPageArray"; + internal const string ObjectIdMapCategory = "ObjectIdMap"; + internal const string OverflowFieldCategory = "OverflowField"; + internal const string LogRecordCategory = "LogRecord"; public static ILoggerFactory TestLoggerFactory = CreateLoggerFactoryInstance(TestContext.Progress, LogLevel.Trace); @@ -56,7 +63,7 @@ internal static void DeleteDirectory(string path, bool wait = false) } } - for (; ; Thread.Yield()) + while (true) { // Exceptions may happen due to a handle briefly remaining held after Dispose(). try @@ -69,6 +76,7 @@ internal static void DeleteDirectory(string path, bool wait = false) } if (!wait || !Directory.Exists(path)) break; + _ = Thread.Yield(); } } @@ -202,12 +210,11 @@ internal static string AzureTestContainer public enum AllocatorType { - FixedBlittable, SpanByte, - Generic + Object } - internal enum CompletionSyncMode { Sync, Async } + public enum CompletionSyncMode { Sync, Async } public enum ReadCopyDestination { Tail, ReadCache } @@ -215,9 +222,9 @@ public enum FlushMode { NoFlush, ReadOnly, OnDisk } public enum KeyEquality { Equal, NotEqual } - public enum ReadCacheMode { UseReadCache, NoReadCache } + public enum ReadCacheMode { UseRC, NoRC } - public enum KeyContentionMode { Contention, NoContention } + public enum KeyContentionMode { Cont, NoCont } public enum BatchMode { Batch, NoBatch } @@ -231,10 +238,18 @@ public enum ScanMode { Scan, Iterate } public enum WaitMode { Wait, NoWait } - internal static (Status status, TOutput output) GetSinglePendingResult(CompletedOutputIterator completedOutputs) + public enum RandomMode { Rng, NoRng } + + /// + /// Extract the status and output from the completed results, and Dispose() the completed results. + /// + internal static (Status status, TOutput output) GetSinglePendingResult(CompletedOutputIterator completedOutputs) => GetSinglePendingResult(completedOutputs, out _); - internal static (Status status, TOutput output) GetSinglePendingResult(CompletedOutputIterator completedOutputs, out RecordMetadata recordMetadata) + /// + /// Extract the status and output from the completed results, and Dispose() the completed results. + /// + internal static (Status status, TOutput output) GetSinglePendingResult(CompletedOutputIterator completedOutputs, out RecordMetadata recordMetadata) { ClassicAssert.IsTrue(completedOutputs.Next()); var result = (completedOutputs.Current.Status, completedOutputs.Current.Output); @@ -261,11 +276,15 @@ internal static async ValueTask DoTwoThreadRandomKeyTest(int count, bool doRando } } - internal static unsafe bool FindHashBucketEntryForKey(this TsavoriteKV store, ref TKey key, out HashBucketEntry entry) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static unsafe bool FindHashBucketEntryForKey(this TsavoriteKV store, TKey key, out HashBucketEntry entry) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(ref key)); + HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(key)); var success = store.FindTag(ref hei); entry = hei.entry; return success; @@ -285,50 +304,198 @@ internal static void OnTearDown(bool waitForDelete = false, ILogger logger = nul } } - internal class LongComparerModulo : IKeyComparer + /// Deterministic equality comparer for ints + public sealed class IntKeyComparer : IKeyComparer { - readonly long mod; - - internal LongComparerModulo(long mod) => this.mod = mod; + /// + /// The default instance. + /// + /// Used to avoid allocating new comparers. + public static readonly IntKeyComparer Instance = new(); + + /// + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef() == k2.KeyBytes.AsRef(); + + /// + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(k.KeyBytes.AsRef()); + } - public bool Equals(ref long k1, ref long k2) => k1 == k2; + /// Deterministic equality comparer for longs + public sealed class LongKeyComparer : IKeyComparer + { + /// + /// The default instance. + /// + /// Used to avoid allocating new comparers. + public static readonly LongKeyComparer Instance = new(); + + /// + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef() == k2.KeyBytes.AsRef(); + + /// + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => Utility.GetHashCode(k.KeyBytes.AsRef()); + } - public long GetHashCode64(ref long k) => mod == 0 ? k : k % mod; + /// Deterministic equality comparer for longs with hash modulo + internal class LongKeyComparerModulo : IKeyComparer + { + internal long mod; + + internal LongKeyComparerModulo(long mod) => this.mod = mod; + + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsSliceRef() == k2.KeyBytes.AsSliceRef(); + + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => mod == 0 ? k.KeyBytes.AsSliceRef() : k.KeyBytes.AsSliceRef() % mod; } - internal struct SpanByteComparerModulo : IKeyComparer + /// Deterministic equality comparer for SpanBytes with hash modulo + internal struct SpanByteKeyComparerModulo : IKeyComparer { readonly HashModulo modRange; - internal SpanByteComparerModulo(HashModulo mod) => modRange = mod; + internal SpanByteKeyComparerModulo(HashModulo mod) => modRange = mod; - public readonly bool Equals(ref SpanByte k1, ref SpanByte k2) => SpanByteComparer.StaticEquals(ref k1, ref k2); + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); // Force collisions to create a chain - public readonly long GetHashCode64(ref SpanByte k) + public readonly long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - var value = SpanByteComparer.StaticGetHashCode64(ref k); + var value = SpanByteComparer.StaticGetHashCode64(k.KeyBytes); return modRange != HashModulo.NoMod ? value % (long)modRange : value; } } static class StaticTestUtils { - internal static (Status status, TOutput output) GetSinglePendingResult( - this ITsavoriteContext sessionContext) - where Functions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static (Status status, TOutput output) GetSinglePendingResult( + this ITsavoriteContext sessionContext) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where Functions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => sessionContext.GetSinglePendingResult(out _); - internal static (Status status, TOutput output) GetSinglePendingResult( - this ITsavoriteContext sessionContext, out RecordMetadata recordMetadata) - where Functions : ISessionFunctions - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static (Status status, TOutput output) GetSinglePendingResult( + this ITsavoriteContext sessionContext, out RecordMetadata recordMetadata) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where Functions : ISessionFunctions + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - sessionContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + _ = sessionContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); return TestUtils.GetSinglePendingResult(completedOutputs, out recordMetadata); } + + /// For use with stack-based single T variable. + public static ref T AsRef(this Span spanByte) where T : unmanaged + { + Debug.Assert(spanByte.Length >= Unsafe.SizeOf(), $"Span length expected to be >= {Unsafe.SizeOf()} but was {spanByte.Length}"); + return ref Unsafe.As(ref spanByte[0]); + } + + /// For use with stack-based byte vector indexed as a vector of T; usually just the 0th item + public static ref readonly T AsRef(this ReadOnlySpan spanByte) where T : unmanaged + { + Debug.Assert(spanByte.Length >= Unsafe.SizeOf(), $"ReadOnlySpan length expected to be >= {Unsafe.SizeOf()} but was {spanByte.Length}"); + return ref MemoryMarshal.Cast(spanByte)[0]; + } + + /// For use with stack-based byte vector indexed as a vector of T; usually just the 0th item + public static ref readonly T AsRef(this TestSpanByteKey spanByte) where T : unmanaged + { + Debug.Assert(spanByte.KeyBytes.Length >= Unsafe.SizeOf(), $"ReadOnlySpan length expected to be >= {Unsafe.SizeOf()} but was {spanByte.KeyBytes.Length}"); + return ref MemoryMarshal.Cast(spanByte.KeyBytes)[0]; + } + + /// For use with stack-based single T variable. + public static ref T AsSliceRef(this Span spanByte, int sliceIndex = 0) where T : unmanaged + => ref Unsafe.As(ref spanByte[sliceIndex]); + + /// For use with stack-based byte vector indexed as a vector of T; usually just the 0th item + public static ref readonly T AsSliceRef(this ReadOnlySpan spanByte, int sliceIndex = 0) where T : unmanaged + => ref MemoryMarshal.Cast(spanByte)[sliceIndex]; + + /// For use with stack-based single T variable. + internal static Span Set(this Span spanByte, T value) where T : unmanaged + { + spanByte.AsRef() = value; + return spanByte; + } + + + /// For use with stack-based single T variable. + internal static TestSpanByteKey Set(this TestSpanByteKey spanByte, T value) where T : unmanaged + { + MemoryMarshal.CreateSpan(ref MemoryMarshal.GetReference(spanByte.KeyBytes), spanByte.KeyBytes.Length).AsRef() = value; + return spanByte; + } + + /// For use with stack-based byte vector indexed as a vector of T; usually just the 0th item + internal static Span SetSlice(this Span spanByte, T value, int sliceIndex = 0) where T : unmanaged + { + spanByte.AsSliceRef(sliceIndex) = value; + return spanByte; + } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/TryEnqueueBasicTests.cs b/libs/storage/Tsavorite/cs/test/TryEnqueueBasicTests.cs index 1657545a27b..a39dd037ba3 100644 --- a/libs/storage/Tsavorite/cs/test/TryEnqueueBasicTests.cs +++ b/libs/storage/Tsavorite/cs/test/TryEnqueueBasicTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,10 +12,8 @@ namespace Tsavorite.test { //** Fundamental basic test for TryEnqueue that covers all the parameters in TryEnqueue //** Other tests in TsavoriteLog.cs provide more coverage for TryEnqueue - - [AllureNUnit] [TestFixture] - internal class TryEnqueueTests : AllureTestBase + internal class TryEnqueueTests : TestBase { private TsavoriteLog log; private IDevice device; @@ -142,7 +139,7 @@ public void TryEnqueueBasicTest([Values] TryEnqueueIteratorType iteratorType, [V // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { while (iter.GetNext(out byte[] result, out _, out _)) { diff --git a/libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj b/libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj index 0e3acfff316..ab68ae674bf 100644 --- a/libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj +++ b/libs/storage/Tsavorite/cs/test/Tsavorite.test.csproj @@ -1,4 +1,4 @@ - + true @@ -11,12 +11,11 @@ - + + - - @@ -25,13 +24,21 @@ + + + + + + + + + - false diff --git a/libs/storage/Tsavorite/cs/test/VLVector.cs b/libs/storage/Tsavorite/cs/test/VLVector.cs index a6bcc134f17..ceaa02b9fad 100644 --- a/libs/storage/Tsavorite/cs/test/VLVector.cs +++ b/libs/storage/Tsavorite/cs/test/VLVector.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. using System; -using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using NUnit.Framework.Legacy; using Tsavorite.core; @@ -11,37 +11,29 @@ namespace Tsavorite.test // Extension class for SpanByte to wrap a non-byte Span internal static unsafe class VLVector { - // Wrap a SpanByte around a Span of (usually) non-byte type, e.g.: + // Wrap a SpanByte around a Span of non-byte type, e.g.: // Span valueSpan = stackalloc int[numElem]; // for (var ii = 0; ii < numElem; ++ii) valueSpan[ii] = someInt; // var valueSpanByte = valueSpan.AsSpanByte(); - public static SpanByte AsSpanByte(this Span span) where T : unmanaged - => new SpanByte(span.Length * sizeof(T), (IntPtr)Unsafe.AsPointer(ref span[0])); + public static PinnedSpanByte FromPinnedSpan(Span span) where T : unmanaged + => PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(span)); - public static SpanByte AsSpanByte(this ReadOnlySpan span) where T : unmanaged - { - fixed (T* ptr = span) - { - return new SpanByte(span.Length * sizeof(T), (IntPtr)ptr); - } - } - - public static Span AsSpan(this ref SpanByte sb) where T : unmanaged - => new Span(sb.MetadataSize + sb.ToPointer(), (sb.Length - sb.MetadataSize) / sizeof(T)); + internal static T[] ToArray(this Span byteSpan) where T : unmanaged + => MemoryMarshal.Cast(byteSpan).ToArray(); - internal static T[] ToArray(this ref SpanByte spanByte) where T : unmanaged - => AsSpan(ref spanByte).ToArray(); + internal static T[] ToArray(this ReadOnlySpan byteSpan) where T : unmanaged + => MemoryMarshal.Cast(byteSpan).ToArray(); } - public class VLVectorFunctions : SpanByteFunctions + public class VLVectorFunctions : SessionFunctionsBase { - public override void RMWCompletionCallback(ref SpanByte key, ref SpanByte input, ref int[] output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref PinnedSpanByte input, ref int[] output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); ClassicAssert.IsTrue(status.Record.CopyUpdated); } - public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input, ref int[] output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref PinnedSpanByte input, ref int[] output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); for (int i = 0; i < output.Length; i++) @@ -49,27 +41,12 @@ public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input } // Read functions - public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref int[] dst, ref ReadInfo readInfo) - { - dst = value.ToArray(); - return true; - } - - public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref int[] dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref int[] output, ref ReadInfo readInfo) { - dst = value.ToArray(); + output = srcLogRecord.ValueSpan.ToArray(); return true; } - // Upsert functions - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref int[] output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - => base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); - - public override bool ConcurrentWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref int[] output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - if (src.Length != dst.Length) - return false; - return base.ConcurrentWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, ref recordInfo); - } + // Upsert functions are unchanged from SessionFunctionsBase } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/WaitForCommit.cs b/libs/storage/Tsavorite/cs/test/WaitForCommit.cs index e0809045650..8d2fe7622c8 100644 --- a/libs/storage/Tsavorite/cs/test/WaitForCommit.cs +++ b/libs/storage/Tsavorite/cs/test/WaitForCommit.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,9 +10,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class WaitForCommitTests : AllureTestBase + internal class WaitForCommitTests : TestBase { static TsavoriteLog log; public IDevice device; @@ -60,26 +58,20 @@ public void WaitForCommitBasicTest(string SyncTest) // Set Default entry data for (int i = 0; i < entryLength; i++) - { entry[i] = (byte)i; - } // Enqueue / WaitForCommit on a task (that will be waited) until the Commit on the separate thread is done if (SyncTest == "Sync") - { new Thread(new ThreadStart(LogWriter)).Start(); - } else - { new Thread(new ThreadStart(LogWriterAsync)).Start(); - } ev.WaitOne(); log.Commit(true); // Read the log to make sure all entries are put in int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { while (iter.GetNext(out byte[] result, out _, out _)) { diff --git a/libs/storage/Tsavorite/cs/test/BasicStorageTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/BasicStorageTests.cs similarity index 63% rename from libs/storage/Tsavorite/cs/test/BasicStorageTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/BasicStorageTests.cs index ecf52102756..0542d00c077 100644 --- a/libs/storage/Tsavorite/cs/test/BasicStorageTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/BasicStorageTests.cs @@ -1,28 +1,26 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; using Tsavorite.devices; +using static Tsavorite.test.TestUtils; namespace Tsavorite.test { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class BasicStorageTests : AllureTestBase + internal class BasicStorageTests : TestBase { [Test] [Category("TsavoriteKV")] public void LocalStorageWriteRead() { - TestDeviceWriteRead(Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "BasicDiskTests.log"), deleteOnClose: true)); + TestDeviceWriteRead(Devices.CreateLogDevice(Path.Join(MethodTestDir, "BasicDiskTests.log"), deleteOnClose: true)); } [Test] @@ -30,8 +28,8 @@ public void LocalStorageWriteRead() [Category("Smoke")] public void PageBlobWriteRead() { - TestUtils.IgnoreIfNotRunningAzureTests(); - TestDeviceWriteRead(new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskTests", logger: TestUtils.TestLoggerFactory.CreateLogger("asd"))); + IgnoreIfNotRunningAzureTests(); + TestDeviceWriteRead(new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, "BasicDiskTests", logger: TestLoggerFactory.CreateLogger("asd"))); } [Test] @@ -39,8 +37,8 @@ public void PageBlobWriteRead() [Category("Smoke")] public void PageBlobWriteReadWithLease() { - TestUtils.IgnoreIfNotRunningAzureTests(); - TestDeviceWriteRead(new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskTests", null, true, true, logger: TestUtils.TestLoggerFactory.CreateLogger("asd"))); + IgnoreIfNotRunningAzureTests(); + TestDeviceWriteRead(new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, "BasicDiskTests", null, true, true, logger: TestLoggerFactory.CreateLogger("asd"))); } [Test] @@ -48,18 +46,18 @@ public void PageBlobWriteReadWithLease() [Category("Smoke")] public void TieredWriteRead() { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + DeleteDirectory(MethodTestDir); IDevice tested; - IDevice localDevice = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "BasicDiskTests.log"), deleteOnClose: true, capacity: 1L << 30); - if (TestUtils.IsRunningAzureTests) + IDevice localDevice = Devices.CreateLogDevice(Path.Join(MethodTestDir, "BasicDiskTests.log"), deleteOnClose: true, capacity: 1L << 30); + if (IsRunningAzureTests) { - IDevice cloudDevice = new AzureStorageDevice(TestUtils.AzureEmulatedStorageString, TestUtils.AzureTestContainer, TestUtils.AzureTestDirectory, "BasicDiskTests", logger: TestUtils.TestLoggerFactory.CreateLogger("asd")); + IDevice cloudDevice = new AzureStorageDevice(AzureEmulatedStorageString, AzureTestContainer, AzureTestDirectory, "BasicDiskTests", logger: TestLoggerFactory.CreateLogger("asd")); tested = new TieredStorageDevice(1, localDevice, cloudDevice); } else { // If no Azure is enabled, just use another disk - IDevice localDevice2 = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "BasicDiskTests2.log"), deleteOnClose: true, capacity: 1L << 30); + IDevice localDevice2 = Devices.CreateLogDevice(Path.Join(MethodTestDir, "BasicDiskTests2.log"), deleteOnClose: true, capacity: 1L << 30); tested = new TieredStorageDevice(1, localDevice, localDevice2); } @@ -71,8 +69,8 @@ public void TieredWriteRead() [Category("Smoke")] public void ShardedWriteRead() { - IDevice localDevice1 = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "BasicDiskTests1.log"), deleteOnClose: true, capacity: 1L << 30); - IDevice localDevice2 = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "BasicDiskTests2.log"), deleteOnClose: true, capacity: 1L << 30); + IDevice localDevice1 = Devices.CreateLogDevice(Path.Join(MethodTestDir, "BasicDiskTests1.log"), deleteOnClose: true, capacity: 1L << 30); + IDevice localDevice2 = Devices.CreateLogDevice(Path.Join(MethodTestDir, "BasicDiskTests2.log"), deleteOnClose: true, capacity: 1L << 30); var device = new ShardedStorageDevice(new UniformPartitionScheme(512, localDevice1, localDevice2)); TestDeviceWriteRead(device); } @@ -82,11 +80,11 @@ public void ShardedWriteRead() [Category("Smoke")] public void OmitSegmentIdTest([Values] TestUtils.TestDeviceType deviceType) { - var filename = Path.Join(TestUtils.MethodTestDir, "test.log"); + var filename = Path.Join(MethodTestDir, "test.log"); var omit = false; for (var ii = 0; ii < 2; ++ii) { - using IDevice device = TestUtils.CreateTestDevice(deviceType, filename, omitSegmentIdFromFilename: omit); + using IDevice device = CreateTestDevice(deviceType, filename, omitSegmentIdFromFilename: omit); var storageBase = (StorageDeviceBase)device; var segmentFilename = storageBase.GetSegmentFilename(filename, 0); if (omit) @@ -99,18 +97,18 @@ public void OmitSegmentIdTest([Values] TestUtils.TestDeviceType deviceType) static void TestDeviceWriteRead(IDevice log) { - var store = new TsavoriteKV( + var store = new TsavoriteKV( new() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, - }, StoreFunctions.Create(KeyStruct.Comparer.Instance) + }, StoreFunctions.Create(KeyStruct.Comparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - var session = store.NewSession(new Functions()); + var session = store.NewSession(new Functions()); var bContext = session.BasicContext; InputStruct input = default; @@ -119,7 +117,7 @@ static void TestDeviceWriteRead(IDevice log) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } _ = bContext.CompletePending(true); @@ -128,7 +126,7 @@ static void TestDeviceWriteRead(IDevice log) { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = 1, ifield2 = 1 }; - var status = bContext.RMW(ref key1, ref input, Empty.Default); + var status = bContext.RMW(key1, ref input, Empty.Default); if (status.IsPending) _ = bContext.CompletePending(true); } @@ -140,7 +138,7 @@ static void TestDeviceWriteRead(IDevice log) var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - if (bContext.Read(ref key1, ref input, ref output, Empty.Default).IsPending) + if (bContext.Read(key1, ref input, ref output, Empty.Default).IsPending) { _ = bContext.CompletePending(true); } @@ -163,7 +161,7 @@ static void TestDeviceWriteRead(IDevice log) store.Dispose(); store = null; log.Dispose(); - TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + DeleteDirectory(MethodTestDir); } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/DeviceLogTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceLogTests.cs similarity index 96% rename from libs/storage/Tsavorite/cs/test/DeviceLogTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/DeviceLogTests.cs index 3ee600662c6..d6910cd14e5 100644 --- a/libs/storage/Tsavorite/cs/test/DeviceLogTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceLogTests.cs @@ -5,18 +5,18 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; using Tsavorite.devices; +#pragma warning disable IDE1006 // Naming Styles + namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class DeviceLogTests : AllureTestBase + internal class DeviceLogTests : TestBase { const int entryLength = 100; const int numEntries = 1000; @@ -70,7 +70,7 @@ public void BasicHighLatencyDeviceTest() for (int i = 0; i < entryLength; i++) { entry[i] = (byte)i; - LocalMemorylog.Enqueue(entry); + _ = LocalMemorylog.Enqueue(entry); } // Commit to the log @@ -97,7 +97,7 @@ private async ValueTask TsavoriteLogTest1(LogChecksumType logChecksum, IDevice d for (int i = 0; i < numEntries; i++) { - log.Enqueue(entry); + _ = log.Enqueue(entry); } log.CompleteLog(true); @@ -119,7 +119,7 @@ private async ValueTask TsavoriteLogTest1(LogChecksumType logChecksum, IDevice d } break; case TsavoriteLogTestBase.IteratorType.AsyncMemoryOwner: - await foreach ((IMemoryOwner result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool.Shared).ConfigureAwait(false)) + await foreach ((IMemoryOwner result, _, _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool.Shared)) { ClassicAssert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); diff --git a/libs/storage/Tsavorite/cs/test/DeviceTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs similarity index 97% rename from libs/storage/Tsavorite/cs/test/DeviceTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs index 0c556fc855b..97408cb78cb 100644 --- a/libs/storage/Tsavorite/cs/test/DeviceTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/DeviceTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,7 +6,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - public class DeviceTests : AllureTestBase + public class DeviceTests : TestBase { const int entryLength = 1024; SectorAlignedBufferPool bufferPool; diff --git a/libs/storage/Tsavorite/cs/test/FlakyDeviceTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/FlakyDeviceTests.cs similarity index 95% rename from libs/storage/Tsavorite/cs/test/FlakyDeviceTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/FlakyDeviceTests.cs index d238e2f1135..bf408ca5295 100644 --- a/libs/storage/Tsavorite/cs/test/FlakyDeviceTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/FlakyDeviceTests.cs @@ -8,14 +8,12 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; namespace Tsavorite.test { - [AllureNUnit] [TestFixture] internal class FlakyDeviceTests : TsavoriteLogTestBase { @@ -27,6 +25,7 @@ internal class FlakyDeviceTests : TsavoriteLogTestBase [Test] [Category("TsavoriteLog")] + //[Repeat(3000)] public async ValueTask FlakyLogTestCleanFailure([Values] bool isAsync) { var errorOptions = new ErrorSimulationOptions @@ -52,9 +51,7 @@ public async ValueTask FlakyLogTestCleanFailure([Values] bool isAsync) for (int j = 0; j < 100; j++) { for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); if (isAsync) await log.CommitAsync().ConfigureAwait(false); @@ -106,7 +103,7 @@ public void FlakyLogTestConcurrentWriteFailure() { for (int i = 0; i < numEntries; i++) { - log.Enqueue(entry); + _ = log.Enqueue(entry); // create randomly interleaved concurrent writes if (random.NextDouble() < 0.1) log.Commit(); @@ -164,7 +161,7 @@ public async ValueTask FlakyLogTestTolerateFailure([Values] IteratorType iterato // Ensure we write enough to trigger errors for (int i = 0; i < 1000; i++) { - log.Enqueue(entry); + _ = log.Enqueue(entry); try { if (IsAsync(iteratorType)) @@ -186,11 +183,11 @@ public async ValueTask FlakyLogTestTolerateFailure([Values] IteratorType iterato switch (iteratorType) { case IteratorType.AsyncByteVector: - await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable().ConfigureAwait(false)) + await foreach ((byte[] result, int _, long _, long _ /*nextAddress*/) in iter.GetAsyncEnumerable().ConfigureAwait(false)) ClassicAssert.IsTrue(result.SequenceEqual(entry)); break; case IteratorType.AsyncMemoryOwner: - await foreach ((IMemoryOwner result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable(MemoryPool.Shared).ConfigureAwait(false)) + await foreach ((IMemoryOwner result, int _, long _, long _ /*nextAddress*/) in iter.GetAsyncEnumerable(MemoryPool.Shared).ConfigureAwait(false)) { ClassicAssert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); result.Dispose(); diff --git a/libs/storage/Tsavorite/cs/test/InsertAtTailSpanByteStressTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/InsertAtTailSpanByteStressTests.cs similarity index 71% rename from libs/storage/Tsavorite/cs/test/InsertAtTailSpanByteStressTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/InsertAtTailSpanByteStressTests.cs index d6717518c67..8ae89ed6f16 100644 --- a/libs/storage/Tsavorite/cs/test/InsertAtTailSpanByteStressTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/InsertAtTailSpanByteStressTests.cs @@ -1,12 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,7 +15,7 @@ namespace Tsavorite.test.InsertAtTailStressTests { - using SpanByteStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; // Number of mutable pages for this test public enum MutablePages @@ -26,14 +24,12 @@ public enum MutablePages One, Two } - - [AllureNUnit] [TestFixture] - class SpanByteInsertAtTailChainTests : AllureTestBase + class SpanByteInsertAtTailChainTests : TestBase { - private TsavoriteKV> store; + private TsavoriteKV> store; private IDevice log; - SpanByteComparerModulo comparer; + SpanByteKeyComparerModulo comparer; const long ValueAdd = 1_000_000_000; const long NumKeys = 2_000; @@ -71,20 +67,20 @@ public void Setup() } // Make the main log mutable region small enough that we force the readonly region to stay close to tail, causing inserts. - int pageBits = 15, memoryBits = 34; - KVSettings kvSettings = new() + int pageBits = 15, memoryBits = 24; + KVSettings kvSettings = new() { LogDevice = log, PageSize = 1L << pageBits, - MemorySize = 1L << memoryBits, + LogMemorySize = 1L << memoryBits, MutableFraction = 8.0 / (1 << (memoryBits - pageBits)), }; store = new(kvSettings - , StoreFunctions.Create(comparer, SpanByteRecordDisposer.Instance) + , StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - comparer = new SpanByteComparerModulo(modRange); + comparer = new SpanByteKeyComparerModulo(modRange); } [TearDown] @@ -100,40 +96,48 @@ public void TearDown() internal class RmwSpanByteFunctions : SpanByteFunctions { /// - public override bool ConcurrentWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - src.CopyTo(ref dst); - src.CopyTo(ref output, memoryPool); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + srcValue.CopyTo(ref output, memoryPool); return true; } /// - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - src.CopyTo(ref dst); - src.CopyTo(ref output, memoryPool); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(srcValue, in sizeInfo)) + return false; + srcValue.CopyTo(ref output, memoryPool); return true; } /// - public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - input.CopyTo(ref newValue); + if (!dstLogRecord.TryCopyFrom(in srcLogRecord, in sizeInfo)) + return false; input.CopyTo(ref output, memoryPool); return true; } /// - public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { // The default implementation of IPU simply writes input to destination, if there is space - base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(input.ReadOnlySpan, in sizeInfo)) + return false; input.CopyTo(ref output, memoryPool); return true; } /// - public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { Assert.Fail("For these tests, InitialUpdater should never be called"); return false; @@ -142,16 +146,15 @@ public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref Sp unsafe void PopulateAndSetReadOnlyToTail() { - using var session = store.NewSession>(new SpanByteFunctions()); + using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[sizeof(long)]; for (long ii = 0; ii < NumKeys; ii++) { - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - var status = bContext.Upsert(ref key, ref key); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), key); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } bContext.CompletePending(true); @@ -169,8 +172,6 @@ public void SpanByteTailInsertMultiThreadTest([Values] HashModulo modRange, [Val Assert.Ignore("Skipped due to 0 threads for both read and update"); if ((numReadThreads > 2 || numWriteThreads > 2) && IsRunningAzureTests) Assert.Ignore("Skipped because > 2 threads when IsRunningAzureTests"); - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); // Initial population so we know we can read the keys. PopulateAndSetReadOnlyToTail(); @@ -178,11 +179,10 @@ public void SpanByteTailInsertMultiThreadTest([Values] HashModulo modRange, [Val const int numIterations = 10; unsafe void runReadThread(int tid) { - using var session = store.NewSession>(new SpanByteFunctions()); + using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[sizeof(long)]; for (var iteration = 0; iteration < numIterations; ++iteration) { @@ -191,8 +191,8 @@ unsafe void runReadThread(int tid) { SpanByteAndMemory output = default; - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - var status = bContext.Read(ref key, ref output); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(key), ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -203,7 +203,7 @@ unsafe void runReadThread(int tid) ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {ii}, {status}, wasPending {false}, pt 1"); ClassicAssert.IsNotNull(output.Memory, $"tid {tid}, key {ii}, wasPending {false}, pt 2"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.Span); ClassicAssert.AreEqual(ii, value % ValueAdd, $"tid {tid}, key {ii}, wasPending {false}, pt 3"); output.Memory.Dispose(); } @@ -220,13 +220,13 @@ unsafe void runReadThread(int tid) status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; // Note: do NOT overwrite 'key' here - long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.AsReadOnlySpan()); + long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.KeyBytes); - ClassicAssert.AreEqual(completedOutputs.Current.RecordMetadata.Address == Constants.kInvalidAddress, status.Record.CopiedToReadCache, $"key {keyLong}: {status}"); + ClassicAssert.AreEqual(completedOutputs.Current.RecordMetadata.Address == LogAddress.kInvalidAddress, status.Record.CopiedToReadCache, $"key {keyLong}: {status}"); ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {keyLong}, {status}, wasPending {true}, pt 1"); ClassicAssert.IsNotNull(output.Memory, $"tid {tid}, key {keyLong}, wasPending {true}, pt 2"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.Span); ClassicAssert.AreEqual(keyLong, value % ValueAdd, $"tid {tid}, key {keyLong}, wasPending {true}, pt 3"); output.Memory.Dispose(); } @@ -239,13 +239,12 @@ unsafe void runReadThread(int tid) unsafe void runUpdateThread(int tid) { - using var session = store.NewSession>(new RmwSpanByteFunctions()); + using var session = store.NewSession>(new RmwSpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); - Span inputVec = stackalloc byte[sizeof(long)]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[sizeof(long)]; + Span input = stackalloc byte[sizeof(long)]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); for (var iteration = 0; iteration < numIterations; ++iteration) { @@ -254,11 +253,11 @@ unsafe void runUpdateThread(int tid) { SpanByteAndMemory output = default; - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(inputVec, ii + ValueAdd)); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(input, ii + ValueAdd)); var status = updateOp == UpdateOp.RMW - ? bContext.RMW(ref key, ref input, ref output) - : bContext.Upsert(ref key, ref input, ref input, ref output); + ? bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, ref output) + : bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -272,7 +271,7 @@ unsafe void runUpdateThread(int tid) if (updateOp == UpdateOp.RMW) // Upsert will not try to find records below HeadAddress, but it may find them in-memory ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {ii}, {status}"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.Span); ClassicAssert.AreEqual(ii + ValueAdd, value, $"tid {tid}, key {ii}, wasPending {false}"); output.Memory?.Dispose(); @@ -290,12 +289,12 @@ unsafe void runUpdateThread(int tid) status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; // Note: do NOT overwrite 'key' here - long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.AsReadOnlySpan()); + long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.KeyBytes); if (updateOp == UpdateOp.RMW) // Upsert will not try to find records below HeadAddress, but it may find them in-memory ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {keyLong}, {status}"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.Span); ClassicAssert.AreEqual(keyLong + ValueAdd, value, $"tid {tid}, key {keyLong}, wasPending {true}"); output.Memory?.Dispose(); diff --git a/libs/storage/Tsavorite/cs/test/LogAndDeviceConfigTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogAndDeviceConfigTests.cs similarity index 95% rename from libs/storage/Tsavorite/cs/test/LogAndDeviceConfigTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogAndDeviceConfigTests.cs index 9405f1f25b2..9745de5a018 100644 --- a/libs/storage/Tsavorite/cs/test/LogAndDeviceConfigTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogAndDeviceConfigTests.cs @@ -1,7 +1,6 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,10 +13,8 @@ namespace Tsavorite.test //* is for areas / parameters not covered by the tests in other areas of the test system //* For completeness, setting other parameters too where possible //* However, the verification is pretty light. Just makes sure log file created and things be added and read from it - - [AllureNUnit] [TestFixture] - internal class LogAndDeviceConfigTests : AllureTestBase + internal class LogAndDeviceConfigTests : TestBase { private TsavoriteLog log; private IDevice device; diff --git a/libs/storage/Tsavorite/cs/test/LogFastCommitTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogFastCommitTests.cs similarity index 93% rename from libs/storage/Tsavorite/cs/test/LogFastCommitTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogFastCommitTests.cs index ddf79a1d5b8..6539699e21b 100644 --- a/libs/storage/Tsavorite/cs/test/LogFastCommitTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogFastCommitTests.cs @@ -5,14 +5,12 @@ using System.Collections.Generic; using System.IO; using System.Threading; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; namespace Tsavorite.test { - [AllureNUnit] [TestFixture] internal class LogFastCommitTests : TsavoriteLogTestBase { @@ -40,9 +38,7 @@ public void TsavoriteLogSimpleFastCommitTest([Values] TestUtils.TestDeviceType d entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie1 = new byte[100]; new Random().NextBytes(cookie1); @@ -50,9 +46,7 @@ public void TsavoriteLogSimpleFastCommitTest([Values] TestUtils.TestDeviceType d ClassicAssert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie2 = new byte[100]; new Random().NextBytes(cookie2); @@ -60,9 +54,7 @@ public void TsavoriteLogSimpleFastCommitTest([Values] TestUtils.TestDeviceType d ClassicAssert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie6 = new byte[100]; new Random().NextBytes(cookie6); @@ -115,7 +107,7 @@ public void CommitRecordBoundedGrowthTest([Values] TestUtils.TestDeviceType devi entry[i] = (byte)i; for (int i = 0; i < 5 * numEntries; i++) - log.Enqueue(entry); + _ = log.Enqueue(entry); // for comparison, insert some entries without any commit records var referenceTailLength = log.TailAddress; @@ -136,16 +128,13 @@ public void CommitRecordBoundedGrowthTest([Values] TestUtils.TestDeviceType devi foreach (var t in commitThreads) t.Start(); for (int i = 0; i < 5 * numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); enqueueDone.Set(); foreach (var t in commitThreads) t.Join(); - - // TODO: Hardcoded constant --- if this number changes in TsavoriteLogRecoverInfo, it needs to be updated here too + // TODO: Hardcoded constant --- if this number changes in TsavoriteLogRecoveryInfo, it needs to be updated here too var commitRecordSize = 44; var logTailGrowth = log.TailAddress - referenceTailLength; // Check that we are not growing the log more than one commit record per user entry diff --git a/libs/storage/Tsavorite/cs/test/LogReadAsyncTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogReadAsyncTests.cs similarity index 97% rename from libs/storage/Tsavorite/cs/test/LogReadAsyncTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogReadAsyncTests.cs index 748d77bc3de..9e98ca88b5e 100644 --- a/libs/storage/Tsavorite/cs/test/LogReadAsyncTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogReadAsyncTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Buffers; using System.IO; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,9 +11,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class LogReadAsyncTests : AllureTestBase + internal class LogReadAsyncTests : TestBase { private TsavoriteLog log; private IDevice device; diff --git a/libs/storage/Tsavorite/cs/test/test.hlog/LogRecordTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogRecordTests.cs new file mode 100644 index 00000000000..1b37aad7bf1 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogRecordTests.cs @@ -0,0 +1,412 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Runtime.InteropServices; +using Garnet.test; +using NUnit.Framework; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.LogRecordTests +{ + using static Utility; + + /// + /// This also tests and , + /// which in turn tests . + /// + [TestFixture] + unsafe class LogRecordTests : TestBase + { + long nativePointer; + ObjectIdMap objectIdMap; + SpanByteAndMemory sbamOutput; + +#pragma warning disable IDE1006 // Naming Styles + const int initialKeyLen = 10; + const int initialValueLen = 40; + const int initialVarbyteSize = RecordDataHeader.MinHeaderBytes; + const int initialOptionalSize = sizeof(long) * 2; + + const int maxInlineKeySize = 64; + const int maxInlineValueSize = 128; + + const long initialETag = 1000; + const long initialExpiration = 2000; +#pragma warning restore IDE1006 // Naming Styles + + int expectedInitialActualInlineRecordSize; + int expectedInitialAllocatedInlineRecordSize; + + [SetUp] + public void Setup() + { + expectedInitialActualInlineRecordSize = RecordInfo.Size + initialVarbyteSize + initialKeyLen + initialValueLen + initialOptionalSize; + expectedInitialAllocatedInlineRecordSize = RoundUp(expectedInitialActualInlineRecordSize, Constants.kRecordAlignment); + + DeleteDirectory(MethodTestDir); + objectIdMap = new(); + } + + [TearDown] + public void TearDown() + { + objectIdMap.Clear(); + if (nativePointer != IntPtr.Zero) + { + NativeMemory.AlignedFree((void*)nativePointer); + nativePointer = IntPtr.Zero; + } + sbamOutput.Dispose(); + DeleteDirectory(MethodTestDir); + } + + static void UpdateRecordSizeInfo(ref RecordSizeInfo sizeInfo, int keySize = -1, int valueSize = -1) + { + if (keySize > 0) + sizeInfo.FieldInfo.KeySize = keySize; + if (valueSize > 0) + sizeInfo.FieldInfo.ValueSize = valueSize; + + // Clear packed word since we are re-evaluating; CalculateSizes will set KeyLengthBytes/RecordLengthBytes. + sizeInfo.word = 0; + + // Key + if (sizeInfo.FieldInfo.KeySize <= maxInlineKeySize) + sizeInfo.SetKeyIsInline(); + keySize = sizeInfo.KeyIsInline ? sizeInfo.FieldInfo.KeySize : ObjectIdMap.ObjectIdSize; + + // Value + sizeInfo.MaxInlineValueSize = maxInlineValueSize; + if (!sizeInfo.ValueIsObject && sizeInfo.FieldInfo.ValueSize <= maxInlineValueSize) + sizeInfo.SetValueIsInline(); + valueSize = sizeInfo.ValueIsInline ? sizeInfo.FieldInfo.ValueSize : ObjectIdMap.ObjectIdSize; + + // Record + sizeInfo.CalculateSizes(keySize, valueSize); + } + + [Test] + [Category(LogRecordCategory), Category(SmokeTestCategory)] + public unsafe void InlineHeaderTests() + { + const int maxRecordAllocation = (1 << 25) + (1 << 20); + nativePointer = (long)NativeMemory.AlignedAlloc(maxRecordAllocation, Constants.kCacheLineBytes); + + Assert.That(RecordDataHeader.GetByteCount(0), Is.EqualTo(1)); + + int inputKeyLength = 16; + var inputValueLength = 1 << 8 - 1; + + // Test 1- and 2-byte valueLengthByte boundary with 1-keyLengthByte key + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(1)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 0, out int keyLengthBytes, out int recordLengthBytes); + Assert.That(keyLengthBytes, Is.EqualTo(1)); + Assert.That(recordLengthBytes, Is.EqualTo(1)); + + inputValueLength = 1 << 8; + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(2)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 2, out _ /*keyLengthBytes*/, out recordLengthBytes); + Assert.That(recordLengthBytes, Is.EqualTo(2)); + + // Test 2- and 3-byte valueLengthByte boundary with 2-keyLengthByte key + inputKeyLength = inputValueLength = (1 << 16) - 1; + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(2)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 4, out keyLengthBytes, out recordLengthBytes); + Assert.That(keyLengthBytes, Is.EqualTo(2)); + Assert.That(recordLengthBytes, Is.EqualTo(3)); // We need an extra byte now + + inputValueLength = 1 << 16; + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(3)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 7, out _ /*keyLengthBytes*/, out recordLengthBytes); + Assert.That(recordLengthBytes, Is.EqualTo(3)); + + // Test 3-byte valueLengthByte boundary with 3-keyLengthByte key, but the combination of keyLength and valueLength mean we need 4 bytes for recordLength. + inputKeyLength = inputValueLength = (1 << 24) - 1024; + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(3)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 0, out keyLengthBytes, out recordLengthBytes); + Assert.That(keyLengthBytes, Is.EqualTo(3)); + Assert.That(recordLengthBytes, Is.EqualTo(4)); // Need an additional byte in recordLength + + // Test 4-byte valueLengthByte boundary with 4-keyLengthByte key, making the recordLength also 4 bytes + inputKeyLength = inputValueLength = 1 << 24; + Assert.That(RecordDataHeader.GetByteCount(inputValueLength), Is.EqualTo(4)); + InitializeKeyAndValue(inputKeyLength, inputValueLength, exNameSpaceLength: 0, out keyLengthBytes, out recordLengthBytes); + Assert.That(keyLengthBytes, Is.EqualTo(4)); + Assert.That(recordLengthBytes, Is.EqualTo(4)); + + void InitializeKeyAndValue(int keyLength, int valueLength, int exNameSpaceLength, out int keyLengthBytes, out int recordLengthBytes) + { + // 8*3 is for optionals, including ETag and Expiration and ObjectLogPosition. And some extra buffer just to be safe for the test. + Assert.That(keyLength + valueLength + exNameSpaceLength + RecordDataHeader.MaxHeaderBytes + 8 * 3 + 1024, Is.LessThanOrEqualTo(maxRecordAllocation)); + + var sizeInfo = new RecordSizeInfo() + { + FieldInfo = new RecordFieldInfo() + { + KeySize = keyLength, + ValueSize = valueLength, + ExtendedNamespaceSize = exNameSpaceLength + }, + MaxInlineValueSize = 1 << LogSettings.kMaxStringSizeBits + }; + sizeInfo.SetKeyIsInline(); + sizeInfo.SetValueIsInline(); + sizeInfo.CalculateSizes(sizeInfo.FieldInfo.KeySize, sizeInfo.FieldInfo.ValueSize); + + var dataHeader = new RecordDataHeader((byte*)nativePointer); + var recordInfo = RecordInfo.InitialValid; + var headerLength = dataHeader.Initialize(ref recordInfo, in sizeInfo, out var keyAddress, out var namespaceAddress, out var valueAddress); + (keyLengthBytes, recordLengthBytes) = dataHeader.DeconstructKVByteLengths(out var deconstructHeaderLength); + Assert.That(headerLength, Is.EqualTo(RecordDataHeader.NumIndicatorBytes + keyLengthBytes + recordLengthBytes)); + Assert.That(deconstructHeaderLength, Is.EqualTo(headerLength)); + Assert.That(keyAddress, Is.EqualTo((long)nativePointer + headerLength + exNameSpaceLength)); + Assert.That(valueAddress, Is.EqualTo(keyAddress + keyLength)); + var (keyLengthBack, keyAddressBack) = dataHeader.GetKeyFieldInfo(); + Assert.That(keyLengthBack, Is.EqualTo(keyLength)); + Assert.That(keyAddressBack, Is.EqualTo(keyAddress)); + var (valueLengthBack, valueAddressBack) = dataHeader.GetValueFieldInfo(recordInfo); + Assert.That(valueLengthBack, Is.EqualTo(valueLength)); + Assert.That(valueAddressBack, Is.EqualTo(valueAddress)); + + // TODO: Will need to change for variable length namespaces + Assert.That(namespaceAddress, Is.EqualTo((long)nativePointer + RecordDataHeader.NamespaceOffsetInHeader)); + } + } + + [Test] + [Category(LogRecordCategory), Category(SmokeTestCategory)] + //[Repeat(900)] + public unsafe void InlineBasicTest() + { + Span key = stackalloc byte[initialKeyLen]; + Span value = stackalloc byte[initialValueLen]; + + key.Fill(0x42); + value.Fill(0x43); + + var sizeInfo = new RecordSizeInfo(); + InitializeRecord(TestSpanByteKey.FromPinnedSpan(key), value, ref sizeInfo, out var logRecord, out var expectedFillerLength, out long eTag, out long expiration); + + // Shrink + var offset = 12; + sizeInfo.FieldInfo.ValueSize = initialValueLen - offset; + Assert.That(logRecord.TrySetContentLengths(in sizeInfo), Is.True); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + offset)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + + // Grow within range + offset = 6; + sizeInfo.FieldInfo.ValueSize = initialValueLen - offset; + Assert.That(logRecord.TrySetContentLengths(in sizeInfo), Is.True); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + offset)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + + // Grow beyond range + offset = -10; + sizeInfo.FieldInfo.ValueSize = initialValueLen - offset; + Assert.That(logRecord.TrySetContentLengths(in sizeInfo), Is.False); + + // Restore to original + sizeInfo.FieldInfo.ValueSize = initialValueLen; + Assert.That(logRecord.TrySetContentLengths(in sizeInfo), Is.True); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + + // Remove ETag and verify Expiration is the same and filler has grown. + Assert.That(logRecord.RemoveETag(), Is.True); + Assert.That(logRecord.Info.HasETag, Is.False); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + LogRecord.ETagSize)); + + // Restore ETag and verify Expiration is the same and filler has grown. + eTag += 10; + Assert.That(logRecord.TrySetETag(eTag), Is.True); + Assert.That(logRecord.Info.HasETag, Is.True); + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + + // Remove Expiration and verify ETag is the same and filler has grown. + Assert.That(logRecord.RemoveExpiration(), Is.True); + Assert.That(logRecord.Info.HasExpiration, Is.False); + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + LogRecord.ExpirationSize)); + + // Restore Expiration and verify ETag is the same and filler has grown. + expiration += 20; + Assert.That(logRecord.TrySetExpiration(expiration), Is.True); + Assert.That(logRecord.Info.HasExpiration, Is.True); + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + } + + [Test] + [Category(LogRecordCategory), Category(SmokeTestCategory)] + //[Repeat(900)] + public unsafe void ConversionTest() + { + Span key = stackalloc byte[initialKeyLen]; + Span value = stackalloc byte[initialValueLen]; + Span overflowValue = stackalloc byte[maxInlineValueSize + 12]; + + key.Fill(0x42); + value.Fill(0x43); + overflowValue.Fill(0x53); + + var sizeInfo = new RecordSizeInfo(); + InitializeRecord(TestSpanByteKey.FromPinnedSpan(key), value, ref sizeInfo, out var logRecord, out var expectedFillerLength, out long eTag, out long expiration); + + // Convert to overflow. Because objectIdSize is 4 bytes our value space will shrink by the original value data size less 4 bytes, but we will use 8 bytes for ObjectLogLogPosition. + var offset = value.Length - 4 - LogRecord.ObjectLogPositionSize; + ConvertToOverflow(overflowValue, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration, offset); + RestoreToOriginal(value, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration); + + // Convert to Object. Because objectIdSize is the same as InlineLengthPrefixSize, we can reuse the same offset as above. + ConvertToObject(ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration, offset); + RestoreToOriginal(value, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration); + + // Convert to overflow, then to object, then back to overflow and back to original + ConvertToOverflow(overflowValue, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration, offset); + ConvertToObject(ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration, offset); + ConvertToOverflow(overflowValue, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration, offset); + RestoreToOriginal(value, ref sizeInfo, ref logRecord, expectedFillerLength, eTag, expiration); + } + + [Test] + [Category(LogRecordCategory), Category(SmokeTestCategory)] + [Explicit("TODO CopyDiskLogRecordToLogRecord")] + public void CopyDiskLogRecordToLogRecord() + { + Assert.Ignore("TODO CopyDiskLogRecordToLogRecord"); + } + + [Test] + [Category(LogRecordCategory), Category(SmokeTestCategory)] + [Explicit("TODO SerializeToMemoryPool")] + public void SerializeToMemoryPool() + { + Assert.Ignore("TODO SerializeToMemoryPool"); + } + + private void InitializeRecord(TKey key, Span value, ref RecordSizeInfo sizeInfo, out LogRecord logRecord, out long expectedFillerLength, out long eTag, out long expiration) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + { + sizeInfo.FieldInfo = new() + { + KeySize = initialKeyLen, + ValueSize = initialValueLen, + HasETag = true, + HasExpiration = true + }; + + UpdateRecordSizeInfo(ref sizeInfo); + Assert.That(sizeInfo.ActualInlineRecordSize, Is.EqualTo(expectedInitialActualInlineRecordSize)); + Assert.That(sizeInfo.AllocatedInlineRecordSize, Is.EqualTo(expectedInitialAllocatedInlineRecordSize)); + Assert.That(sizeInfo.KeyIsInline, Is.True); + Assert.That(sizeInfo.ValueIsInline, Is.True); + + nativePointer = (long)NativeMemory.AlignedAlloc((nuint)sizeInfo.AllocatedInlineRecordSize, Constants.kCacheLineBytes); + logRecord = new LogRecord(nativePointer, objectIdMap) { InfoRef = default }; + logRecord.InitializeRecord(key, in sizeInfo); + + // InitializeValue + Assert.That(logRecord.ValueSpan.Length, Is.EqualTo(initialValueLen)); + + expectedFillerLength = logRecord.AllocatedSize - logRecord.ActualSize; + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + + Assert.That(logRecord.TrySetValueSpanAndPrepareOptionals(value, in sizeInfo), Is.True); + + // Now that we have set the ValueSpan it includes optionals, so FillerLength should have been adjusted for them + expectedFillerLength -= LogRecord.ETagSize + LogRecord.ExpirationSize; + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + + Assert.That(logRecord.Info.ValueIsInline, Is.True); + Assert.That(logRecord.Info.ValueIsOverflow, Is.False); + Assert.That(logRecord.Info.ValueIsObject, Is.False); + Assert.That(logRecord.ValueSpan.Length, Is.EqualTo(value.Length)); + Assert.That(logRecord.ValueSpan.Slice(0, sizeof(int)).AsRef(), Is.EqualTo(0x43434343)); + + eTag = initialETag; + Assert.That(logRecord.TrySetETag(eTag), Is.True); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); // Should not have changed + + expiration = initialExpiration; + Assert.That(logRecord.TrySetExpiration(expiration), Is.True); + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); // Should not have changed + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + } + + private static void ConvertToOverflow(Span overflowValue, ref RecordSizeInfo sizeInfo, ref LogRecord logRecord, long expectedFillerLength, long eTag, long expiration, int offset) + { + sizeInfo.FieldInfo.ValueSize = overflowValue.Length; + sizeInfo.FieldInfo.ValueIsObject = false; + UpdateRecordSizeInfo(ref sizeInfo); + + Assert.That(logRecord.TrySetValueSpanAndPrepareOptionals(overflowValue, in sizeInfo), Is.True); + + Assert.That(logRecord.Info.ValueIsInline, Is.False); + Assert.That(logRecord.Info.ValueIsOverflow, Is.True); + Assert.That(logRecord.ValueSpan.Length, Is.EqualTo(overflowValue.Length)); + Assert.That(logRecord.ValueSpan.Slice(0, sizeof(int)).AsRef(), Is.EqualTo(0x53535353)); + + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + offset)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + } + + private static void ConvertToObject(ref RecordSizeInfo sizeInfo, ref LogRecord logRecord, long expectedFillerLength, long eTag, long expiration, int offset) + { + sizeInfo.FieldInfo.ValueSize = ObjectIdMap.ObjectIdSize; + sizeInfo.FieldInfo.ValueIsObject = true; + UpdateRecordSizeInfo(ref sizeInfo); + + var valueObject = new TestObjectValue() { value = 0x63636363 }; + Assert.That(logRecord.TrySetValueObjectAndPrepareOptionals(valueObject, in sizeInfo), Is.True); + + Assert.That(logRecord.Info.ValueIsInline, Is.False); + Assert.That(logRecord.Info.ValueIsOverflow, Is.False); + Assert.That(logRecord.Info.ValueIsObject, Is.True); + Assert.That(((TestObjectValue)logRecord.ValueObject).value, Is.EqualTo(0x63636363)); + + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength + offset)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + } + + private static void RestoreToOriginal(Span value, ref RecordSizeInfo sizeInfo, ref LogRecord logRecord, long expectedFillerLength, long eTag, long expiration) + { + sizeInfo.FieldInfo.ValueSize = initialValueLen; + sizeInfo.FieldInfo.ValueIsObject = false; + UpdateRecordSizeInfo(ref sizeInfo); + + Assert.That(logRecord.TrySetValueSpanAndPrepareOptionals(value, in sizeInfo), Is.True); + + Assert.That(logRecord.Info.ValueIsInline, Is.True); + Assert.That(logRecord.Info.ValueIsOverflow, Is.False); + Assert.That(logRecord.ValueSpan.Length, Is.EqualTo(value.Length)); + Assert.That(logRecord.ValueSpan.Slice(0, sizeof(int)).AsRef(), Is.EqualTo(0x43434343)); + + Assert.That(logRecord.RecordDataHeader.GetFillerLength(logRecord.Info, out _), Is.EqualTo(expectedFillerLength)); + + Assert.That(logRecord.ETag, Is.EqualTo(eTag)); + Assert.That(logRecord.Expiration, Is.EqualTo(expiration)); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/LogRecoverReadOnlyTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogRecoverReadOnlyTests.cs similarity index 98% rename from libs/storage/Tsavorite/cs/test/LogRecoverReadOnlyTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogRecoverReadOnlyTests.cs index 8fcc102721f..4d303b21cd7 100644 --- a/libs/storage/Tsavorite/cs/test/LogRecoverReadOnlyTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogRecoverReadOnlyTests.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,9 +14,8 @@ namespace Tsavorite.test.recovery { - [AllureNUnit] [TestFixture] - public class LogRecoverReadOnlyTests : AllureTestBase + public class LogRecoverReadOnlyTests : TestBase { const int ProducerPauseMs = 1; const int CommitPeriodMs = 20; diff --git a/libs/storage/Tsavorite/cs/test/LogScanTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogScanTests.cs similarity index 92% rename from libs/storage/Tsavorite/cs/test/LogScanTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogScanTests.cs index 919fc02ca78..b7f7027570b 100644 --- a/libs/storage/Tsavorite/cs/test/LogScanTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogScanTests.cs @@ -1,19 +1,19 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; +#pragma warning disable IDE1006 // Naming Styles + namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class LogScanTests : AllureTestBase + internal class LogScanTests : TestBase { private TsavoriteLog log; private IDevice device; @@ -69,7 +69,7 @@ public void PopulateLog(TsavoriteLog log) entry[i - 1] = (byte)(i - 1); // Add to TsavoriteLog - log.Enqueue(entry); + _ = log.Enqueue(entry); } // Commit to the log @@ -95,13 +95,12 @@ public void PopulateUncommittedLog(TsavoriteLog logUncommitted) entry[j - 1] = (byte)(j - 1); // Add to TsavoriteLog - logUncommitted.Enqueue(entry); + _ = logUncommitted.Enqueue(entry); } // Wait for safe tail to catch up - while (logUncommitted.SafeTailAddress < logUncommitted.TailAddress) - Thread.Yield(); - + while (logUncommitted.RefreshSafeTailAddress() < logUncommitted.TailAddress) + _ = Thread.Yield(); } [Test] @@ -120,7 +119,7 @@ public void ScanBasicDefaultTest([Values] TestUtils.TestDeviceType deviceType) // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -152,7 +151,7 @@ public void ScanBehindBeginAddressTest([Values] TestUtils.TestDeviceType deviceT // Indirectly used in other tests, but good to have the basic test here for completeness // Read the log - Look for the flag so know each entry is unique - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { var next = iter.GetNext(out byte[] result, out _, out _); ClassicAssert.IsTrue(next); @@ -168,7 +167,7 @@ public void ScanBehindBeginAddressTest([Values] TestUtils.TestDeviceType deviceT // Wait for allocator to realize the new BeginAddress // Needed as this is done post-commit while (log.AllocatorBeginAddress < log.TailAddress) - Thread.Yield(); + _ = Thread.Yield(); // Iterator will skip ahead to tail next = iter.GetNext(out result, out _, out _); @@ -181,7 +180,7 @@ public void ScanBehindBeginAddressTest([Values] TestUtils.TestDeviceType deviceT tcs.Cancel(); try { - task.GetAwaiter().GetResult(); + _ = task.GetAwaiter().GetResult(); } catch { } } @@ -218,7 +217,7 @@ public void ScanConsumerTest([Values] TestUtils.TestDeviceType deviceType) // Read the log - Look for the flag so know each entry is unique var consumer = new TestConsumer(); - using (var iter = log.Scan(0, 100_000_000)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress)) { while (iter.TryConsumeNext(consumer)) { } } @@ -241,7 +240,7 @@ public void ScanNoDefaultTest([Values] TestUtils.TestDeviceType deviceType) // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000, recover: true, scanBufferingMode: ScanBufferingMode.DoublePageBuffering, scanUncommitted: false)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress, recover: true, scanBufferingMode: DiskScanBufferingMode.DoublePageBuffering, scanUncommitted: false)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -273,7 +272,7 @@ public void ScanByNameTest([Values] TestUtils.TestDeviceType deviceType) // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000, recover: true)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress, recover: true)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -305,7 +304,7 @@ public void ScanWithoutRecoverTest([Values] TestUtils.TestDeviceType deviceType) // Read the log int currentEntry = 9; // since starting at specified address of 1000, need to set current entry as 9 so verification starts at proper spot - using (var iter = log.Scan(1000, 100_000_000, recover: false)) + using (var iter = log.Scan(1000, LogAddress.MaxValidAddress, recover: false)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -337,7 +336,7 @@ public void ScanBufferingModeDoublePageTest([Values] TestUtils.TestDeviceType de // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000, scanBufferingMode: ScanBufferingMode.DoublePageBuffering)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress, scanBufferingMode: DiskScanBufferingMode.DoublePageBuffering)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -367,7 +366,7 @@ public void ScanBufferingModeSinglePageTest([Values] TestUtils.TestDeviceType de // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000, scanBufferingMode: ScanBufferingMode.SinglePageBuffering)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress, scanBufferingMode: DiskScanBufferingMode.SinglePageBuffering)) { while (iter.GetNext(out byte[] result, out _, out _)) { @@ -392,13 +391,13 @@ public void ScanUncommittedTest([Values] TestUtils.TestDeviceType deviceType) // Create log and device here (not in setup) because using DeviceType Enum which can't be used in Setup string filename = Path.Join(TestUtils.MethodTestDir, "LogScan" + deviceType.ToString() + ".log"); device = TestUtils.CreateTestDevice(deviceType, filename); - log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = TestUtils.MethodTestDir, SafeTailRefreshFrequencyMs = 0 }); + log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, SegmentSizeBits = 22, LogCommitDir = TestUtils.MethodTestDir }); PopulateUncommittedLog(log); // Setting scanUnCommitted to true is actual test here. // Read the log - Look for the flag so know each entry is unique and still reads uncommitted int currentEntry = 0; - using (var iter = log.Scan(0, 100_000_000, scanUncommitted: true)) + using (var iter = log.Scan(0, LogAddress.MaxValidAddress, scanUncommitted: true)) { while (iter.GetNext(out byte[] result, out _, out _)) { diff --git a/libs/storage/Tsavorite/cs/test/LogShiftTailStressTest.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogShiftTailStressTest.cs similarity index 94% rename from libs/storage/Tsavorite/cs/test/LogShiftTailStressTest.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogShiftTailStressTest.cs index 7947051cb8e..cf78395fa66 100644 --- a/libs/storage/Tsavorite/cs/test/LogShiftTailStressTest.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogShiftTailStressTest.cs @@ -4,13 +4,11 @@ using System; using System.Collections.Generic; using System.Threading; -using Allure.NUnit; using NUnit.Framework; using Tsavorite.core; namespace Tsavorite.test { - [AllureNUnit] [TestFixture] internal class LogShiftTailStressTest : TsavoriteLogTestBase { @@ -34,7 +32,7 @@ public void TsavoriteLogShiftTailStressTest() entry[i] = (byte)i; for (int i = 0; i < 5 * numEntries; i++) - log.Enqueue(entry); + _ = log.Enqueue(entry); // for comparison, insert some entries without any commit records var referenceTailLength = log.TailAddress; @@ -55,9 +53,7 @@ public void TsavoriteLogShiftTailStressTest() foreach (var t in commitThreads) t.Start(); for (int i = 0; i < 5 * numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); enqueueDone.Set(); foreach (var t in commitThreads) diff --git a/libs/storage/Tsavorite/cs/test/LogTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/LogTests.cs similarity index 83% rename from libs/storage/Tsavorite/cs/test/LogTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/LogTests.cs index bbe2f444545..00b19cd27ae 100644 --- a/libs/storage/Tsavorite/cs/test/LogTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/LogTests.cs @@ -8,32 +8,33 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +#pragma warning disable IDE1006 // Naming Styles namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class TsavoriteLogStandAloneTests : AllureTestBase + internal class TsavoriteLogStandAloneTests : TestBase { [Test] [Category("TsavoriteLog")] [Category("Smoke")] - public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.TestDeviceType deviceType) + public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestDeviceType deviceType) { - string filename = Path.Join(TestUtils.MethodTestDir, "TestDisposeRelease" + deviceType.ToString() + ".log"); + string filename = Path.Join(MethodTestDir, "TestDisposeRelease" + deviceType.ToString() + ".log"); - DirectoryInfo di = Directory.CreateDirectory(TestUtils.MethodTestDir); - IDevice device = TestUtils.CreateTestDevice(deviceType, filename); + _ = Directory.CreateDirectory(MethodTestDir); + IDevice device = CreateTestDevice(deviceType, filename); TsavoriteLog log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, SegmentSizeBits = 22, - LogCommitDir = TestUtils.MethodTestDir, + LogCommitDir = MethodTestDir, LogChecksum = LogChecksumType.PerEntry }); @@ -42,22 +43,12 @@ public void TestDisposeReleasesFileLocksWithCompletedCommit([Values] TestUtils.T log.Commit(spinWait: true); log.Dispose(); device.Dispose(); - while (true) - { - try - { - di.Delete(recursive: true); - break; - } - catch - { - } - } + DeleteDirectory(MethodTestDir, wait: true); } } // This test base class allows splitting up the tests into separate fixtures that can be run in parallel - internal class TsavoriteLogTestBase : AllureTestBase + internal class TsavoriteLogTestBase : TestBase { protected const int entryLength = 100; protected const int numEntries = 10000; //1000000; @@ -67,26 +58,26 @@ internal class TsavoriteLogTestBase : AllureTestBase protected DeviceLogCommitCheckpointManager manager; protected static readonly byte[] entry = new byte[100]; - protected static readonly ReadOnlySpanBatch spanBatch = new ReadOnlySpanBatch(10000); + protected static readonly ReadOnlySpanBatch spanBatch = new(10000); private bool deleteOnClose; - protected struct ReadOnlySpanBatch : IReadOnlySpanBatch + protected struct ReadOnlySpanBatch(int batchSize) : IReadOnlySpanBatch { - private readonly int batchSize; - public ReadOnlySpanBatch(int batchSize) => this.batchSize = batchSize; - public ReadOnlySpan Get(int index) => entry; - public int TotalEntries() => batchSize; + private readonly int batchSize = batchSize; + + public readonly ReadOnlySpan Get(int index) => entry; + public readonly int TotalEntries() => batchSize; } protected void BaseSetup(bool deleteOnClose = true) { // Clean up log files from previous test runs in case they weren't cleaned up - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + DeleteDirectory(MethodTestDir, wait: true); manager = new DeviceLogCommitCheckpointManager( new LocalStorageNamedDeviceFactoryCreator(deleteOnClose: deleteOnClose), - new DefaultCheckpointNamingScheme(TestUtils.MethodTestDir), false); + new DefaultCheckpointNamingScheme(MethodTestDir), false); this.deleteOnClose = deleteOnClose; } @@ -129,11 +120,11 @@ public enum IteratorType { AsyncByteVector, AsyncMemoryOwner, - Sync, + Sync } internal static bool IsAsync(IteratorType iterType) => - iterType == IteratorType.AsyncByteVector || iterType == IteratorType.AsyncMemoryOwner; + iterType is IteratorType.AsyncByteVector or IteratorType.AsyncMemoryOwner; protected async ValueTask AssertGetNext( IAsyncEnumerator<(byte[] entry, int entryLength, long currentAddress, long nextAddress)> @@ -181,15 +172,13 @@ protected static async Task LogWriterAsync(TsavoriteLog log, byte[] entry) CancellationToken token = cts.Token; // Enter in some entries then wait on this separate thread - await log.EnqueueAsync(entry).ConfigureAwait(false); - await log.EnqueueAsync(entry).ConfigureAwait(false); - var commitTask = await log.CommitAsync(null, null, token).ConfigureAwait(false); - await log.EnqueueAsync(entry).ConfigureAwait(false); - await log.CommitAsync(commitTask, null, token).ConfigureAwait(false); + _ = await log.EnqueueAsync(entry).ConfigureAwait(false); + _ = await log.EnqueueAsync(entry).ConfigureAwait(false); + var commitTask = await log.CommitAsync(null, null, token); + _ = await log.EnqueueAsync(entry).ConfigureAwait(false); + _ = await log.CommitAsync(commitTask, null, token).ConfigureAwait(false); } } - - [AllureNUnit] [TestFixture] internal class TsavoriteLogGeneralTests : TsavoriteLogTestBase { @@ -203,7 +192,7 @@ internal class TsavoriteLogGeneralTests : TsavoriteLogTestBase [Category("TsavoriteLog")] public async ValueTask TsavoriteLogTest1([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false) : new TsavoriteLog(logSettings); @@ -213,9 +202,7 @@ public async ValueTask TsavoriteLogTest1([Values] LogChecksumType logChecksum, [ entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); @@ -226,7 +213,7 @@ public async ValueTask TsavoriteLogTest1([Values] LogChecksumType logChecksum, [ switch (iteratorType) { case IteratorType.AsyncByteVector: - await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable().ConfigureAwait(false)) + await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable().ConfigureAwait(false)) { ClassicAssert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(nextAddress); @@ -234,7 +221,7 @@ public async ValueTask TsavoriteLogTest1([Values] LogChecksumType logChecksum, [ break; case IteratorType.AsyncMemoryOwner: - await foreach ((IMemoryOwner result, int _, long _, long nextAddress) in iter + await foreach ((IMemoryOwner result, _, _, long nextAddress) in iter .GetAsyncEnumerable(MemoryPool.Shared).ConfigureAwait(false)) { ClassicAssert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); @@ -265,7 +252,7 @@ public async ValueTask TsavoriteLogTest2([Values] LogChecksumType logChecksum) { var iteratorType = IteratorType.Sync; - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false) : new TsavoriteLog(logSettings); @@ -279,9 +266,7 @@ public async ValueTask TsavoriteLogTest2([Values] LogChecksumType logChecksum) entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); @@ -292,7 +277,7 @@ public async ValueTask TsavoriteLogTest2([Values] LogChecksumType logChecksum) switch (iteratorType) { case IteratorType.AsyncByteVector: - await foreach ((byte[] result, int _, long _, long nextAddress) in iter.GetAsyncEnumerable().ConfigureAwait(false)) + await foreach ((byte[] result, _, _, long nextAddress) in iter.GetAsyncEnumerable().ConfigureAwait(false)) { ClassicAssert.IsTrue(result.SequenceEqual(entry)); counter.IncrementAndMaybeTruncateUntil(nextAddress); @@ -300,7 +285,7 @@ public async ValueTask TsavoriteLogTest2([Values] LogChecksumType logChecksum) break; case IteratorType.AsyncMemoryOwner: - await foreach ((IMemoryOwner result, int _, long _, long nextAddress) in iter + await foreach ((IMemoryOwner result, _, _, long nextAddress) in iter .GetAsyncEnumerable(MemoryPool.Shared).ConfigureAwait(false)) { ClassicAssert.IsTrue(result.Memory.Span.ToArray().Take(entry.Length).SequenceEqual(entry)); @@ -327,8 +312,8 @@ public async ValueTask TsavoriteLogTest2([Values] LogChecksumType logChecksum) internal class TestConsumer : ILogEntryConsumer { - private Counter counter; - private byte[] entry; + private readonly Counter counter; + private readonly byte[] entry; internal TestConsumer(Counter counter, byte[] entry) { @@ -347,7 +332,7 @@ public unsafe void Consume(byte* payloadPtr, int payloadLength, long currentAddr [Category("TsavoriteLog")] public void TsavoriteLogConsumerTest([Values] LogChecksumType logChecksum) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = new TsavoriteLog(logSettings); @@ -357,9 +342,7 @@ public void TsavoriteLogConsumerTest([Values] LogChecksumType logChecksum) entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); @@ -378,7 +361,7 @@ public void TsavoriteLogConsumerTest([Values] LogChecksumType logChecksum) [Category("TsavoriteLog")] public async ValueTask TsavoriteLogAsyncConsumerTest([Values] LogChecksumType logChecksum) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false); @@ -388,9 +371,7 @@ public async ValueTask TsavoriteLogAsyncConsumerTest([Values] LogChecksumType lo entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); log.CompleteLog(true); @@ -402,9 +383,6 @@ public async ValueTask TsavoriteLogAsyncConsumerTest([Values] LogChecksumType lo ClassicAssert.AreEqual(numEntries, counter.count); } } - - - [AllureNUnit] [TestFixture] internal class TsavoriteLogEnqueueTests : TsavoriteLogTestBase { @@ -421,7 +399,7 @@ public async ValueTask TryEnqueue1([Values] LogChecksumType logChecksum, [Values CancellationTokenSource cts = new CancellationTokenSource(); CancellationToken token = cts.Token; - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false) : new TsavoriteLog(logSettings); @@ -430,7 +408,7 @@ public async ValueTask TryEnqueue1([Values] LogChecksumType logChecksum, [Values byte[] data1 = new byte[dataLength]; for (int i = 0; i < dataLength; i++) data1[i] = (byte)i; - using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering)) + using (var iter = log.Scan(0, long.MaxValue, scanBufferingMode: DiskScanBufferingMode.SinglePageBuffering)) { var asyncByteVectorIter = iteratorType == IteratorType.AsyncByteVector ? iter.GetAsyncEnumerable().GetAsyncEnumerator() @@ -469,11 +447,10 @@ public async ValueTask TryEnqueue1([Values] LogChecksumType logChecksum, [Values [Test] [Category("TsavoriteLog")] [Category("Smoke")] - public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType, - [Values] TestUtils.TestDeviceType deviceType) + public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType, [Values] TestDeviceType deviceType) { - string filename = Path.Join(TestUtils.MethodTestDir, "TryEnqueue2" + deviceType.ToString() + ".log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, "TryEnqueue2" + deviceType.ToString() + ".log"); + device = CreateTestDevice(deviceType, filename); var logSettings = new TsavoriteLogSettings { @@ -490,7 +467,7 @@ public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values byte[] data1 = new byte[dataLength]; for (int i = 0; i < dataLength; i++) data1[i] = (byte)i; - using var iter = log.Scan(0, long.MaxValue, scanBufferingMode: ScanBufferingMode.SinglePageBuffering); + using var iter = log.Scan(0, long.MaxValue, scanBufferingMode: DiskScanBufferingMode.SinglePageBuffering); var asyncByteVectorIter = iteratorType == IteratorType.AsyncByteVector ? iter.GetAsyncEnumerable().GetAsyncEnumerator() : default; @@ -501,7 +478,7 @@ public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values var appendResult = log.TryEnqueue(data1, out _); ClassicAssert.IsTrue(appendResult); await log.CommitAsync().ConfigureAwait(false); - await iter.WaitAsync().ConfigureAwait(false); + _ = await iter.WaitAsync().ConfigureAwait(false); await AssertGetNext(asyncByteVectorIter, asyncMemoryOwnerIter, iter, data1).ConfigureAwait(false); @@ -509,7 +486,7 @@ public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values appendResult = log.TryEnqueue(data1, out _); ClassicAssert.IsTrue(appendResult); await log.CommitAsync().ConfigureAwait(false); - await iter.WaitAsync().ConfigureAwait(false); + _ = await iter.WaitAsync().ConfigureAwait(false); switch (iteratorType) { @@ -541,8 +518,6 @@ public async ValueTask TryEnqueue2([Values] LogChecksumType logChecksum, [Values } } } - - [AllureNUnit] [TestFixture] internal class TsavoriteLogTruncateTests : TsavoriteLogTestBase { @@ -556,10 +531,10 @@ internal class TsavoriteLogTruncateTests : TsavoriteLogTestBase [Category("TsavoriteLog")] [Category("Smoke")] public async ValueTask TruncateUntilBasic([Values] LogChecksumType logChecksum, - [Values] IteratorType iteratorType, [Values] TestUtils.TestDeviceType deviceType) + [Values] IteratorType iteratorType, [Values] TestDeviceType deviceType) { - string filename = Path.Join(TestUtils.MethodTestDir, "TruncateUntilBasic" + deviceType.ToString() + ".log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, "TruncateUntilBasic" + deviceType.ToString() + ".log"); + device = CreateTestDevice(deviceType, filename); var logSettings = new TsavoriteLogSettings { @@ -576,9 +551,7 @@ public async ValueTask TruncateUntilBasic([Values] LogChecksumType logChecksum, for (int i = 0; i < 100; i++) data1[i] = (byte)i; for (int i = 0; i < 100; i++) - { - log.Enqueue(data1); - } + _ = log.Enqueue(data1); ClassicAssert.AreEqual(log.BeginAddress, log.CommittedUntilAddress); await log.CommitAsync().ConfigureAwait(false); @@ -612,14 +585,14 @@ public async ValueTask TruncateUntilBasic([Values] LogChecksumType logChecksum, [Category("TsavoriteLog")] [Category("Smoke")] public async ValueTask EnqueueAndWaitForCommitAsyncBasicTest([Values] LogChecksumType logChecksum, - [Values] TestUtils.TestDeviceType deviceType) + [Values] TestDeviceType deviceType) { CancellationToken cancellationToken = default; ReadOnlySpanBatch spanBatch = new ReadOnlySpanBatch(numSpanEntries); - string filename = Path.Join(TestUtils.MethodTestDir, "EnqueueAndWaitForCommitAsyncBasicTest" + deviceType.ToString() + ".log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, "EnqueueAndWaitForCommitAsyncBasicTest" + deviceType.ToString() + ".log"); + device = CreateTestDevice(deviceType, filename); log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, @@ -647,22 +620,22 @@ public async ValueTask EnqueueAndWaitForCommitAsyncBasicTest([Values] LogChecksu commit.Start(); // 65536=page size|headerSize|64=log header - add cancellation token on end just so not assuming default on at least one - await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize - 64], cancellationToken).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize - 64], cancellationToken).ConfigureAwait(false); // 65536=page size|headerSize - await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(new byte[65536 - headerSize]).ConfigureAwait(false); // 65536=page size|headerSize - await log.EnqueueAndWaitForCommitAsync(spanBatch).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(spanBatch).ConfigureAwait(false); // 65536=page size|headerSize - await log.EnqueueAndWaitForCommitAsync(spanBatch, cancellationToken).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(spanBatch, cancellationToken).ConfigureAwait(false); // 65536=page size|headerSize - await log.EnqueueAndWaitForCommitAsync(readOnlyMemoryByte).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(readOnlyMemoryByte).ConfigureAwait(false); // 65536=page size|headerSize - await log.EnqueueAndWaitForCommitAsync(readOnlyMemoryByte, cancellationToken).ConfigureAwait(false); + _ = await log.EnqueueAndWaitForCommitAsync(readOnlyMemoryByte, cancellationToken).ConfigureAwait(false); // TO DO: Probably do more verification - could read it but in reality, if fails it locks up waiting @@ -675,7 +648,7 @@ public async ValueTask EnqueueAndWaitForCommitAsyncBasicTest([Values] LogChecksu [Category("TsavoriteLog")] public async ValueTask TruncateUntil2([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, @@ -683,21 +656,19 @@ public async ValueTask TruncateUntil2([Values] LogChecksumType logChecksum, [Val PageSizeBits = 14, LogChecksum = logChecksum, LogCommitManager = manager, - TryRecoverLatest = false, - SafeTailRefreshFrequencyMs = 0 + TryRecoverLatest = false }; log = IsAsync(iteratorType) ? await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false) : new TsavoriteLog(logSettings); byte[] data1 = new byte[1000]; - for (int i = 0; i < 100; i++) data1[i] = (byte)i; + for (int i = 0; i < 100; i++) + data1[i] = (byte)i; for (int i = 0; i < 100; i++) - { - log.Enqueue(data1); - } + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); ClassicAssert.AreEqual(log.TailAddress, log.SafeTailAddress); @@ -743,10 +714,10 @@ public async ValueTask TruncateUntil2([Values] LogChecksumType logChecksum, [Val } // Enqueue data, becomes auto-visible - log.Enqueue(data1); + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); await AssertGetNext(asyncByteVectorIter, asyncMemoryOwnerIter, iter, data1, verifyAtEnd: true).ConfigureAwait(false); @@ -759,26 +730,23 @@ public async ValueTask TruncateUntil2([Values] LogChecksumType logChecksum, [Val public async ValueTask TruncateUntilPageStart([Values] LogChecksumType logChecksum, [Values] IteratorType iteratorType) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, MemorySizeBits = 20, PageSizeBits = 14, LogChecksum = logChecksum, - LogCommitManager = manager, - SafeTailRefreshFrequencyMs = 0 + LogCommitManager = manager }); byte[] data1 = new byte[1000]; for (int i = 0; i < 100; i++) data1[i] = (byte)i; for (int i = 0; i < 100; i++) - { - log.Enqueue(data1); - } + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); ClassicAssert.AreEqual(log.TailAddress, log.SafeTailAddress); @@ -824,13 +792,12 @@ public async ValueTask TruncateUntilPageStart([Values] LogChecksumType logChecks } // Enqueue data, becomes auto-visible - log.Enqueue(data1); + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); - await AssertGetNext(asyncByteVectorIter, asyncMemoryOwnerIter, iter, data1, verifyAtEnd: true).ConfigureAwait(false); } @@ -841,10 +808,10 @@ public async ValueTask TruncateUntilPageStart([Values] LogChecksumType logChecks [Test] [Category("TsavoriteLog")] [Category("Smoke")] - public void CommitNoSpinWait([Values] TestUtils.TestDeviceType deviceType) + public void CommitNoSpinWait([Values] TestDeviceType deviceType) { - string filename = Path.Join(TestUtils.MethodTestDir, "CommitNoSpinWait" + deviceType.ToString() + ".log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, "CommitNoSpinWait" + deviceType.ToString() + ".log"); + device = CreateTestDevice(deviceType, filename); log = new TsavoriteLog(new TsavoriteLogSettings { LogDevice = device, LogCommitManager = manager, SegmentSizeBits = 22 }); @@ -855,9 +822,7 @@ public void CommitNoSpinWait([Values] TestUtils.TestDeviceType deviceType) entry[i] = (byte)i; for (int i = 0; i < commitFalseEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); //******* // Main point of the test ... If commit(true) (like other tests do) it waits until commit completes before moving on. @@ -868,7 +833,8 @@ public void CommitNoSpinWait([Values] TestUtils.TestDeviceType deviceType) //******* log.Commit(false); - while (log.CommittedUntilAddress < log.TailAddress) Thread.Yield(); + while (log.CommittedUntilAddress < log.TailAddress) + _ = Thread.Yield(); // Read the log - Look for the flag so know each entry is unique int currentEntry = 0; @@ -893,13 +859,13 @@ public void CommitNoSpinWait([Values] TestUtils.TestDeviceType deviceType) [Test] [Category("TsavoriteLog")] [Category("Smoke")] - public async ValueTask CommitAsyncPrevTask([Values] TestUtils.TestDeviceType deviceType) + public async ValueTask CommitAsyncPrevTask([Values] TestDeviceType deviceType) { CancellationTokenSource cts = new CancellationTokenSource(); CancellationToken token = cts.Token; - string filename = Path.Join(TestUtils.MethodTestDir, $"CommitAsyncPrevTask_{deviceType}.log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, $"CommitAsyncPrevTask_{deviceType}.log"); + device = CreateTestDevice(deviceType, filename); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogCommitManager = manager, SegmentSizeBits = 22, TryRecoverLatest = false }; log = await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false); @@ -918,7 +884,7 @@ public async ValueTask CommitAsyncPrevTask([Values] TestUtils.TestDeviceType dev Task currentTask = Task.Run(() => LogWriterAsync(log, entry), token); // Commit to the log - currentTask.Wait(4000, token); + _ = currentTask.Wait(4000, token); // double check to make sure finished - seen cases where timing kept running even after commit done bool wasCanceled = false; @@ -960,14 +926,10 @@ public async ValueTask CommitAsyncPrevTask([Values] TestUtils.TestDeviceType dev [Test] [Category("TsavoriteLog")] [Category("Smoke")] - public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iteratorType, - [Values] TestUtils.TestDeviceType deviceType) + public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iteratorType, [Values] TestDeviceType deviceType) { - CancellationTokenSource cts = new CancellationTokenSource(); - CancellationToken token = cts.Token; - - string filename = Path.Join(TestUtils.MethodTestDir, "RefreshUncommittedAsyncTest" + deviceType.ToString() + ".log"); - device = TestUtils.CreateTestDevice(deviceType, filename); + string filename = Path.Join(MethodTestDir, "RefreshUncommittedAsyncTest" + deviceType.ToString() + ".log"); + device = CreateTestDevice(deviceType, filename); log = new TsavoriteLog(new TsavoriteLogSettings { @@ -975,19 +937,17 @@ public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iterato MemorySizeBits = 20, PageSizeBits = 14, LogCommitManager = manager, - SegmentSizeBits = 22, - SafeTailRefreshFrequencyMs = 0 + SegmentSizeBits = 22 }); byte[] data1 = new byte[1000]; - for (int i = 0; i < 100; i++) data1[i] = (byte)i; + for (int i = 0; i < 100; i++) + data1[i] = (byte)i; for (int i = 0; i < 100; i++) - { - log.Enqueue(data1); - } + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); ClassicAssert.AreEqual(log.TailAddress, log.SafeTailAddress); @@ -1033,10 +993,10 @@ public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iterato } // Enqueue additional data item, becomes auto-visible - log.Enqueue(data1); + _ = log.Enqueue(data1); // Wait for safe tail to catch up - while (log.SafeTailAddress < log.TailAddress) + while (log.RefreshSafeTailAddress() < log.TailAddress) await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); await AssertGetNext(asyncByteVectorIter, asyncMemoryOwnerIter, iter, data1, verifyAtEnd: true).ConfigureAwait(false); @@ -1045,8 +1005,6 @@ public async ValueTask RefreshUncommittedAsyncTest([Values] IteratorType iterato log.Dispose(); } } - - [AllureNUnit] [TestFixture] internal class TsavoriteLogCustomCommitTests : TsavoriteLogTestBase { @@ -1064,7 +1022,7 @@ public void TsavoriteLogSimpleCommitCookieTest([Values] bool fastCommit) var cookie = new byte[100]; new Random().NextBytes(cookie); - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "SimpleCommitCookie" + fastCommit + ".log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "SimpleCommitCookie" + fastCommit + ".log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, @@ -1079,11 +1037,9 @@ public void TsavoriteLogSimpleCommitCookieTest([Values] bool fastCommit) entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); - log.CommitStrongly(out _, out _, true, cookie); + _ = log.CommitStrongly(out _, out _, true, cookie); var recoveredLog = new TsavoriteLog(logSettings); ClassicAssert.AreEqual(cookie, recoveredLog.RecoveredCookie); @@ -1094,7 +1050,7 @@ public void TsavoriteLogSimpleCommitCookieTest([Values] bool fastCommit) [Category("TsavoriteLog")] public void TsavoriteLogManualCommitTest() { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "logManualCommitTest.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "logManualCommitTest.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, @@ -1109,9 +1065,7 @@ public void TsavoriteLogManualCommitTest() entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie1 = new byte[100]; new Random().NextBytes(cookie1); @@ -1119,9 +1073,7 @@ public void TsavoriteLogManualCommitTest() ClassicAssert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie2 = new byte[100]; new Random().NextBytes(cookie2); @@ -1129,9 +1081,7 @@ public void TsavoriteLogManualCommitTest() ClassicAssert.IsTrue(commitSuccessful); for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); var cookie6 = new byte[100]; new Random().NextBytes(cookie6); @@ -1174,7 +1124,7 @@ public void TsavoriteLogManualCommitTest() [Category("TsavoriteLog")] public async ValueTask TsavoriteLogAsyncConsumerTestAfterDisposeIterator([Values] LogChecksumType logChecksum) { - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(MethodTestDir, "tsavoritelog.log"), deleteOnClose: true); var logSettings = new TsavoriteLogSettings { LogDevice = device, LogChecksum = logChecksum, LogCommitManager = manager, TryRecoverLatest = false }; log = await TsavoriteLog.CreateAsync(logSettings).ConfigureAwait(false); @@ -1184,14 +1134,10 @@ public async ValueTask TsavoriteLogAsyncConsumerTestAfterDisposeIterator([Values entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); - - var nextAddress = 0L; using (var iter = log.Scan(0, long.MaxValue)) { @@ -1206,9 +1152,7 @@ public async ValueTask TsavoriteLogAsyncConsumerTestAfterDisposeIterator([Values entry[i] = (byte)i; for (int i = 0; i < numEntries; i++) - { - log.Enqueue(entry); - } + _ = log.Enqueue(entry); log.Commit(true); log.CompleteLog(true); diff --git a/libs/storage/Tsavorite/cs/test/ManagedLocalStorageTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/ManagedLocalStorageTests.cs similarity index 96% rename from libs/storage/Tsavorite/cs/test/ManagedLocalStorageTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/ManagedLocalStorageTests.cs index 2d5b847e96e..d6b0283c281 100644 --- a/libs/storage/Tsavorite/cs/test/ManagedLocalStorageTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/ManagedLocalStorageTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; using System.Threading; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,9 +10,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class ManageLocalStorageTests : AllureTestBase + internal class ManageLocalStorageTests : TestBase { private TsavoriteLog log; private IDevice device; @@ -66,18 +64,14 @@ public void ManagedLocalStoreBasicTest() // Set Default entry data for (int i = 0; i < entryLength; i++) - { entry[i] = (byte)i; - } bool disposeCommitThread = false; var commit = new Thread(() => { while (!disposeCommitThread) - { log.Commit(true); - } }); if (commitThread) diff --git a/libs/storage/Tsavorite/cs/test/MoreLogCompactionTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/MoreLogCompactionTests.cs similarity index 61% rename from libs/storage/Tsavorite/cs/test/MoreLogCompactionTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/MoreLogCompactionTests.cs index 1489c5ca603..ce50c0a2e0a 100644 --- a/libs/storage/Tsavorite/cs/test/MoreLogCompactionTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/MoreLogCompactionTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,14 +9,12 @@ namespace Tsavorite.test { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; [TestFixture] - internal class MoreLogCompactionTests : AllureTestBase + internal class MoreLogCompactionTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; [SetUp] @@ -29,9 +26,9 @@ public void Setup() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 9 - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -53,34 +50,34 @@ public void TearDown() public void DeleteCompactLookup([Values] CompactionType compactionType) { - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; const int totalRecords = 2000; var start = store.Log.TailAddress; long compactUntil = 0; - for (int i = 0; i < totalRecords; i++) + for (long key = 0; key < totalRecords; key++) { - if (i == 1010) + if (key == 1010) compactUntil = store.Log.TailAddress; - _ = bContext.Upsert(i, i); + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); } - for (int i = 0; i < totalRecords / 2; i++) - _ = bContext.Delete(i); + for (long key = 0; key < totalRecords / 2; key++) + _ = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); compactUntil = session.Compact(compactUntil, compactionType); ClassicAssert.AreEqual(compactUntil, store.Log.BeginAddress); - using var session2 = store.NewSession>(new SimpleSimpleFunctions()); + using var session2 = store.NewSession(new SimpleLongSimpleFunctions()); var bContext2 = session2.BasicContext; // Verify records by reading - for (int i = 0; i < totalRecords; i++) + for (long key = 0; key < totalRecords; key++) { - (var status, var output) = bContext2.Read(i); + (var status, var output) = bContext2.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); if (status.IsPending) { _ = bContext2.CompletePendingWithOutputs(out var completedOutputs, true); @@ -89,14 +86,14 @@ public void DeleteCompactLookup([Values] CompactionType compactionType) ClassicAssert.IsFalse(completedOutputs.Next()); } - if (i < totalRecords / 2) + if (key < totalRecords / 2) { ClassicAssert.IsTrue(status.NotFound); } else { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(i, output); + ClassicAssert.AreEqual(key, output); } } } diff --git a/libs/storage/Tsavorite/cs/test/SpanByteIterationTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteIterationTests.cs similarity index 53% rename from libs/storage/Tsavorite/cs/test/SpanByteIterationTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/SpanByteIterationTests.cs index 21fadbea266..1fdad3176ab 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteIterationTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteIterationTests.cs @@ -1,11 +1,11 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,13 +14,11 @@ namespace Tsavorite.test { - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] + using SpanByteStoreFunctions = StoreFunctions; [TestFixture] - internal class SpanByteIterationTests : AllureTestBase + internal class SpanByteIterationTests : TestBase { - private TsavoriteKV> store; + private TsavoriteKV> store; private IDevice log; // Note: We always set value.length to 2, which includes both VLValue members; we are not exercising the "Variable Length" aspect here. @@ -43,47 +41,43 @@ public void TearDown() OnTearDown(); } - internal struct SpanBytePushIterationTestFunctions : IScanIteratorFunctions + internal struct SpanBytePushIterationTestFunctions : IScanIteratorFunctions { internal int keyMultToValue; internal long numRecords; internal int stopAt; - public unsafe bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; // default; not used here if (keyMultToValue > 0) { - var keyItem = key.AsSpan()[0]; - var valueItem = value.AsSpan()[0]; + var keyItem = MemoryMarshal.Cast(logRecord.Key)[0]; + var valueItem = MemoryMarshal.Cast(logRecord.ValueSpan)[0]; ClassicAssert.AreEqual(keyItem * keyMultToValue, valueItem); } return stopAt != ++numRecords; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public readonly bool OnStart(long beginAddress, long endAddress) => true; public readonly void OnException(Exception exception, long numberOfRecords) { } public readonly void OnStop(bool completed, long numberOfRecords) { } } - internal struct IterationCollisionTestFunctions : IScanIteratorFunctions + internal struct IterationCollisionTestFunctions : IScanIteratorFunctions { internal List keys; - public IterationCollisionTestFunctions() => keys = new(); + public IterationCollisionTestFunctions() => keys = []; - public unsafe bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public readonly bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - keys.Add(*(long*)key.ToPointer()); + keys.Add(MemoryMarshal.Cast(logRecord.Key)[0]); cursorRecordResult = CursorRecordResult.Accept; // default; not used here return true; } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - public readonly bool OnStart(long beginAddress, long endAddress) => true; public readonly void OnException(Exception exception, long numberOfRecords) { } public readonly void OnStop(bool completed, long numberOfRecords) { } @@ -99,14 +93,14 @@ public unsafe void SpanByteIterationBasicTest([Values] TestDeviceType deviceType { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 9, SegmentSize = 1L << 22 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; SpanBytePushIterationTestFunctions scanIteratorFunctions = new(); @@ -121,8 +115,8 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) if (scanIteratorType == ScanIteratorType.Pull) { using var iter = session.Iterate(); - while (iter.GetNext(out var recordInfo)) - _ = scanIteratorFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), default, default, out _); + while (iter.GetNext()) + _ = scanIteratorFunctions.Reader(in iter, default, default, out _); } else ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); @@ -133,15 +127,15 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) // Note: We only have a single value element; we are not exercising the "Variable Length" aspect here. Span keySpan = stackalloc long[1]; Span valueSpan = stackalloc int[1]; - var key = keySpan.AsSpanByte(); - var value = valueSpan.AsSpanByte(); + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); + var value = MemoryMarshal.Cast(valueSpan); // Initial population for (int i = 0; i < totalRecords; i++) { keySpan[0] = i; valueSpan[0] = i; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } iterateAndVerify(1, totalRecords); @@ -149,7 +143,7 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) { keySpan[0] = i; valueSpan[0] = i * 2; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } iterateAndVerify(2, totalRecords); @@ -157,7 +151,7 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) { keySpan[0] = i; valueSpan[0] = i; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } iterateAndVerify(0, totalRecords); @@ -165,14 +159,14 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) { keySpan[0] = i; valueSpan[0] = i; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } iterateAndVerify(0, totalRecords); for (int i = 0; i < totalRecords; i += 2) { keySpan[0] = i; - _ = bContext.Delete(ref key); + _ = bContext.Delete(key); } iterateAndVerify(0, totalRecords / 2); @@ -180,7 +174,7 @@ void iterateAndVerify(int keyMultToValue, int expectedRecs) { keySpan[0] = i; valueSpan[0] = i * 3; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } iterateAndVerify(3, totalRecords); @@ -198,14 +192,14 @@ public void SpanByteIterationPushStopTest([Values] TestDeviceType deviceType) { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 9, SegmentSize = 1L << 22 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; SpanBytePushIterationTestFunctions scanIteratorFunctions = new(); @@ -226,15 +220,15 @@ void scanAndVerify(int stopAt, bool useScan) // Note: We only have a single value element; we are not exercising the "Variable Length" aspect here. Span keySpan = stackalloc long[1]; Span valueSpan = stackalloc int[1]; - var key = keySpan.AsSpanByte(); - var value = valueSpan.AsSpanByte(); + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); + var value = MemoryMarshal.Cast(valueSpan); // Initial population for (int i = 0; i < totalRecords; i++) { keySpan[0] = i; valueSpan[0] = i; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } scanAndVerify(42, useScan: true); @@ -253,10 +247,10 @@ public unsafe void SpanByteIterationPushLockTest([Values(1, 4)] int scanThreads, { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 25, + LogMemorySize = 1L << 25, PageSize = 1L << 19, SegmentSize = 1L << 22 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -265,7 +259,7 @@ public unsafe void SpanByteIterationPushLockTest([Values(1, 4)] int scanThreads, void LocalScan(int i) { - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); SpanBytePushIterationTestFunctions scanIteratorFunctions = new(); if (scanMode == ScanMode.Scan) ClassicAssert.IsTrue(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to complete push scan; numRecords = {scanIteratorFunctions.numRecords}"); @@ -279,16 +273,16 @@ void LocalUpdate(int tid) // Note: We only have a single value element; we are not exercising the "Variable Length" aspect here. Span keySpan = stackalloc long[1]; Span valueSpan = stackalloc int[1]; - var key = keySpan.AsSpanByte(); - var value = valueSpan.AsSpanByte(); + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); + var value = MemoryMarshal.Cast(valueSpan); - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; for (int i = 0; i < totalRecords; i++) { keySpan[0] = i; valueSpan[0] = i * (tid + 1); - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } } @@ -297,16 +291,16 @@ void LocalUpdate(int tid) // Note: We only have a single value element; we are not exercising the "Variable Length" aspect here. Span keySpan = stackalloc long[1]; Span valueSpan = stackalloc int[1]; - var key = keySpan.AsSpanByte(); - var value = valueSpan.AsSpanByte(); + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); + var value = MemoryMarshal.Cast(valueSpan); - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; for (int i = 0; i < totalRecords; i++) { keySpan[0] = i; valueSpan[0] = i; - _ = bContext.Upsert(ref key, ref value); + _ = bContext.Upsert(key, value); } } @@ -322,5 +316,109 @@ void LocalUpdate(int tid) } Task.WaitAll([.. tasks]); } + + /// + /// Basic correctness test for IterateLookupSnapshot: after a series of RCU updates + /// to multiple keys (each older record gets sealed when its replacement is upserted), the + /// snapshot variant emits each unique live key exactly once and returns the latest value + /// in the snapshot. + /// + /// Note: the rule-S2 protection (snapshot vs. maxAddress = long.MaxValue default) + /// only manifests during an in-flight RCU race window where the older record has not yet + /// been sealed. That race window is too narrow to reproduce deterministically without + /// test-only instrumentation hooks; the parameter pattern this API uses internally + /// (untilAddress = maxAddress = capturedTail) is exercised by + /// MigrateOperation.Scan in the cluster code path. + /// + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + public unsafe void SpanByteIterateLookupSnapshotBasicCorrectness() + { + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "snapshot_basic.log")); + store = new(new() + { + IndexSize = 1L << 26, + LogDevice = log, + LogMemorySize = 1L << 25, + PageSize = 1L << 19, + SegmentSize = 1L << 22 + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var session = store.NewSession(new VLVectorFunctions()); + var bContext = session.BasicContext; + + Span keySpan = stackalloc long[1]; + Span valueSpan = stackalloc int[1]; + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)); + var value = MemoryMarshal.Cast(valueSpan); + + const int totalRecords = 200; + + // Phase 1: insert keys [0, totalRecords) with value = key * 1. + for (int i = 0; i < totalRecords; i++) + { + keySpan[0] = i; + valueSpan[0] = i; + _ = bContext.Upsert(key, value); + } + + // Phase 2: RCU each key to a new value (key * 10). Force RCU by flushing to read-only + // first so the upsert can't update in place. + store.Log.Flush(wait: true); + for (int i = 0; i < totalRecords; i++) + { + keySpan[0] = i; + valueSpan[0] = i * 10; + _ = bContext.Upsert(key, value); + } + + // IterateLookupSnapshot should emit each unique live key exactly once, with the + // post-RCU value (key * 10). + var fns = new SnapshotProbeAllFunctions { observed = new Dictionary() }; + _ = session.IterateLookupSnapshot(ref fns); + + ClassicAssert.AreEqual(totalRecords, fns.observed.Count, + "IterateLookupSnapshot should emit each unique live key exactly once"); + for (int i = 0; i < totalRecords; i++) + { + ClassicAssert.IsTrue(fns.observed.TryGetValue(i, out var emittedValue), + $"Key {i} not emitted by IterateLookupSnapshot"); + ClassicAssert.AreEqual(i * 10, emittedValue, + $"Key {i}: expected snapshot to expose post-RCU value but got {emittedValue}"); + } + + // Stable: a second IterateLookupSnapshot call returns the same set. + var fns2 = new SnapshotProbeAllFunctions { observed = new Dictionary() }; + _ = session.IterateLookupSnapshot(ref fns2); + ClassicAssert.AreEqual(totalRecords, fns2.observed.Count, + "Second IterateLookupSnapshot should emit the same number of records (snapshot is stable across calls)"); + } + + private struct SnapshotProbeAllFunctions : IScanIteratorFunctions + { + // NOTE: Must hold a reference type because the struct is boxed when stored in + // ScanCursorState.functions (interface field) — mutations on the boxed copy land + // there, not on the caller's struct. The Dictionary reference itself is shared. + public Dictionary observed; + + public readonly bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord + { + cursorRecordResult = CursorRecordResult.Accept; + var k = MemoryMarshal.Cast(logRecord.Key)[0]; + var v = MemoryMarshal.Cast(logRecord.ValueSpan)[0]; + ClassicAssert.IsFalse(observed.ContainsKey(k), + $"IterateLookupSnapshot emitted key {k} more than once (snapshot semantics violated)"); + observed[k] = v; + return true; + } + + public readonly bool OnStart(long beginAddress, long endAddress) => true; + public readonly void OnException(Exception exception, long numberOfRecords) { } + public readonly void OnStop(bool completed, long numberOfRecords) { } + } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/BlittableLogCompactionTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogCompactionTests.cs similarity index 60% rename from libs/storage/Tsavorite/cs/test/BlittableLogCompactionTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogCompactionTests.cs index c0a7190514c..b22a76dde49 100644 --- a/libs/storage/Tsavorite/cs/test/BlittableLogCompactionTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogCompactionTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Diagnostics; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,18 +14,31 @@ namespace Tsavorite.test { // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - struct HashModuloKeyStructComparer : IKeyComparer + struct HashModuloKeyStructComparer : IKeyComparer { readonly HashModulo modRange; internal HashModuloKeyStructComparer(HashModulo mod) => modRange = mod; - public readonly bool Equals(ref KeyStruct k1, ref KeyStruct k2) => k1.kfield1 == k2.kfield1; + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef().kfield1 == k2.KeyBytes.AsRef().kfield1; // Force collisions to create a chain - public readonly long GetHashCode64(ref KeyStruct k) + public readonly long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - var value = Utility.GetHashCode(k.kfield1); + var value = Utility.GetHashCode(k.KeyBytes.AsRef().kfield1); return modRange != HashModulo.NoMod ? value % (long)modRange : value; } } @@ -34,21 +46,19 @@ public readonly long GetHashCode64(ref KeyStruct k) namespace Tsavorite.test { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - public class BlittableLogCompactionTests : AllureTestBase + public class SpanByteLogCompactionTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; [SetUp] public void Setup() { DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "BlittableLogCompactionTests.log"), deleteOnClose: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "SpanByteLogCompactionTests.log"), deleteOnClose: true); var hashMod = HashModulo.NoMod; foreach (var arg in TestContext.CurrentContext.Test.Arguments) @@ -64,9 +74,9 @@ public void Setup() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 9 - }, StoreFunctions.Create(new HashModuloKeyStructComparer(hashMod)) + }, StoreFunctions.Create(new HashModuloKeyStructComparer(hashMod), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -81,7 +91,7 @@ public void TearDown() OnTearDown(); } - static void VerifyRead(ClientSession session, int totalRecords, Func isDeleted) + static void VerifyRead(ClientSession session, int totalRecords, Func isDeleted) { InputStruct input = default; int numPending = 0; @@ -94,14 +104,14 @@ void drainPending() { for (; outputs.Next(); --numPending) { - if (isDeleted((int)outputs.Current.Key.kfield1)) + if (isDeleted((int)outputs.Current.Key.KeyBytes.AsRef().kfield1)) { ClassicAssert.IsFalse(outputs.Current.Status.Found); continue; } ClassicAssert.IsTrue(outputs.Current.Status.Found); - ClassicAssert.AreEqual(outputs.Current.Key.kfield1, outputs.Current.Output.value.vfield1); - ClassicAssert.AreEqual(outputs.Current.Key.kfield2, outputs.Current.Output.value.vfield2); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield1, outputs.Current.Output.value.vfield1); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield2, outputs.Current.Output.value.vfield2); } } ClassicAssert.AreEqual(numPending, 0); @@ -113,7 +123,7 @@ void drainPending() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, isDeleted(i) ? 1 : 0); + var status = bContext.Read(key1, ref input, ref output, isDeleted(i) ? 1 : 0); if (!status.IsPending) { if (isDeleted(i)) @@ -138,9 +148,9 @@ void drainPending() [Category("Compaction")] [Category("Smoke")] - public void BlittableLogCompactionTest1([Values] CompactionType compactionType) + public void SpanByteLogCompactionTest1([Values] CompactionType compactionType) { - using var session = store.NewSession(new FunctionsCompaction()); + using var session = store.NewSession(new FunctionsCompaction()); var bContext = session.BasicContext; const int totalRecords = 2_000; @@ -152,9 +162,10 @@ public void BlittableLogCompactionTest1([Values] CompactionType compactionType) if (i == totalRecords - 1000) compactUntil = store.Log.TailAddress; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + _ = bContext.Upsert(key, value, 0); } store.Log.FlushAndEvict(wait: true); @@ -164,15 +175,15 @@ public void BlittableLogCompactionTest1([Values] CompactionType compactionType) ClassicAssert.AreEqual(compactUntil, store.Log.BeginAddress); // Read all keys - all should be present - BlittableLogCompactionTests.VerifyRead(session, totalRecords, key => false); + VerifyRead(session, totalRecords, key => false); } [Test] [Category("TsavoriteKV")] [Category("Compaction")] - public void BlittableLogCompactionTest2([Values] CompactionType compactionType, [Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void SpanByteLogCompactionTest2([Values] CompactionType compactionType, [Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { - using var session = store.NewSession(new FunctionsCompaction()); + using var session = store.NewSession(new FunctionsCompaction()); var bContext = session.BasicContext; const int totalRecords = 2_000; @@ -184,9 +195,10 @@ public void BlittableLogCompactionTest2([Values] CompactionType compactionType, if (i == totalRecords - 1000) compactUntil = store.Log.TailAddress; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + _ = bContext.Upsert(key, value, 0); } store.Log.FlushAndEvict(true); @@ -202,9 +214,10 @@ public void BlittableLogCompactionTest2([Values] CompactionType compactionType, // test that the address is < minAddress, so no IO is needed. for (int i = 0; i < totalRecords / 2; i++) { - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + _ = bContext.Upsert(key, value, 0); } compactUntil = session.Compact(compactUntil, compactionType); @@ -212,15 +225,15 @@ public void BlittableLogCompactionTest2([Values] CompactionType compactionType, ClassicAssert.AreEqual(compactUntil, store.Log.BeginAddress); // Read all keys - all should be present - BlittableLogCompactionTests.VerifyRead(session, totalRecords, key => false); + VerifyRead(session, totalRecords, key => false); } [Test] [Category("TsavoriteKV")] [Category("Compaction")] - public void BlittableLogCompactionTest3([Values] CompactionType compactionType) + public void SpanByteLogCompactionTest3([Values] CompactionType compactionType) { - using var session = store.NewSession(new FunctionsCompaction()); + using var session = store.NewSession(new FunctionsCompaction()); var bContext = session.BasicContext; const int totalRecords = 2_000; @@ -232,15 +245,16 @@ public void BlittableLogCompactionTest3([Values] CompactionType compactionType) if (i == totalRecords / 2) compactUntil = store.Log.TailAddress; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + _ = bContext.Upsert(key, value, 0); if (i % 8 == 0) { int j = i / 4; - key1 = new KeyStruct { kfield1 = j, kfield2 = j + 1 }; - _ = bContext.Delete(ref key1, 0); + key = new KeyStruct { kfield1 = j, kfield2 = j + 1 }; + _ = bContext.Delete(key, 0); } } @@ -250,7 +264,7 @@ public void BlittableLogCompactionTest3([Values] CompactionType compactionType) ClassicAssert.AreEqual(compactUntil, store.Log.BeginAddress); // Read all keys - all should be present except those we deleted - BlittableLogCompactionTests.VerifyRead(session, totalRecords, key => (key < totalRecords / 4) && (key % 2 == 0)); + VerifyRead(session, totalRecords, key => (key < totalRecords / 4) && (key % 2 == 0)); } [Test] @@ -258,9 +272,9 @@ public void BlittableLogCompactionTest3([Values] CompactionType compactionType) [Category("Compaction")] [Category("Smoke")] - public void BlittableLogCompactionCustomFunctionsTest1([Values] CompactionType compactionType) + public void SpanByteLogCompactionCustomFunctionsTest1([Values] CompactionType compactionType) { - using var session = store.NewSession(new FunctionsCompaction()); + using var session = store.NewSession(new FunctionsCompaction()); var bContext = session.BasicContext; InputStruct input = default; @@ -274,9 +288,10 @@ public void BlittableLogCompactionCustomFunctionsTest1([Values] CompactionType c if (i == totalRecords / 2) compactUntil = store.Log.TailAddress; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + _ = bContext.Upsert(key, value, 0); } var tail = store.Log.TailAddress; @@ -290,12 +305,13 @@ public void BlittableLogCompactionCustomFunctionsTest1([Values] CompactionType c for (var i = 0; i < totalRecords; i++) { OutputStruct output = default; - var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; + var valueStruct = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); var ctx = (i < (totalRecords / 2) && (i % 2 != 0)) ? 1 : 0; - var status = bContext.Read(ref key1, ref input, ref output, ctx); + var status = bContext.Read(key, ref input, ref output, ctx); if (status.IsPending) { ClassicAssert.IsTrue(bContext.CompletePendingWithOutputs(out var outputs, wait: true)); @@ -305,8 +321,8 @@ public void BlittableLogCompactionCustomFunctionsTest1([Values] CompactionType c if (ctx == 0) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); - ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); + ClassicAssert.AreEqual(valueStruct.vfield1, output.value.vfield1); + ClassicAssert.AreEqual(valueStruct.vfield2, output.value.vfield2); } else { @@ -318,31 +334,33 @@ public void BlittableLogCompactionCustomFunctionsTest1([Values] CompactionType c [Test] [Category("TsavoriteKV")] [Category("Compaction")] - public void BlittableLogCompactionCustomFunctionsTest2([Values] CompactionType compactionType, [Values] bool flushAndEvict) + public void SpanByteLogCompactionCustomFunctionsTest2([Values] CompactionType compactionType, [Values(FlushMode.ReadOnly, FlushMode.OnDisk)] FlushMode flushMode) { // Update: irrelevant as session compaction no longer uses Copy/CopyInPlace // This test checks if CopyInPlace returning false triggers call to Copy - using var session = store.NewSession(new FunctionsCompaction()); + using var session = store.NewSession(new FunctionsCompaction()); var bContext = session.BasicContext; var key = new KeyStruct { kfield1 = 100, kfield2 = 101 }; - var value = new ValueStruct { vfield1 = 10, vfield2 = 20 }; + var valueStruct = new ValueStruct { vfield1 = 10, vfield2 = 20 }; + var value = SpanByte.FromPinnedVariable(ref valueStruct); + var input = default(InputStruct); var output = default(OutputStruct); - _ = bContext.Upsert(ref key, ref value, 0); - var status = bContext.Read(ref key, ref input, ref output, 0); + _ = bContext.Upsert(key, value, 0); + var status = bContext.Read(key, ref input, ref output, 0); Debug.Assert(status.Found); store.Log.Flush(true); - value = new ValueStruct { vfield1 = 11, vfield2 = 21 }; - _ = bContext.Upsert(ref key, ref value, 0); - status = bContext.Read(ref key, ref input, ref output, 0); + valueStruct = new ValueStruct { vfield1 = 11, vfield2 = 21 }; + _ = bContext.Upsert(key, value, 0); + status = bContext.Read(key, ref input, ref output, 0); Debug.Assert(status.Found); - if (flushAndEvict) + if (flushMode == FlushMode.OnDisk) store.Log.FlushAndEvict(true); else store.Log.Flush(true); @@ -350,7 +368,7 @@ public void BlittableLogCompactionCustomFunctionsTest2([Values] CompactionType c var compactUntil = session.Compact(store.Log.TailAddress, compactionType); store.Log.Truncate(); - status = bContext.Read(ref key, ref input, ref output, 0); + status = bContext.Read(key, ref input, ref output, 0); if (status.IsPending) { ClassicAssert.IsTrue(bContext.CompletePendingWithOutputs(out var outputs, wait: true)); @@ -358,13 +376,15 @@ public void BlittableLogCompactionCustomFunctionsTest2([Values] CompactionType c } ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); - ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); + ClassicAssert.AreEqual(valueStruct.vfield1, output.value.vfield1); + ClassicAssert.AreEqual(valueStruct.vfield2, output.value.vfield2); } - private struct EvenCompactionFunctions : ICompactionFunctions + private struct EvenCompactionFunctions : ICompactionFunctions { - public readonly bool IsDeleted(ref KeyStruct key, ref ValueStruct value) => value.vfield1 % 2 != 0; + public bool IsDeleted(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => logRecord.ValueSpan.AsRef().vfield1 % 2 != 0; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogScanTests.cs similarity index 78% rename from libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogScanTests.cs index aa5b4693689..a7b7d57015a 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteLogScanTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteLogScanTests.cs @@ -1,34 +1,45 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; using System.Runtime.InteropServices; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; -using static Tsavorite.core.Utility; using static Tsavorite.test.SpanByteIterationTests; using static Tsavorite.test.TestUtils; namespace Tsavorite.test.spanbyte { // Must be in a separate block so the "using SpanByteStoreFunctions" is the first line in its namespace declaration. - struct SpanByteComparerModulo : IKeyComparer + struct SpanByteComparerModulo : IKeyComparer { readonly long mod; internal SpanByteComparerModulo(long mod) => this.mod = mod; - public bool Equals(ref SpanByte k1, ref SpanByte k2) => SpanByteComparer.StaticEquals(ref k1, ref k2); + public readonly bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => SpanByteComparer.StaticEquals(k1.KeyBytes, k2.KeyBytes); // Force collisions to create a chain - public long GetHashCode64(ref SpanByte k) + public readonly long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - long hash = SpanByteComparer.StaticGetHashCode64(ref k); + long hash = SpanByteComparer.StaticGetHashCode64(k.KeyBytes); return mod > 0 ? hash % mod : hash; } } @@ -36,13 +47,11 @@ public long GetHashCode64(ref SpanByte k) namespace Tsavorite.test.spanbyte { - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] + using SpanByteStoreFunctions = StoreFunctions; [TestFixture] - internal class SpanByteLogScanTests : AllureTestBase + internal class SpanByteLogScanTests : TestBase { - private TsavoriteKV> store; + private TsavoriteKV> store; private IDevice log; const int TotalRecords = 2000; const int PageSizeBits = 15; @@ -67,9 +76,9 @@ public void Setup() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 25, + LogMemorySize = 1L << 25, PageSize = 1L << PageSizeBits - }, StoreFunctions.Create(comparer, SpanByteRecordDisposer.Instance) + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -89,10 +98,10 @@ public class ScanFunctions : SpanByteFunctions // Right now this is unused but helped with debugging so I'm keeping it around. internal long insertedAddress; - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan src, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { insertedAddress = upsertInfo.Address; - return base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); + return base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, src, ref output, ref upsertInfo); } } @@ -103,7 +112,7 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H { const long PageSize = 1L << PageSizeBits; - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; Random rng = new(101); @@ -112,12 +121,12 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H { var valueFill = new string('x', rng.Next(120)); // Make the record lengths random var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); + var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } } @@ -157,11 +166,9 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H var valueFill = new string('x', rng.Next(120)); // Make the record lengths random var key = MemoryMarshal.Cast($"key_{i + TotalRecords}".AsSpan()); var value = MemoryMarshal.Cast($"v{valueFill}_{i + TotalRecords}".AsSpan()); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } } scanCursorFuncs.Initialize(verifyKeys); @@ -175,17 +182,18 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H cursor = store.hlogBase.BeginAddress - 1; do { - ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 1, scanCursorFuncs, long.MaxValue, validateCursor: true), "Expected scan to finish and return false, pt 1"); - cursor = scanCursorFuncs.lastAddress + scanCursorFuncs.lastRecordSize + 1; + ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 1, scanCursorFuncs, long.MaxValue, validateCursor: true), "Expected scan to finish and return false, pt 3"); + Assert.That(cursor, Is.EqualTo(scanCursorFuncs.lastAddress + scanCursorFuncs.lastRecordSize)); + cursor += 1; } while (cursor < PageSize * 3); // Now try an invalid cursor in-memory. First we have to read what's at the target start address (let's use HeadAddress) to find what the value is. - SpanByte input = default; + PinnedSpanByte input = default; SpanByteAndMemory output = default; ReadOptions readOptions = default; var readStatus = bContext.ReadAtAddress(store.hlogBase.HeadAddress, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(readStatus.Found, $"Could not read at HeadAddress; {readStatus}"); - var keyString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); + var keyString = new string(MemoryMarshal.Cast(output.ReadOnlySpan)); var keyOrdinal = int.Parse(keyString.Substring(keyString.IndexOf('_') + 1)); output.Memory.Dispose(); @@ -195,7 +203,8 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H do { ClassicAssert.IsTrue(session.ScanCursor(ref cursor, 1, scanCursorFuncs, long.MaxValue, validateCursor: true), "Expected scan to finish and return false, pt 1"); - cursor = scanCursorFuncs.lastAddress + scanCursorFuncs.lastRecordSize + 1; + Assert.That(cursor, Is.EqualTo(scanCursorFuncs.lastAddress + scanCursorFuncs.lastRecordSize)); + cursor += 1; } while (cursor < store.hlogBase.HeadAddress + PageSize * 3); } @@ -204,7 +213,7 @@ public unsafe void SpanByteScanCursorTest([Values(HashModulo.NoMod, HashModulo.H [Category("Smoke")] public unsafe void SpanByteScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; Random rng = new(101); @@ -214,11 +223,9 @@ public unsafe void SpanByteScanCursorFilterTest([Values(HashModulo.NoMod, HashMo var valueFill = new string('x', rng.Next(120)); // Make the record lengths random var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } } @@ -245,7 +252,7 @@ internal enum RCULocation { RCUNone, RCUBefore, RCUAfter }; [Category("Smoke")] public unsafe void SpanByteScanCursorWithRCUTest([Values(RCULocation.RCUBefore, RCULocation.RCUAfter)] RCULocation rcuLocation, [Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; Random rng = new(101); @@ -255,11 +262,9 @@ public unsafe void SpanByteScanCursorWithRCUTest([Values(RCULocation.RCUBefore, var valueFill = new string('x', rng.Next(120)); // Make the record lengths random var key = MemoryMarshal.Cast($"key_{i}".AsSpan()); var value = MemoryMarshal.Cast($"v{valueFill}_{i}".AsSpan()); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } } @@ -286,9 +291,9 @@ public unsafe void SpanByteScanCursorWithRCUTest([Values(RCULocation.RCUBefore, ClassicAssert.IsTrue(scanCursorFuncs.rcuDone, "RCU was not done"); } - internal sealed class ScanCursorFuncs : IScanIteratorFunctions + internal sealed class ScanCursorFuncs : IScanIteratorFunctions { - readonly TsavoriteKV> store; + readonly TsavoriteKV> store; internal int numRecords; internal long lastAddress; @@ -298,7 +303,7 @@ internal sealed class ScanCursorFuncs : IScanIteratorFunctions filter; - internal ScanCursorFuncs(TsavoriteKV> store) + internal ScanCursorFuncs(TsavoriteKV> store) { this.store = store; Initialize(verifyKeys: true); @@ -325,17 +330,15 @@ unsafe void CheckForRCU() // Must run this on another thread because we are epoch-protected on this one. Task.Run(() => { - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; var valueFill = new string('x', 220); // Update the specified key with a longer value that requires RCU. var key = MemoryMarshal.Cast($"key_{rcuRecord}".AsSpan()); var value = MemoryMarshal.Cast($"v{valueFill}_{rcuRecord}".AsSpan()); - fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } }).Wait(); @@ -346,9 +349,10 @@ unsafe void CheckForRCU() } } - public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); + var keyString = new string(MemoryMarshal.Cast(logRecord.Key)); var kfield1 = int.Parse(keyString.Substring(keyString.IndexOf('_') + 1)); cursorRecordResult = filter(kfield1) ? CursorRecordResult.Accept : CursorRecordResult.Skip; @@ -365,7 +369,7 @@ public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadat ClassicAssert.Greater(recordMetadata.Address, 0); lastAddress = recordMetadata.Address; - lastRecordSize = RecordInfo.GetLength() + RoundUp(key.TotalSize, 8) + RoundUp(value.TotalSize, 8); + lastRecordSize = logRecord.AllocatedSize; CheckForRCU(); ++numRecords; // Do this *after* RCU @@ -378,9 +382,6 @@ public void OnException(Exception exception, long numberOfRecords) public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } - - public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); } [Test] @@ -399,13 +400,13 @@ public unsafe void SpanByteJumpToBeginAddressTest() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, PageSize = 1L << PageSizeBits - }, StoreFunctions.Create(new SpanByteComparerModulo(0), SpanByteRecordDisposer.Instance) + }, StoreFunctions.Create(new SpanByteComparerModulo(0), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession>(new SpanByteFunctions()); + using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; const int numRecords = 200; @@ -424,9 +425,8 @@ public unsafe void SpanByteJumpToBeginAddressTest() var value = MemoryMarshal.Cast($"{i}".AsSpan()); fixed (byte* keyPtr = key) - fixed (byte* valuePtr = value) { - _ = bContext.Upsert(SpanByte.FromPinnedPointer(keyPtr, key.Length), SpanByte.FromPinnedPointer(valuePtr, value.Length)); + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); } } @@ -434,21 +434,21 @@ public unsafe void SpanByteJumpToBeginAddressTest() for (int i = 0; i < 100; ++i) { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); - ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.GetKey().AsSpan()))); - ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.GetValue().AsSpan()))); + ClassicAssert.IsTrue(iter.GetNext()); + ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.Key))); + ClassicAssert.AreEqual(i, int.Parse(MemoryMarshal.Cast(iter.ValueSpan))); } store.Log.ShiftBeginAddress(shiftBeginAddressTo); for (int i = 0; i < numTailRecords; ++i) { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); + ClassicAssert.IsTrue(iter.GetNext()); if (i == 0) ClassicAssert.AreEqual(store.Log.BeginAddress, iter.CurrentAddress); var expectedKey = numRecords - numTailRecords + i; - ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.GetKey().AsSpan()))); - ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.GetValue().AsSpan()))); + ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.Key))); + ClassicAssert.AreEqual(expectedKey, int.Parse(MemoryMarshal.Cast(iter.ValueSpan))); } } @@ -456,9 +456,11 @@ public unsafe void SpanByteJumpToBeginAddressTest() [Category(TsavoriteKVTestCategory)] [Category(IteratorCategory)] [Category(SmokeTestCategory)] +#pragma warning disable IDE0060 // Remove unused parameter (hashMod is used by Setup) public void SpanByteIterationPendingCollisionTest([Values(HashModulo.Hundred)] HashModulo hashMod) +#pragma warning restore IDE0060 // Remove unused parameter { - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; IterationCollisionTestFunctions scanIteratorFunctions = new(); @@ -466,14 +468,13 @@ public void SpanByteIterationPendingCollisionTest([Values(HashModulo.Hundred)] H var start = store.Log.TailAddress; // Note: We only have a single value element; we are not exercising the "Variable Length" aspect here. - Span keySpan = stackalloc long[1], valueSpan = stackalloc long[1]; - SpanByte key = keySpan.AsSpanByte(), value = valueSpan.AsSpanByte(); + long key, value; // Initial population for (int ii = 0; ii < totalRecords; ii++) { - keySpan[0] = valueSpan[0] = ii; - _ = bContext.Upsert(ref key, ref value); + key = value = ii; + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); } // Evict so we can test the pending scan push diff --git a/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteTests.cs new file mode 100644 index 00000000000..022d68dd083 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteTests.cs @@ -0,0 +1,221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; + +namespace Tsavorite.test.spanbyte +{ + using SpanByteStoreFunctions = StoreFunctions; + [TestFixture] + internal class SpanByteTests : TestBase + { + [Test] + [Category("TsavoriteKV")] + [Category("Smoke")] + public unsafe void SpanByteTest1() + { + Span output = stackalloc byte[20]; + PinnedSpanByte input = default; + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + + try + { + using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "hlog1.log"), deleteOnClose: true); + using var store = new TsavoriteKV>( + new() + { + IndexSize = 1L << 13, + LogDevice = log, + LogMemorySize = 1L << 17, + PageSize = 1L << 12 + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + using var session = store.NewSession>(new SpanByteFunctions()); + var bContext = session.BasicContext; + + var key1 = MemoryMarshal.Cast("key1".AsSpan()); + var value1 = MemoryMarshal.Cast("value1".AsSpan()); + var output1 = SpanByteAndMemory.FromPinnedSpan(output); + + fixed (byte* key1Ptr = key1) + { + _ = bContext.Upsert(TestSpanByteKey.FromPointer(key1Ptr, key1.Length), value1); + _ = bContext.Read(TestSpanByteKey.FromPointer(key1Ptr, key1.Length), ref input, ref output1); + } + + ClassicAssert.IsTrue(output1.IsSpanByte); + ClassicAssert.IsTrue(output1.SpanByte.ReadOnlySpan.SequenceEqual(value1)); + + var key2 = MemoryMarshal.Cast("key2".AsSpan()); + var value2 = MemoryMarshal.Cast("value2value2value2".AsSpan()); + var output2 = SpanByteAndMemory.FromPinnedSpan(output); + + fixed (byte* key2Ptr = key2) + { + _ = bContext.Upsert(TestSpanByteKey.FromPointer(key2Ptr, key2.Length), value2); + _ = bContext.Read(TestSpanByteKey.FromPointer(key2Ptr, key2.Length), ref input, ref output2); + } + + ClassicAssert.IsTrue(!output2.IsSpanByte); + ClassicAssert.IsTrue(output2.Memory.Memory.Span.Slice(0, output2.Length).SequenceEqual(value2)); + output2.Memory.Dispose(); + } + finally + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + } + } + + [Test] + [Category("TsavoriteKV")] + [Category("Smoke")] + public unsafe void MultiRead_SpanByte_Test() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + + try + { + using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "test.log"), deleteOnClose: true); + using var store = new TsavoriteKV>( + new() + { + IndexSize = 1L << 16, + LogDevice = log, + LogMemorySize = 1L << 15, + PageSize = 1L << 12 + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + using var session = store.NewSession>(new SpanByteFunctions()); + var bContext = session.BasicContext; + + for (int i = 0; i < 200; i++) + { + var key = MemoryMarshal.Cast($"{i}".AsSpan()); + var value = MemoryMarshal.Cast($"{i + 1000}".AsSpan()); + fixed (byte* keyPtr = key) + { + _ = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); + } + } + + // Read, evict all records to disk, read again + MultiRead(evicted: false); + store.Log.FlushAndEvict(true); + MultiRead(evicted: true); + + void MultiRead(bool evicted) + { + for (long key = 0; key < 50; key++) + { + // read each key multiple times + for (int i = 0; i < 10; i++) + ReadKey(key, key + 1000, evicted); + } + } + + void ReadKey(long key, long value, bool evicted) + { + Status status; + SpanByteAndMemory output = default; + + var keySpan = MemoryMarshal.Cast($"{key}".AsSpan()); + + fixed (byte* keyPtr = keySpan) + { + status = bContext.Read(TestSpanByteKey.FromPointer(keyPtr, keySpan.Length), ref output); + ClassicAssert.AreEqual(evicted, status.IsPending, "evicted/pending mismatch"); + } + + if (evicted) + (status, output) = bContext.GetSinglePendingResult(); + ClassicAssert.IsTrue(status.Found, $"expected to find key; status = {status}, pending = {evicted}"); + + ClassicAssert.IsFalse(output.IsSpanByte, "Output should not have a valid SpanByte"); + var outputString = new string(MemoryMarshal.Cast(output.ReadOnlySpan)); + ClassicAssert.AreEqual(value, long.Parse(outputString), $"outputString mismatch; pending = {evicted}"); + output.Memory?.Dispose(); + } + } + finally + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + } + } + + [Test] + [Category("TsavoriteKV")] + public void ShouldSkipEmptySpaceAtEndOfPage() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + + using var log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "vl-iter.log"), deleteOnClose: true); + using var store = new TsavoriteKV>( + new() + { + IndexSize = 1L << 13, + LogDevice = log, + LogMemorySize = 1L << 17, + PageSize = 1L << 10 // 1KB page + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + using var session = store.NewSession(new VLVectorFunctions()); + var bContext = session.BasicContext; + + const int PageSize = 1024; + Span valueSpan = stackalloc byte[PageSize]; + + Set(1L, valueSpan, 800, 1); // Inserted on page#0 and leaves empty space + Set(2L, valueSpan, 800, 2); // Inserted on page#1 because there is not enough space in page#0, and leaves empty space + + // Add a second record on page#1 to fill it exactly. Page#1 starts at offset 0 on the page (unlike page#0, which starts at 24 or 64, + // depending on data). Subtract the RecordInfo and key space for both the first record and the second record we're about to insert, + // the value space for the first record, and the length header for the second record. This is the space available for the second record's value. + var availableSpaceForRecord3 = PageSize * 2 - store.Log.TailAddress; + var p2value2len = (int)availableSpaceForRecord3 + - RecordInfo.Size + - RecordDataHeader.MinHeaderBytes + - sizeof(long); // key size + Set(3L, valueSpan, p2value2len, 3); // Inserted on page#1 + ClassicAssert.AreEqual(PageSize * 2, store.Log.TailAddress, "TailAddress should be at the end of page#2"); + + Set(4L, valueSpan, 64, 4); // Inserted on page#2 + + var data = new List<(long, int, int)>(); + using (var iterator = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress)) + { + while (iterator.GetNext()) + { + var scanKey = iterator.Key.AsRef(); + var scanValue = iterator.ValueSpan; + + data.Add((scanKey, scanValue.Length, scanValue[0])); + } + } + + ClassicAssert.AreEqual(4, data.Count); + + ClassicAssert.AreEqual((1L, 800, 1), data[0]); + ClassicAssert.AreEqual((2L, 800, 2), data[1]); + ClassicAssert.AreEqual((3L, p2value2len, 3), data[2]); + ClassicAssert.AreEqual((4L, 64, 4), data[3]); + + TestUtils.DeleteDirectory(TestUtils.MethodTestDir); + + void Set(long keyValue, Span valueSpan, int valueLength, byte tag) + { + valueSpan[0] = tag; + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyValue)), valueSpan.Slice(0, valueLength), Empty.Default); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SpanByteVLVectorTests.cs b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteVLVectorTests.cs similarity index 72% rename from libs/storage/Tsavorite/cs/test/SpanByteVLVectorTests.cs rename to libs/storage/Tsavorite/cs/test/test.hlog/SpanByteVLVectorTests.cs index 9721347477b..1b69a70b5e3 100644 --- a/libs/storage/Tsavorite/cs/test/SpanByteVLVectorTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.hlog/SpanByteVLVectorTests.cs @@ -1,9 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; +using System.Runtime.InteropServices; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,11 +12,9 @@ namespace Tsavorite.test.spanbyte { - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] + using SpanByteStoreFunctions = StoreFunctions; [TestFixture] - internal class SpanByteVLVectorTests : AllureTestBase + internal class SpanByteVLVectorTests : TestBase { const int StackAllocMax = 12; @@ -25,22 +23,22 @@ internal class SpanByteVLVectorTests : AllureTestBase [Test] [Category(TsavoriteKVTestCategory)] [Category(SmokeTestCategory)] - public unsafe void VLVectorSingleKeyTest() + public void VLVectorSingleKeyTest() { DeleteDirectory(MethodTestDir, wait: true); var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog1.log"), deleteOnClose: true); - var store = new TsavoriteKV>( + var store = new TsavoriteKV>( new() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 17, + LogMemorySize = 1L << 17, PageSize = 1L << 12 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - var session = store.NewSession(new VLVectorFunctions()); + var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; // Single alloc outside the loop, to the max length we'll need. @@ -51,14 +49,11 @@ public unsafe void VLVectorSingleKeyTest() for (int i = 0; i < 5000; i++) { keySpan[0] = i; - var keySpanByte = keySpan.AsSpanByte(); - var len = GetRandomLength(rng); for (int j = 0; j < len; j++) valueSpan[j] = len; - var valueSpanByte = valueSpan.Slice(0, len).AsSpanByte(); - _ = bContext.Upsert(ref keySpanByte, ref valueSpanByte, Empty.Default); + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)), MemoryMarshal.Cast(valueSpan.Slice(0, len)), Empty.Default); } // Reset rng to get the same sequence of value lengths @@ -66,11 +61,10 @@ public unsafe void VLVectorSingleKeyTest() for (int i = 0; i < 5000; i++) { keySpan[0] = i; - var keySpanByte = keySpan.AsSpanByte(); var valueLen = GetRandomLength(rng); int[] output = null; - var status = bContext.Read(ref keySpanByte, ref output, Empty.Default); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)), ref output, Empty.Default); if (status.IsPending) { @@ -92,22 +86,22 @@ public unsafe void VLVectorSingleKeyTest() [Test] [Category(TsavoriteKVTestCategory)] [Category(SmokeTestCategory)] - public unsafe void VLVectorMultiKeyTest() + public void VLVectorMultiKeyTest() { DeleteDirectory(MethodTestDir, wait: true); var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog1.log"), deleteOnClose: true); - var store = new TsavoriteKV>( + var store = new TsavoriteKV>( new() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 17, + LogMemorySize = 1L << 17, PageSize = 1L << 12 - }, StoreFunctions.Create() + }, StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - var session = store.NewSession(new VLVectorFunctions()); + var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; // Single alloc outside the loop, to the max length we'll need. @@ -120,14 +114,12 @@ public unsafe void VLVectorMultiKeyTest() var keyLen = GetRandomLength(rng); for (int j = 0; j < keyLen; j++) keySpan[j] = i; - var keySpanByte = keySpan.AsSpanByte(); var valueLen = GetRandomLength(rng); for (int j = 0; j < valueLen; j++) valueSpan[j] = valueLen; - var valueSpanByte = valueSpan.Slice(0, valueLen).AsSpanByte(); - _ = bContext.Upsert(ref keySpanByte, ref valueSpanByte, Empty.Default); + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)), MemoryMarshal.Cast(valueSpan.Slice(0, valueLen)), Empty.Default); } // Reset rng to get the same sequence of key and value lengths @@ -137,11 +129,10 @@ public unsafe void VLVectorMultiKeyTest() var keyLen = GetRandomLength(rng); for (int j = 0; j < keyLen; j++) keySpan[j] = i; - var keySpanByte = keySpan.AsSpanByte(); var valueLen = GetRandomLength(rng); int[] output = null; - var status = bContext.Read(ref keySpanByte, ref output, Empty.Default); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(MemoryMarshal.Cast(keySpan)), ref output, Empty.Default); if (status.IsPending) { diff --git a/libs/storage/Tsavorite/cs/test/test.hlog/Tsavorite.test.hlog.csproj b/libs/storage/Tsavorite/cs/test/test.hlog/Tsavorite.test.hlog.csproj new file mode 100644 index 00000000000..493794ca137 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.hlog/Tsavorite.test.hlog.csproj @@ -0,0 +1,32 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + false + + + diff --git a/libs/storage/Tsavorite/cs/test/test.recordops/DeleteDisposeTests.cs b/libs/storage/Tsavorite/cs/test/test.recordops/DeleteDisposeTests.cs new file mode 100644 index 00000000000..31ae6e9fc66 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recordops/DeleteDisposeTests.cs @@ -0,0 +1,491 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.IO; +using System.Threading; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test +{ + using ObjTrackingAllocator = ObjectAllocator>; + using ObjTrackingStoreFunctions = StoreFunctions; + + using TrackingAllocator = SpanByteAllocator>; + using TrackingStoreFunctions = StoreFunctions; + [TestFixture] + internal class DeleteDisposeTests : TestBase + { + internal struct TrackingRecordTriggers : IRecordTriggers + { + internal readonly DisposeTracker tracker; + public TrackingRecordTriggers(DisposeTracker tracker) => this.tracker = tracker; + public readonly bool CallOnEvict => false; + public readonly bool CallOnFlush => false; + public readonly bool CallOnDiskRead => false; + public readonly void OnDispose(ref LogRecord logRecord, DisposeReason reason) => tracker?.RecordDispose(reason); + public readonly void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) { } + } + + internal class DisposeTracker + { + private int _deletedCount; + public int DeletedCount => _deletedCount; + public void RecordDispose(DisposeReason reason) { if (reason == DisposeReason.Deleted) Interlocked.Increment(ref _deletedCount); } + public void Reset() => Interlocked.Exchange(ref _deletedCount, 0); + } + + internal class ExpirableFunctions : SimpleIntSimpleFunctions + { + internal RMWAction expireAction = RMWAction.Default; + internal bool requestLargerReinit; + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref int input, ref int output, ref RMWInfo rmwInfo) + { + if (expireAction != RMWAction.Default) { rmwInfo.Action = expireAction; return false; } + return base.InPlaceUpdater(ref logRecord, ref input, ref output, ref rmwInfo); + } + + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref int input, ref int output, ref RMWInfo rmwInfo) + { + if (expireAction != RMWAction.Default) { rmwInfo.Action = expireAction; return false; } + return true; + } + + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref int input) + { + var info = base.GetRMWInitialFieldInfo(key, ref input); + // Request a larger value so TryReinitializeValueLength fails on the existing record + if (requestLargerReinit) + info.ValueSize = sizeof(int) * 4; + return info; + } + + public override unsafe bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref int input, ref int output, ref RMWInfo rmwInfo) + { + if (requestLargerReinit) + { + // Write input into the first int of the larger value span + dstLogRecord.ValueSpan.Clear(); + dstLogRecord.ValueSpan.AsRef() = input; + return true; + } + return base.InitialUpdater(ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); + } + } + + private TsavoriteKV store; + private IDevice log; + private DisposeTracker tracker; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "DeleteDisposeTests.log"), deleteOnClose: true); + tracker = new DisposeTracker(); + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + LogMemorySize = 1L << 15, + PageSize = 1L << 10 + }, StoreFunctions.Create(IntKeyComparer.Instance, new TrackingRecordTriggers(tracker)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + OnTearDown(); + } + + private void UpsertKey(int key, int value) + { + using var s = store.NewSession(new SimpleIntSimpleFunctions()); + _ = s.BasicContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); + } + + private static TestSpanByteKey Key(ref int k) => TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref k)); + + #region Delete + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnMutableDeleteTest() + { + UpsertKey(1, 100); + tracker.Reset(); + using var s = store.NewSession(new SimpleIntSimpleFunctions()); + var k = 1; + _ = s.BasicContext.Delete(Key(ref k)); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Mutable delete: expected exactly 1"); + } + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnImmutableDeleteTest() + { + UpsertKey(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + using var s = store.NewSession(new SimpleIntSimpleFunctions()); + var k = 1; + _ = s.BasicContext.Delete(Key(ref k)); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Immutable delete: expected exactly 1"); + } + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnImmutableDeleteMultipleTest() + { + const int n = 10; + for (int i = 0; i < n; i++) UpsertKey(i, i * 10); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + using var s = store.NewSession(new SimpleIntSimpleFunctions()); + for (int i = 0; i < n; i++) { var k = i; _ = s.BasicContext.Delete(Key(ref k)); } + ClassicAssert.AreEqual(n, tracker.DeletedCount, $"Immutable delete multiple: expected exactly {n}"); + } + + #endregion + + #region ExpireAndStop + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnMutableExpireAndStopTest() + { + UpsertKey(1, 100); + tracker.Reset(); + var fn = new ExpirableFunctions { expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var k = 1; var input = 0; + _ = s.BasicContext.RMW(Key(ref k), ref input); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Mutable ExpireAndStop: expected exactly 1"); + } + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnImmutableExpireAndStopTest() + { + UpsertKey(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + var fn = new ExpirableFunctions { expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var k = 1; var input = 0; + _ = s.BasicContext.RMW(Key(ref k), ref input); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Immutable ExpireAndStop: expected exactly 1"); + } + + #endregion + + #region ExpireAndResume + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnMutableExpireAndResumeFitsTest() + { + UpsertKey(1, 100); + tracker.Reset(); + // Same-sized value: ReinitializeExpiredRecord succeeds in-place + var fn = new ExpirableFunctions { expireAction = RMWAction.ExpireAndResume }; + using var s = store.NewSession(fn); + var k = 1; var input = 50; + _ = s.BasicContext.RMW(Key(ref k), ref input); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Mutable ExpireAndResume (fits): expected exactly 1"); + } + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnMutableExpireAndResumeDoesNotFitTest() + { + UpsertKey(1, 100); + tracker.Reset(); + // Request larger reinit (HasETag) so TryReinitializeValueLength fails, + // forcing fallback to CreateNewRecordRMW → InitialUpdater + var fn = new ExpirableFunctions { expireAction = RMWAction.ExpireAndResume, requestLargerReinit = true }; + using var s = store.NewSession(fn); + var k = 1; var input = 50; + _ = s.BasicContext.RMW(Key(ref k), ref input); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Mutable ExpireAndResume (does not fit): expected exactly 1"); + } + + [Test] + [Category("TsavoriteKV")] + public void DisposeOnImmutableExpireAndResumeTest() + { + UpsertKey(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + var fn = new ExpirableFunctions { expireAction = RMWAction.ExpireAndResume }; + using var s = store.NewSession(fn); + var k = 1; var input = 50; + _ = s.BasicContext.RMW(Key(ref k), ref input); + ClassicAssert.AreEqual(1, tracker.DeletedCount, "Immutable ExpireAndResume: expected exactly 1"); + } + + #endregion + } + + /// + /// Tests that is called exactly once for IHeapObject records through all delete and expiration paths. + /// + [TestFixture] + internal class ObjectDeleteDisposeTests : TestBase + { + internal struct ObjTrackingRecordTriggers : IRecordTriggers + { + internal readonly ObjDisposeTracker tracker; + public ObjTrackingRecordTriggers(ObjDisposeTracker tracker) => this.tracker = tracker; + public readonly bool CallOnEvict => false; + public readonly bool CallOnFlush => false; + public readonly bool CallOnDiskRead => false; + + public readonly void OnDispose(ref LogRecord logRecord, DisposeReason reason) + => tracker?.RecordDispose(reason); + + public readonly void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) { } + } + + internal class ObjDisposeTracker + { + private int _disposeRecordDeletedCount; + + public int DisposeRecordDeletedCount => _disposeRecordDeletedCount; + + public void RecordDispose(DisposeReason reason) + { + if (reason == DisposeReason.Deleted) + Interlocked.Increment(ref _disposeRecordDeletedCount); + } + + public void Reset() + { + Interlocked.Exchange(ref _disposeRecordDeletedCount, 0); + } + } + + internal class ObjExpirableFunctions : TestObjectFunctionsDelete + { + internal RMWAction expireAction = RMWAction.Default; + internal bool failIPU; + internal bool expireOnlyInCU; // If true, only CopyUpdater returns the expire action (not IPU/NCU) + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + if (!expireOnlyInCU && expireAction != RMWAction.Default) { rmwInfo.Action = expireAction; return false; } + if (failIPU) return false; + return base.InPlaceUpdater(ref logRecord, ref input, ref output, ref rmwInfo); + } + + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + if (!expireOnlyInCU && expireAction != RMWAction.Default) { rmwInfo.Action = expireAction; return false; } + return true; + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + if (expireAction != RMWAction.Default) { rmwInfo.Action = expireAction; return false; } + return base.CopyUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); + } + } + + private TsavoriteKV store; + private IDevice log, objlog; + private ObjDisposeTracker tracker; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjDeleteDisposeTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjDeleteDisposeTests.obj.log"), deleteOnClose: true); + tracker = new ObjDisposeTracker(); + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << 15, + PageSize = 1L << 10 + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), + new ObjTrackingRecordTriggers(tracker)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + objlog?.Dispose(); + objlog = null; + OnTearDown(); + } + + private void UpsertObj(int key, int value) + { + using var s = store.NewSession(new TestObjectFunctionsDelete()); + _ = s.BasicContext.Upsert(new TestObjectKey { key = key }, new TestObjectValue { value = value }, 0); + } + + #region Delete + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnMutableDeleteTest() + { + UpsertObj(1, 100); + tracker.Reset(); + using var s = store.NewSession(new TestObjectFunctionsDelete()); + _ = s.BasicContext.Delete(new TestObjectKey { key = 1 }); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnImmutableDeleteTest() + { + UpsertObj(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + using var s = store.NewSession(new TestObjectFunctionsDelete()); + _ = s.BasicContext.Delete(new TestObjectKey { key = 1 }); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnImmutableDeleteMultipleTest() + { + const int n = 10; + for (int i = 0; i < n; i++) UpsertObj(i, i * 10); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + using var s = store.NewSession(new TestObjectFunctionsDelete()); + for (int i = 0; i < n; i++) _ = s.BasicContext.Delete(new TestObjectKey { key = i }); + ClassicAssert.AreEqual(n, tracker.DisposeRecordDeletedCount, $"OnDispose(Deleted) should be called exactly {n} times"); + } + + #endregion + + #region ExpireAndStop + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnMutableExpireAndStopTest() + { + UpsertObj(1, 100); + tracker.Reset(); + var fn = new ObjExpirableFunctions { expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 0 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnImmutableExpireAndStopTest() + { + UpsertObj(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + var fn = new ObjExpirableFunctions { expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 0 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + #endregion + + #region ExpireAndResume + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnMutableExpireAndResumeTest() + { + UpsertObj(1, 100); + tracker.Reset(); + var fn = new ObjExpirableFunctions { expireAction = RMWAction.ExpireAndResume }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 50 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnImmutableExpireAndResumeTest() + { + UpsertObj(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + var fn = new ObjExpirableFunctions { expireAction = RMWAction.ExpireAndResume }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 50 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + #endregion + + #region IPU fails → CopyUpdater expires (simulates insufficient space forcing CU path) + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnIPUFailThenNCUExpireAndStopTest() + { + UpsertObj(1, 100); + tracker.Reset(); + var fn = new ObjExpirableFunctions { failIPU = true, expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 0 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnIPUFailThenNCUExpireAndResumeTest() + { + UpsertObj(1, 100); + tracker.Reset(); + var fn = new ObjExpirableFunctions { failIPU = true, expireAction = RMWAction.ExpireAndResume }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 50 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + [Test] + [Category("TsavoriteKV")] + public void ObjDisposeOnIPUFailThenCUExpireAndStopTest() + { + // IPU fails → NCU passes → CopyUpdater returns ExpireAndStop → source disposed + UpsertObj(1, 100); + tracker.Reset(); + var fn = new ObjExpirableFunctions { failIPU = true, expireOnlyInCU = true, expireAction = RMWAction.ExpireAndStop }; + using var s = store.NewSession(fn); + var input = new TestObjectInput { value = 0 }; + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, 0); + ClassicAssert.AreEqual(1, tracker.DisposeRecordDeletedCount, "OnDispose(Deleted) should be called exactly once"); + } + + #endregion + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ModifiedBitTests.cs b/libs/storage/Tsavorite/cs/test/test.recordops/ModifiedBitTests.cs similarity index 59% rename from libs/storage/Tsavorite/cs/test/ModifiedBitTests.cs rename to libs/storage/Tsavorite/cs/test/test.recordops/ModifiedBitTests.cs index 6ca943d0fd8..2888ef1332e 100644 --- a/libs/storage/Tsavorite/cs/test/ModifiedBitTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recordops/ModifiedBitTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,49 +12,36 @@ namespace Tsavorite.test.ModifiedBit { - // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - internal struct ModifiedBitTestComparer : IKeyComparer - { - public readonly bool Equals(ref int k1, ref int k2) => k1 == k2; - - public readonly long GetHashCode64(ref int k) => Utility.GetHashCode(k); - } -} - -namespace Tsavorite.test.ModifiedBit -{ - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using IntAllocator = SpanByteAllocator>; + using IntStoreFunctions = StoreFunctions; [TestFixture] - class ModifiedBitTests : AllureTestBase + class ModifiedBitTests : TestBase { const int NumRecords = 1000; const int ValueMult = 1_000_000; - ModifiedBitTestComparer comparer; + IntKeyComparer comparer; - private TsavoriteKV store; - private ClientSession, IntStoreFunctions, IntAllocator> session; - private BasicContext, IntStoreFunctions, IntAllocator> bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; [SetUp] public void Setup() { log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.log"), deleteOnClose: false); - comparer = new ModifiedBitTestComparer(); + comparer = IntKeyComparer.Instance; store = new(new() { IndexSize = 1L << 26, LogDevice = log, PageSize = 1L << 12, - MemorySize = 1L << 22 - }, StoreFunctions.Create(comparer) + LogMemorySize = 1L << 22 + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession>(new SimpleSimpleFunctions()); + session = store.NewSession(new SimpleIntSimpleFunctions()); bContext = session.BasicContext; } @@ -74,30 +60,33 @@ public void TearDown() void Populate() { for (int key = 0; key < NumRecords; key++) - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + { + var value = key * ValueMult; + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)).IsPending); + } } - void AssertLockandModified(LockableUnsafeContext, IntStoreFunctions, IntAllocator> luContext, int key, bool xlock, bool slock, bool modified = false) + void AssertLockandModified(TransactionalUnsafeContext luContext, int key, bool xlock, bool slock, bool modified = false) { - OverflowBucketLockTableTests.AssertLockCounts(store, ref key, xlock, slock); - var isM = luContext.IsModified(key); + OverflowBucketLockTableTests.AssertLockCounts(store, SpanByte.FromPinnedVariable(ref key), xlock, slock); + var isM = luContext.IsModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(modified, isM, "modified mismatch"); } - void AssertLockandModified(LockableContext, IntStoreFunctions, IntAllocator> luContext, int key, bool xlock, bool slock, bool modified = false) + void AssertLockandModified(TransactionalContext luContext, int key, bool xlock, bool slock, bool modified = false) { - OverflowBucketLockTableTests.AssertLockCounts(store, ref key, xlock, slock); - var isM = luContext.IsModified(key); + OverflowBucketLockTableTests.AssertLockCounts(store, SpanByte.FromPinnedVariable(ref key), xlock, slock); + var isM = luContext.IsModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(modified, isM, "modified mismatch"); } - void AssertLockandModified(ClientSession, IntStoreFunctions, IntAllocator> session, int key, bool xlock, bool slock, bool modified = false) + void AssertLockandModified(ClientSession intSession, int key, bool xlock, bool slock, bool modified = false) { - var luContext = session.LockableUnsafeContext; + var luContext = intSession.TransactionalUnsafeContext; luContext.BeginUnsafe(); - OverflowBucketLockTableTests.AssertLockCounts(store, ref key, xlock, slock); - var isM = luContext.IsModified(key); + OverflowBucketLockTableTests.AssertLockCounts(store, SpanByte.FromPinnedVariable(ref key), xlock, slock); + var isM = luContext.IsModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(modified, isM, "Modified mismatch"); luContext.EndUnsafe(); @@ -110,28 +99,28 @@ public void LockAndNotModify() Populate(); Random r = new(100); int key = r.Next(NumRecords); - bContext.ResetModified(key); + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); - var lContext = session.LockableContext; - lContext.BeginLockable(); + var lContext = session.TransactionalContext; + lContext.BeginTransaction(); AssertLockandModified(lContext, key, xlock: false, slock: false, modified: false); - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Exclusive, lContext) }; + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key), LockType.Exclusive, lContext) }; - lContext.Lock>(keyVec); + lContext.Lock(keyVec); AssertLockandModified(lContext, key, xlock: true, slock: false, modified: false); - lContext.Unlock>(keyVec); + lContext.Unlock(keyVec); AssertLockandModified(lContext, key, xlock: false, slock: false, modified: false); keyVec[0].LockType = LockType.Shared; - lContext.Lock>(keyVec); + lContext.Lock(keyVec); AssertLockandModified(lContext, key, xlock: false, slock: true, modified: false); - lContext.Unlock>(keyVec); + lContext.Unlock(keyVec); AssertLockandModified(lContext, key, xlock: false, slock: false, modified: false); - lContext.EndLockable(); + lContext.EndTransaction(); } [Test] @@ -140,7 +129,7 @@ public void ResetModifyForNonExistingKey() { Populate(); int key = NumRecords + 100; - bContext.ResetModified(key); + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); AssertLockandModified(session, key, xlock: false, slock: false, modified: false); } @@ -152,7 +141,7 @@ public void ModifyClientSession([Values(true, false)] bool flushToDisk, [Values] int key = NumRecords - 500; int value = 14; - bContext.ResetModified(key); + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); AssertLockandModified(session, key, xlock: false, slock: false, modified: false); if (flushToDisk) @@ -162,13 +151,13 @@ public void ModifyClientSession([Values(true, false)] bool flushToDisk, [Values] switch (updateOp) { case UpdateOp.Upsert: - status = bContext.Upsert(key, value); + status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); break; case UpdateOp.RMW: - status = bContext.RMW(key, value); + status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); break; case UpdateOp.Delete: - status = bContext.Delete(key); + status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); break; default: break; @@ -185,7 +174,7 @@ public void ModifyClientSession([Values(true, false)] bool flushToDisk, [Values] ClassicAssert.IsTrue(status.NotFound); break; } - (status, _) = bContext.Read(key); + (status, _) = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.IsTrue(status.Found || updateOp == UpdateOp.Delete); } @@ -203,12 +192,12 @@ public void ModifyLUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp int key = NumRecords - 500; int value = 14; - bContext.ResetModified(key); - var luContext = session.LockableUnsafeContext; + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); AssertLockandModified(luContext, key, xlock: false, slock: false, modified: false); - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); if (flushToDisk) @@ -217,22 +206,22 @@ public void ModifyLUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp Status status = default; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Exclusive, luContext) }; + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key), LockType.Exclusive, luContext) }; - luContext.Lock>(keyVec); + luContext.Lock(keyVec); switch (updateOp) { case UpdateOp.Upsert: - status = luContext.Upsert(key, value); + status = luContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); break; case UpdateOp.RMW: - status = luContext.RMW(key, value); + status = luContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); break; case UpdateOp.Delete: - status = luContext.Delete(key); + status = luContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); break; default: break; @@ -252,20 +241,20 @@ public void ModifyLUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp } } - luContext.Unlock>(keyVec); + luContext.Unlock(keyVec); if (flushToDisk) { keyVec[0].LockType = LockType.Shared; - luContext.Lock>(keyVec); - (status, _) = luContext.Read(key); + luContext.Lock(keyVec); + (status, _) = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(updateOp != UpdateOp.Delete, status.Found, status.ToString()); - luContext.Unlock>(keyVec); + luContext.Unlock(keyVec); } AssertLockandModified(luContext, key, xlock: false, slock: false, modified: updateOp != UpdateOp.Delete); - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } @@ -277,7 +266,7 @@ public void ModifyUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u int key = NumRecords - 500; int value = 14; - bContext.ResetModified(key); + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); AssertLockandModified(session, key, xlock: false, slock: false, modified: false); if (flushToDisk) @@ -290,13 +279,13 @@ public void ModifyUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u switch (updateOp) { case UpdateOp.Upsert: - status = unsafeContext.Upsert(key, value); + status = unsafeContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); break; case UpdateOp.RMW: - status = unsafeContext.RMW(key, value); + status = unsafeContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); break; case UpdateOp.Delete: - status = unsafeContext.Delete(key); + status = unsafeContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); break; default: break; @@ -313,7 +302,7 @@ public void ModifyUC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u ClassicAssert.IsTrue(status.NotFound); break; } - (status, _) = unsafeContext.Read(key); + (status, _) = unsafeContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.IsTrue(status.Found || updateOp == UpdateOp.Delete); } unsafeContext.EndUnsafe(); @@ -329,14 +318,14 @@ public void ModifyLC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u int key = NumRecords - 500; int value = 14; - bContext.ResetModified(key); - var lContext = session.LockableContext; - lContext.BeginLockable(); + bContext.ResetModified(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); + var lContext = session.TransactionalContext; + lContext.BeginTransaction(); AssertLockandModified(lContext, key, xlock: false, slock: false, modified: false); - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Exclusive, lContext) }; + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key), LockType.Exclusive, lContext) }; - lContext.Lock>(keyVec); + lContext.Lock(keyVec); if (flushToDisk) store.Log.FlushAndEvict(wait: true); @@ -346,13 +335,13 @@ public void ModifyLC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u switch (updateOp) { case UpdateOp.Upsert: - status = lContext.Upsert(key, value); + status = lContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); break; case UpdateOp.RMW: - status = lContext.RMW(key, value); + status = lContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value); break; case UpdateOp.Delete: - status = lContext.Delete(key); + status = lContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); break; default: break; @@ -372,19 +361,19 @@ public void ModifyLC([Values(true, false)] bool flushToDisk, [Values] UpdateOp u } } - lContext.Unlock>(keyVec); + lContext.Unlock(keyVec); if (flushToDisk) { keyVec[0].LockType = LockType.Shared; - lContext.Lock>(keyVec); - (status, _) = lContext.Read(key); + lContext.Lock(keyVec); + (status, _) = lContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))); ClassicAssert.AreEqual(updateOp != UpdateOp.Delete, status.Found, status.ToString()); - lContext.Unlock>(keyVec); + lContext.Unlock(keyVec); } AssertLockandModified(lContext, key, xlock: false, slock: false, modified: updateOp != UpdateOp.Delete); - lContext.EndLockable(); + lContext.EndTransaction(); } [Test] @@ -394,40 +383,40 @@ public void CopyToTailTest() Populate(); store.Log.FlushAndEvict(wait: true); - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; int input = 0, output = 0, key = 200; ReadOptions readOptions = new() { CopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog) }; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); AssertLockandModified(luContext, key, xlock: false, slock: false, modified: true); - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Shared, luContext) }; + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key), LockType.Shared, luContext) }; - luContext.Lock>(keyVec); + luContext.Lock(keyVec); AssertLockandModified(luContext, key, xlock: false, slock: true, modified: true); // Check Read Copy to Tail resets the modified - var status = luContext.Read(ref key, ref input, ref output, ref readOptions, out _); + var status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = luContext.CompletePending(wait: true); - luContext.Unlock>(keyVec); - AssertLockandModified(luContext, key, xlock: false, slock: false, modified: true); + luContext.Unlock(keyVec); + AssertLockandModified(luContext, key, xlock: false, slock: false, modified: false); // Check Read Copy to Tail resets the modified on locked key key += 10; - keyVec[0] = new(key, LockType.Exclusive, luContext); - luContext.Lock>(keyVec); - status = luContext.Read(ref key, ref input, ref output, ref readOptions, out _); + keyVec[0] = new(SpanByte.FromPinnedVariable(ref key), LockType.Exclusive, luContext); + luContext.Lock(keyVec); + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = luContext.CompletePending(wait: true); - AssertLockandModified(luContext, key, xlock: true, slock: false, modified: true); - luContext.Unlock>(keyVec); - AssertLockandModified(luContext, key, xlock: false, slock: false, modified: true); + AssertLockandModified(luContext, key, xlock: true, slock: false, modified: false); + luContext.Unlock(keyVec); + AssertLockandModified(luContext, key, xlock: false, slock: false, modified: false); - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } diff --git a/libs/storage/Tsavorite/cs/test/test.recordops/RecordLifecycleTests.cs b/libs/storage/Tsavorite/cs/test/test.recordops/RecordLifecycleTests.cs new file mode 100644 index 00000000000..d9feedb0ca7 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recordops/RecordLifecycleTests.cs @@ -0,0 +1,699 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Concurrent; +using System.IO; +using System.Threading; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test +{ + using LifecycleAllocator = ObjectAllocator>; + using LifecycleStoreFunctions = StoreFunctions; + + /// + /// Lifecycle tests for on the object allocator. These tests go beyond + /// counting calls — they assert the EXACT number of calls to + /// (per reason), + /// (per reason), and (per source), and they track the number + /// of times IHeapObject.Dispose is invoked on individual value objects. This protects + /// against double-dispose, leaked triggers, and asymmetric DiskLogRecord disposal. + /// + [TestFixture] + internal class RecordLifecycleTests : TestBase + { + /// + /// Per-instance dispose-counting heap object. Subclasses so existing + /// casts succeed; overrides Dispose() to increment a counter + /// and registers itself in a static registry so tests can verify that no value object is ever + /// disposed more than once across its entire lifecycle. + /// + internal class TrackedObjectValue : TestObjectValue + { + private static int _nextId; + private static readonly ConcurrentDictionary Registry = new(); + + public int Id { get; } + public int DisposeCount; + + public TrackedObjectValue() + { + Id = Interlocked.Increment(ref _nextId); + _ = Registry.TryAdd(Id, this); + } + + public override void Dispose() => _ = Interlocked.Increment(ref DisposeCount); + + public override HeapObjectBase Clone() => new TrackedObjectValue { value = value }; + + public override string ToString() => $"TrackedObjectValue(Id={Id}, Value={value}, Disposes={DisposeCount})"; + + /// Returns (totalDisposes, maxDisposesPerInstance) across every live tracked object. + public static (int total, int max) Snapshot() + { + int total = 0, max = 0; + foreach (var kv in Registry) + { + var d = Volatile.Read(ref kv.Value.DisposeCount); + total += d; + if (d > max) max = d; + } + return (total, max); + } + + public static void Clear() => Registry.Clear(); + + public new class Serializer : BinaryObjectSerializer + { + public override void Deserialize(out IHeapObject obj) => obj = new TrackedObjectValue { value = reader.ReadInt32() }; + public override void Serialize(IHeapObject obj) => writer.Write(((TrackedObjectValue)obj).value); + } + } + + /// + /// Rich lifecycle counter. Separates counts by and by + /// so tests can assert precise call patterns. + /// Optionally invokes IHeapObject.Dispose from the trigger — used to + /// verify that handler-driven disposal flows correctly (and that Tsavorite itself + /// never calls Dispose behind the handler's back). + /// + internal class LifecycleTracker + { + // OnDispose(reason) + public readonly int[] DisposeCounts = new int[Enum.GetValues().Length]; + // OnDisposeDiskRecord(reason) + public readonly int[] DisposeDiskCounts = new int[Enum.GetValues().Length]; + // OnEvict(source) + public readonly int[] EvictCounts = new int[Enum.GetValues().Length]; + + /// If true, invokes IHeapObject.Dispose + /// on and only. + public bool DisposeValuesOnDispose; + + /// If true, invokes IHeapObject.Dispose. + /// NOTE: Unsafe for scan-iterator wrappers that share a value-object reference with the on-log record; + /// tests that use this flag must avoid in-memory scans or use it only for known-owned disk records. + public bool DisposeValuesOnDisposeDiskRecord; + + // Gating flags read by LifecycleRecordTriggers. Tests toggle these to verify that + // Tsavorite only walks the corresponding per-record paths when the application opts in. + public bool CallOnFlushFlag; + public bool CallOnDiskReadFlag; + // CallOnEvict flag. Default true preserves the behaviour expected by the + // original tests; the gating test sets it to false. + public bool CallOnEvictFlag = true; + + // OnFlush(record) count and OnDiskRead(record) count + public int FlushCount; + public int DiskReadCount; + + public int DisposeCount(DisposeReason r) => Volatile.Read(ref DisposeCounts[(int)r]); + public int DisposeDiskCount(DisposeReason r) => Volatile.Read(ref DisposeDiskCounts[(int)r]); + public int EvictCount(EvictionSource s) => Volatile.Read(ref EvictCounts[(int)s]); + public int TotalDispose() { int sum = 0; foreach (var v in DisposeCounts) sum += v; return sum; } + public int TotalDisposeDisk() { int sum = 0; foreach (var v in DisposeDiskCounts) sum += v; return sum; } + public int TotalEvict() { int sum = 0; foreach (var v in EvictCounts) sum += v; return sum; } + + public void Reset() + { + for (int i = 0; i < DisposeCounts.Length; i++) DisposeCounts[i] = 0; + for (int i = 0; i < DisposeDiskCounts.Length; i++) DisposeDiskCounts[i] = 0; + for (int i = 0; i < EvictCounts.Length; i++) EvictCounts[i] = 0; + FlushCount = 0; + DiskReadCount = 0; + } + } + + internal struct LifecycleRecordTriggers : IRecordTriggers + { + internal readonly LifecycleTracker tracker; + public LifecycleRecordTriggers(LifecycleTracker tracker) => this.tracker = tracker; + + public readonly bool CallOnFlush => tracker?.CallOnFlushFlag ?? false; + public readonly bool CallOnDiskRead => tracker?.CallOnDiskReadFlag ?? false; + public readonly bool CallOnEvict => tracker?.CallOnEvictFlag ?? false; + + public readonly void OnFlush(ref LogRecord logRecord, long logicalAddress) + { + if (tracker is null) return; + _ = Interlocked.Increment(ref tracker.FlushCount); + } + + public readonly void OnDiskRead(ref LogRecord logRecord) + { + if (tracker is null) return; + _ = Interlocked.Increment(ref tracker.DiskReadCount); + } + + public readonly void OnDispose(ref LogRecord logRecord, DisposeReason reason) + { + if (tracker is null) return; + _ = Interlocked.Increment(ref tracker.DisposeCounts[(int)reason]); + + if (tracker.DisposeValuesOnDispose + && logRecord.Info.ValueIsObject + && (reason == DisposeReason.Deleted || reason == DisposeReason.CopyUpdated)) + { + logRecord.ValueObject?.Dispose(); + } + } + + public readonly void OnDisposeDiskRecord(ref DiskLogRecord logRecord, DisposeReason reason) + { + if (tracker is null) return; + // Defensive-no-op calls from AsyncIOContextCompletionEvent pass a default (unset) DiskLogRecord; + // filter them out so call-count assertions reflect only records that actually held data. + if (!logRecord.IsSet) return; + _ = Interlocked.Increment(ref tracker.DisposeDiskCounts[(int)reason]); + + if (tracker.DisposeValuesOnDisposeDiskRecord && logRecord.Info.ValueIsObject) + logRecord.ValueObject?.Dispose(); + } + + public readonly void OnEvict(ref LogRecord logRecord, EvictionSource source) + { + if (tracker is null) return; + _ = Interlocked.Increment(ref tracker.EvictCounts[(int)source]); + } + } + + private TsavoriteKV store; + private IDevice log, objlog; + private LifecycleTracker tracker; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + TrackedObjectValue.Clear(); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "RecordLifecycleTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "RecordLifecycleTests.obj.log"), deleteOnClose: true); + tracker = new LifecycleTracker(); + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << 15, + PageSize = 1L << 10, + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TrackedObjectValue.Serializer(), + new LifecycleRecordTriggers(tracker)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); store = null; + log?.Dispose(); log = null; + objlog?.Dispose(); objlog = null; + TrackedObjectValue.Clear(); + OnTearDown(); + } + + private ClientSession NewSession() + => store.NewSession(new TestObjectFunctionsDelete()); + + private void Upsert(int key, int value) + { + using var s = NewSession(); + _ = s.BasicContext.Upsert(new TestObjectKey { key = key }, new TrackedObjectValue { value = value }, 0); + } + + #region CopyUpdate — value-object slot clearing + + /// + /// RMW on a record in the immutable region forces CopyUpdate. After the CAS succeeds, + /// Tsavorite internally clears the source value-object slot and decrements the logSizeTracker + /// for the value-object's heap. The trigger is NOT involved — + /// does not fire via . The source record stays alive + /// (sealed) until eviction, where OnEvict picks up any remaining key overflow. + /// + [Test, Category("TsavoriteKV")] + public void CopyUpdateDoesNotFireOnDisposeCopyUpdated() + { + Upsert(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + + using (var s = NewSession()) + { + var input = new TestObjectInput { value = 7 }; + var output = new TestObjectOutput(); + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, ref output, 0); + } + + ClassicAssert.AreEqual(0, tracker.DisposeCount(DisposeReason.CopyUpdated), + "CopyUpdated is handled internally by logSizeTracker — OnDispose must not fire for it"); + ClassicAssert.AreEqual(0, tracker.DisposeCount(DisposeReason.Deleted), + "Deleted must not fire on a CopyUpdate path"); + ClassicAssert.AreEqual(0, tracker.TotalEvict(), + "No page eviction should have happened in this test window"); + } + + /// + /// After CopyUpdate, the source value-object is cleared from the ObjectIdMap (via ClearValueIfHeap). + /// Tsavorite calls IHeapObject.Dispose on the freed object internally — the trigger is not involved. + /// Verify that exactly one IHeapObject.Dispose fires for the source value object. + /// + [Test, Category("TsavoriteKV")] + public void CopyUpdateDisposesSourceValueExactlyOnce() + { + Upsert(1, 100); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + tracker.Reset(); + + using (var s = NewSession()) + { + var input = new TestObjectInput { value = 7 }; + var output = new TestObjectOutput(); + _ = s.BasicContext.RMW(new TestObjectKey { key = 1 }, ref input, ref output, 0); + } + + ClassicAssert.AreEqual(0, tracker.DisposeCount(DisposeReason.CopyUpdated), + "CopyUpdated must not fire — handled internally"); + + var (total, max) = TrackedObjectValue.Snapshot(); + ClassicAssert.AreEqual(1, total, "Exactly one IHeapObject.Dispose call expected (the CU source object)"); + ClassicAssert.AreEqual(1, max, "No value object should be disposed more than once"); + } + + #endregion + + #region Delete + + [Test, Category("TsavoriteKV")] + public void DeleteFiresOnDisposeDeletedAndNoOnDisposeDiskRecord() + { + Upsert(1, 100); + tracker.Reset(); + + using (var s = NewSession()) + _ = s.BasicContext.Delete(new TestObjectKey { key = 1 }); + + ClassicAssert.AreEqual(1, tracker.DisposeCount(DisposeReason.Deleted), + "OnDispose(Deleted) should fire exactly once"); + ClassicAssert.AreEqual(0, tracker.TotalDisposeDisk(), + "Delete should not invoke OnDisposeDiskRecord for any reason"); + } + + #endregion + + #region Scan — in-memory records + + /// + /// Scanning N records that are all in memory fires + /// exactly N times with (the iterator wraps each + /// remapped in-memory record as a transient DiskLogRecord). Value objects are SHARED with the + /// live on-log records, so the default trigger (no-op) must not dispose them. + /// + [Test, Category("TsavoriteKV")] + public void ScanInMemoryFiresOnDisposeDiskRecordOncePerRecord() + { + const int n = 20; + for (int i = 0; i < n; i++) Upsert(i, i * 10); + tracker.Reset(); + + int scanned = 0; + using (var iter = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress, DiskScanBufferingMode.SinglePageBuffering)) + { + while (iter.GetNext()) scanned++; + } + + ClassicAssert.AreEqual(n, scanned, "Iterator should yield exactly N in-memory records"); + ClassicAssert.AreEqual(n, tracker.DisposeDiskCount(DisposeReason.DeserializedFromDisk), + "OnDisposeDiskRecord should fire exactly once per scanned in-memory record"); + ClassicAssert.AreEqual(0, tracker.TotalDispose(), + "In-memory scan must not invoke OnDispose (on-log records are not being disposed)"); + + var (total, _) = TrackedObjectValue.Snapshot(); + ClassicAssert.AreEqual(0, total, + "In-memory scan with a no-op OnDisposeDiskRecord handler must leave value objects undisposed " + + "(they are shared references to the still-live on-log records)"); + + // The on-log records remain fully readable after the scan — no corruption from the scan wrappers. + using (var s = NewSession()) + { + for (int i = 0; i < n; i++) + { + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + var status = s.BasicContext.Read(new TestObjectKey { key = i }, ref input, ref output, 0); + ClassicAssert.IsTrue(status.Found && !status.IsPending, $"Read {i} should hit memory"); + ClassicAssert.AreEqual(i * 10, output.value.value, $"Record {i} value should be intact after scan"); + } + } + } + + #endregion + + #region Scan — records on disk + + /// + /// After FlushAndEvict pushes all records to disk, scanning K records fires + /// exactly K times. These are genuinely + /// disk-deserialized records that OWN their value objects, so a handler that opts to Dispose + /// will see exactly K IHeapObject.Dispose invocations (one per iterated record) + /// with no double-dispose. + /// + [Test, Category("TsavoriteKV")] + public void ScanDiskFiresOnDisposeDiskRecordOncePerRecord() + { + const int n = 40; + for (int i = 0; i < n; i++) Upsert(i, i * 10); + + store.Log.FlushAndEvict(wait: true); + tracker.Reset(); + tracker.DisposeValuesOnDisposeDiskRecord = true; // disk-deserialized records own their values; safe to dispose + + // Snapshot existing disposes so we only count new ones from this scan. + var (beforeTotal, _) = TrackedObjectValue.Snapshot(); + + int scanned = 0; + using (var iter = store.Log.Scan(store.Log.BeginAddress, store.Log.TailAddress, DiskScanBufferingMode.SinglePageBuffering)) + { + while (iter.GetNext()) scanned++; + } + + ClassicAssert.AreEqual(n, scanned, "Iterator should yield exactly N disk-resident records"); + ClassicAssert.AreEqual(n, tracker.DisposeDiskCount(DisposeReason.DeserializedFromDisk), + "OnDisposeDiskRecord should fire exactly once per scanned disk record"); + ClassicAssert.AreEqual(n, tracker.TotalDisposeDisk(), + "Only DeserializedFromDisk reason should fire — no other DisposeReason should appear for a disk scan"); + ClassicAssert.AreEqual(0, tracker.TotalDispose(), + "Disk scan must not invoke OnDispose (there are no on-log records being disposed)"); + ClassicAssert.AreEqual(0, tracker.TotalEvict(), + "Scan is not eviction — OnEvict must not fire"); + + var (afterTotal, afterMax) = TrackedObjectValue.Snapshot(); + var newDisposes = afterTotal - beforeTotal; + ClassicAssert.AreEqual(n, newDisposes, + "Handler opts to Dispose each disk value — expected exactly N new IHeapObject.Dispose calls"); + ClassicAssert.LessOrEqual(afterMax, 1, "No individual value object should be disposed more than once"); + } + + #endregion + + #region Pending read from disk + + /// + /// Reading a key whose record is on disk issues a pending IO; on completion, + /// fires exactly once with + /// . If the handler opts in, + /// the deserialized value is disposed exactly once. + /// + [Test, Category("TsavoriteKV")] + public void PendingReadFromDiskFiresOnDisposeDiskRecordOnce() + { + const int key = 42; + // Value matches key so TestObjectFunctionsDelete.ReadCompletionCallback's key==value + // assertion (ObjectTestTypes.cs:192) succeeds on the pending-read path. + Upsert(key, 42); + store.Log.FlushAndEvict(wait: true); + + tracker.Reset(); + tracker.DisposeValuesOnDisposeDiskRecord = true; + var (beforeTotal, _) = TrackedObjectValue.Snapshot(); + + using (var s = NewSession()) + { + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + var status = s.BasicContext.Read(new TestObjectKey { key = key }, ref input, ref output, 0); + ClassicAssert.IsTrue(status.IsPending, "Record should be disk-resident and Read should go pending"); + _ = s.BasicContext.CompletePending(wait: true); + } + + ClassicAssert.AreEqual(1, tracker.DisposeDiskCount(DisposeReason.DeserializedFromDisk), + "Exactly one OnDisposeDiskRecord(DeserializedFromDisk) call expected per pending IO"); + ClassicAssert.AreEqual(1, tracker.TotalDisposeDisk(), + "Only DeserializedFromDisk reason should fire — no other DisposeReason"); + ClassicAssert.AreEqual(0, tracker.TotalDispose(), + "Pending read must not invoke OnDispose (no on-log records are being disposed)"); + ClassicAssert.AreEqual(0, tracker.TotalEvict(), + "Pending read must not invoke OnEvict"); + + var (afterTotal, afterMax) = TrackedObjectValue.Snapshot(); + ClassicAssert.AreEqual(1, afterTotal - beforeTotal, + "Exactly one IHeapObject.Dispose call expected (the pending IO's deserialized value)"); + ClassicAssert.LessOrEqual(afterMax, 1, "No double-dispose"); + } + + #endregion + + #region OnEvict + + /// + /// Filling the log well beyond its mutable window forces page eviction. OnEvict must fire for + /// every non-tombstoned, non-invalid record evicted past HeadAddress — including sealed source + /// records from immutable-region deletes. Tombstoned records are skipped (heap was decremented + /// at the delete site). Invalid/elided records are skipped (already cleaned up). + /// + [Test, Category("TsavoriteKV")] + public void PageEvictionFiresOnEvictForEveryLiveRecord() + { + const int n = 500; // enough to push pages past HeadAddress given 32KB memory / 1KB pages + for (int i = 0; i < n; i++) Upsert(i, i); + + // Delete ~10% of records. Track which go through the immutable path (TailAddress moves) + // vs in-place mutable path (TailAddress stays). + tracker.Reset(); + const int deleted = 50; + int mutableDeletes = 0; + using (var s = NewSession()) + { + for (int i = 0; i < deleted; i++) + { + var tailBefore = store.Log.TailAddress; + _ = s.BasicContext.Delete(new TestObjectKey { key = i }); + if (store.Log.TailAddress == tailBefore) + mutableDeletes++; + } + } + var immutableDeletes = deleted - mutableDeletes; + + ClassicAssert.AreEqual(deleted, tracker.DisposeCount(DisposeReason.Deleted), + "Each Delete should fire OnDispose(Deleted) exactly once"); + var deletedDisposeCountBeforeEvict = tracker.DisposeCount(DisposeReason.Deleted); + + // Force all records out to disk. + store.Log.FlushAndEvict(wait: true); + + // Precise count: + // - (n - deleted) live records: visited by OnEvict. + // - mutableDeletes records: tombstoned in-place, skipped by OnEvict. + // - immutableDeletes sealed source records: NOT tombstoned, visited by OnEvict. + // - immutableDeletes new tombstone records at tail: tombstoned, skipped by OnEvict. + // Total = (n - deleted) + immutableDeletes = n - mutableDeletes. + ClassicAssert.AreEqual(n - mutableDeletes, tracker.EvictCount(EvictionSource.MainLog), + $"OnEvict(MainLog) must fire exactly {n - mutableDeletes} times: " + + $"{n - deleted} live + {immutableDeletes} sealed sources, skipping {mutableDeletes} in-place tombstones"); + ClassicAssert.AreEqual(0, tracker.EvictCount(EvictionSource.ReadCache), + "No read cache is configured, OnEvict(ReadCache) must never fire"); + ClassicAssert.AreEqual(deletedDisposeCountBeforeEvict, tracker.DisposeCount(DisposeReason.Deleted), + "Page eviction must not re-fire OnDispose(Deleted) for tombstoned records"); + ClassicAssert.AreEqual(0, tracker.TotalDisposeDisk(), + "Page eviction must not route through OnDisposeDiskRecord"); + } + + #endregion + + #region Trigger gating — CallOnFlush / CallOnDiskRead / CallOnEvict + + /// + /// With enabled, OnFlush must fire exactly once per + /// non-tombstoned record on the in-memory page being flushed to disk. With the flag disabled + /// (default), OnFlush must never fire regardless of how many flushes run. + /// + [Test, Category("TsavoriteKV")] + public void CallOnFlushGatesOnFlushInvocation() + { + // Control: flag off — OnFlush never fires even across a full flush. + tracker.CallOnFlushFlag = false; + const int n = 20; + for (int i = 0; i < n; i++) Upsert(i, i); + store.Log.FlushAndEvict(wait: true); + ClassicAssert.AreEqual(0, tracker.FlushCount, + "CallOnFlush=false must fully suppress OnFlush invocation"); + + // Experiment: populate fresh records, enable flag, flush. Some of the new records may share + // a page that Tsavorite auto-sealed during insert (before the flag was flipped); the exact + // post-flag count is therefore a lower bound. The gating invariant is the key assertion: + // flipping the flag from false → true must cause OnFlush to fire for at least one record. + tracker.Reset(); + tracker.CallOnFlushFlag = true; + for (int i = 0; i < n; i++) Upsert(n + i, i); + store.Log.Flush(wait: true); + ClassicAssert.Greater(tracker.FlushCount, 0, + "CallOnFlush=true must fire OnFlush for records flushed while the flag was set"); + ClassicAssert.LessOrEqual(tracker.FlushCount, n, + "OnFlush must not fire more times than there are records flushed on this page range"); + } + + /// + /// With enabled, OnDiskRead must fire exactly once + /// per record loaded from disk into memory. With the flag disabled, OnDiskRead must never fire + /// even when pending reads pull records from disk. + /// + [Test, Category("TsavoriteKV")] + public void CallOnDiskReadGatesOnDiskReadInvocation() + { + const int n = 10; + for (int i = 0; i < n; i++) Upsert(i, i); + store.Log.FlushAndEvict(wait: true); + + // Control: flag off — pending reads must not fire OnDiskRead. + tracker.CallOnDiskReadFlag = false; + tracker.Reset(); + using (var s = NewSession()) + { + for (int i = 0; i < n; i++) + { + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + _ = s.BasicContext.Read(new TestObjectKey { key = i }, ref input, ref output, 0); + } + _ = s.BasicContext.CompletePending(wait: true); + } + ClassicAssert.AreEqual(0, tracker.DiskReadCount, + "CallOnDiskRead=false must fully suppress OnDiskRead invocation"); + } + + /// + /// gates the OnEvict callback. When the application + /// opts out, no OnEvict calls should reach the trigger — but Tsavorite's internal heap + /// accounting still runs. + /// + [Test, Category("TsavoriteKV")] + public void CallOnEvictGatingSuppressesOnEvictCallback() + { + tracker.CallOnEvictFlag = false; // opt out of eviction callbacks + + const int n = 100; + for (int i = 0; i < n; i++) Upsert(i, i); + store.Log.FlushAndEvict(wait: true); + + ClassicAssert.AreEqual(0, tracker.EvictCount(EvictionSource.MainLog), + "CallOnEvict=false must fully suppress OnEvict invocation"); + ClassicAssert.AreEqual(0, tracker.EvictCount(EvictionSource.ReadCache), + "No read-cache configured — ReadCache OnEvict must stay zero"); + } + + #endregion + + #region Read cache eviction + + /// + /// With a read cache configured, pending reads populate the read-cache page tail. Flushing and + /// evicting the read cache must fire with + /// — separately from main-log eviction so heap-accounting + /// handlers can route to the correct counter. + /// + [Test, Category("TsavoriteKV")] + public void ReadCacheEvictionFiresOnEvictWithReadCacheSource() + { + // Tear down the default main-log-only store and recreate with read cache enabled. + store.Dispose(); store = null; + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << 15, + PageSize = 1L << 10, + ReadCacheMemorySize = 1L << 15, + ReadCachePageSize = 1L << 10, + ReadCacheEnabled = true, + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TrackedObjectValue.Serializer(), + new LifecycleRecordTriggers(tracker)) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + + const int n = 50; + for (int i = 0; i < n; i++) Upsert(i, i); + store.Log.FlushAndEvict(wait: true); // push to disk so next Read populates read cache + + tracker.Reset(); + // Issue reads — each pending completion brings a record into the read cache. + using (var s = NewSession()) + { + for (int i = 0; i < n; i++) + { + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + _ = s.BasicContext.Read(new TestObjectKey { key = i }, ref input, ref output, 0); + } + _ = s.BasicContext.CompletePending(wait: true); + } + + // Evict the read cache. Every live readcache record must fire OnEvict(ReadCache); zero main-log evictions. + var readCacheEvictBefore = tracker.EvictCount(EvictionSource.ReadCache); + store.ReadCache.FlushAndEvict(wait: true); + var readCacheEvicted = tracker.EvictCount(EvictionSource.ReadCache) - readCacheEvictBefore; + + ClassicAssert.Greater(readCacheEvicted, 0, + "Read-cache eviction must fire OnEvict(ReadCache) at least once for cached records"); + ClassicAssert.AreEqual(0, tracker.EvictCount(EvictionSource.MainLog), + "Read-cache flush must not route through main-log OnEvict"); + } + + #endregion + + #region No double-dispose across lifecycle + + /// + /// Cross-path regression test: upsert → delete → upsert-same-key → evict-to-disk → pending-read. + /// Across all these operations, no individual instance should + /// be disposed more than once — regardless of which trigger disposed it. + /// + [Test, Category("TsavoriteKV")] + public void NoValueObjectIsDisposedMoreThanOnceAcrossLifecycle() + { + tracker.DisposeValuesOnDispose = true; + tracker.DisposeValuesOnDisposeDiskRecord = true; + + const int n = 30; + // Round 1: upsert 30 records + for (int i = 0; i < n; i++) Upsert(i, i); + + // Delete half — fires OnDispose(Deleted) which disposes the value + using (var s = NewSession()) + { + for (int i = 0; i < n / 2; i++) + _ = s.BasicContext.Delete(new TestObjectKey { key = i }); + } + + // Upsert all keys again with new values — for the still-live halves this fires OnDispose(Deleted) + // via the tombstone path; for deleted-then-upserted keys the prior tombstone was elided. + // Keep value==key so the pending-read completion callback's key==value assertion passes below. + for (int i = 0; i < n; i++) Upsert(i, i); + + // Push everything to disk to force a fresh disk-deserialization cycle. + store.Log.FlushAndEvict(wait: true); + + // Read each key — all pending. Handler disposes each deserialized value exactly once. + using (var s = NewSession()) + { + for (int i = 0; i < n; i++) + { + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + var status = s.BasicContext.Read(new TestObjectKey { key = i }, ref input, ref output, 0); + if (status.IsPending) _ = s.BasicContext.CompletePending(wait: true); + } + } + + var (_, maxDisposes) = TrackedObjectValue.Snapshot(); + ClassicAssert.LessOrEqual(maxDisposes, 1, + "No individual value object should ever be disposed more than once across the full lifecycle"); + } + + #endregion + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/RevivificationTests.cs b/libs/storage/Tsavorite/cs/test/test.recordops/RevivificationTests.cs similarity index 53% rename from libs/storage/Tsavorite/cs/test/RevivificationTests.cs rename to libs/storage/Tsavorite/cs/test/test.recordops/RevivificationTests.cs index 4078d3bd38a..0fd16bc44d1 100644 --- a/libs/storage/Tsavorite/cs/test/RevivificationTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recordops/RevivificationTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,9 +6,9 @@ using System.Diagnostics; using System.IO; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -19,7 +19,7 @@ namespace Tsavorite.test.Revivification { // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - internal readonly struct RevivificationSpanByteComparer : IKeyComparer + internal readonly struct RevivificationSpanByteComparer : IKeyComparer { private readonly SpanByteComparer defaultComparer; private readonly int collisionRange; @@ -30,22 +30,36 @@ internal RevivificationSpanByteComparer(CollisionRange range) collisionRange = (int)range; } - public bool Equals(ref SpanByte k1, ref SpanByte k2) => defaultComparer.Equals(ref k1, ref k2); + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => defaultComparer.Equals(k1, k2); // The hash code ends with 0 so mod Ten isn't so helpful, so shift - public long GetHashCode64(ref SpanByte k) => (defaultComparer.GetHashCode64(ref k) >> 4) % collisionRange; + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => (defaultComparer.GetHashCode64(k) >> 4) % collisionRange; } } namespace Tsavorite.test.Revivification { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; - using IntAllocator = BlittableAllocator>>; - using IntStoreFunctions = StoreFunctions>; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; - using SpanByteStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; public enum DeleteDest { FreeList, InChain } @@ -59,6 +73,18 @@ public enum RecordElision { Elide, NoElide } struct RevivificationTestUtils { + internal static RevivificationSettings FixedLengthBins = new() + { + FreeRecordBins = + [ + new RevivificationBin() + { + RecordSize = RoundUp(RecordInfo.Size + 2 * (sizeof(int) + sizeof(long)), Constants.kRecordAlignment), // We have "fixed length" for these integer bins, with long Key and Value + BestFitScanLimit = RevivificationBin.UseFirstFit + } + ] + }; + internal const double HalfOfMutableFraction = 0.5; // Half of the mutable region internal static double GetRevivifiableFraction(RevivifiableFraction frac) @@ -76,30 +102,28 @@ internal static RMWInfo CopyToRMWInfo(ref UpsertInfo upsertInfo) SessionID = upsertInfo.SessionID, Address = upsertInfo.Address, KeyHash = upsertInfo.KeyHash, - UsedValueLength = upsertInfo.UsedValueLength, - FullValueLength = upsertInfo.FullValueLength, Action = RMWAction.Default, }; - internal static FreeRecordPool CreateSingleBinFreeRecordPool( - TsavoriteKV store, RevivificationBin binDef, int fixedRecordLength = 0) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => new(store, new RevivificationSettings() { FreeRecordBins = [binDef] }, fixedRecordLength); - - internal static bool HasRecords(TsavoriteKV store) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => HasRecords(store.RevivificationManager.FreeRecordPool); - - internal static bool HasRecords(TsavoriteKV store, FreeRecordPool pool) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => HasRecords(pool ?? store.RevivificationManager.FreeRecordPool); - - internal static bool HasRecords(FreeRecordPool pool) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static FreeRecordPool CreateSingleBinFreeRecordPool( + TsavoriteKV store, RevivificationBin binDef) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => new(store, new RevivificationSettings() { FreeRecordBins = [binDef] }); + + internal static bool HasRecords(TsavoriteKV store) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => HasRecords(store.RevivificationManager.freeRecordPool); + + internal static bool HasRecords(TsavoriteKV store, FreeRecordPool pool) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => HasRecords(pool ?? store.RevivificationManager.freeRecordPool); + + internal static bool HasRecords(FreeRecordPool pool) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { if (pool is not null) { @@ -112,59 +136,59 @@ internal static bool HasRecords(FreeR return false; } - internal static FreeRecordPool SwapFreeRecordPool( - TsavoriteKV store, FreeRecordPool inPool) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static FreeRecordPool SwapFreeRecordPool( + TsavoriteKV store, FreeRecordPool inPool) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - var pool = store.RevivificationManager.FreeRecordPool; - store.RevivificationManager.FreeRecordPool = inPool; + var pool = store.RevivificationManager.freeRecordPool; + store.RevivificationManager.freeRecordPool = inPool; return pool; } internal const int DefaultRecordWaitTimeoutMs = 2000; - internal static bool GetBinIndex(FreeRecordPool pool, int recordSize, out int binIndex) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static bool GetBinIndex(FreeRecordPool pool, int recordSize, out int binIndex) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.GetBinIndex(recordSize, out binIndex); - internal static int GetBinCount(FreeRecordPool pool) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static int GetBinCount(FreeRecordPool pool) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.bins.Length; - internal static int GetRecordCount(FreeRecordPool pool, int binIndex) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static int GetRecordCount(FreeRecordPool pool, int binIndex) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.bins[binIndex].recordCount; - internal static int GetMaxRecordSize(FreeRecordPool pool, int binIndex) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static int GetMaxRecordSize(FreeRecordPool pool, int binIndex) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.bins[binIndex].maxRecordSize; - internal static unsafe bool IsSet(FreeRecordPool pool, int binIndex, int recordIndex) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static unsafe bool IsSet(FreeRecordPool pool, int binIndex, int recordIndex) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.bins[binIndex].records[recordIndex].IsSet; - internal static bool TryTakeFromBin(FreeRecordPool pool, int binIndex, int recordSize, long minAddress, - TsavoriteKV store, out long address, ref RevivificationStats revivStats) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => pool.bins[binIndex].TryTake(recordSize, minAddress, store, out address, ref revivStats); + internal static bool TryTakeFromBin(FreeRecordPool pool, int binIndex, in RecordSizeInfo sizeInfo, long minAddress, + TsavoriteKV store, out long address, ref RevivificationStats revivStats) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => pool.bins[binIndex].TryTake(sizeInfo.ActualInlineRecordSize, minAddress, store, out address, ref revivStats); - internal static int GetSegmentStart(FreeRecordPool pool, int binIndex, int recordSize) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static int GetSegmentStart(FreeRecordPool pool, int binIndex, int recordSize) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => pool.bins[binIndex].GetSegmentStart(recordSize); - internal static void WaitForRecords(TsavoriteKV store, bool want, FreeRecordPool pool = default) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void WaitForRecords(TsavoriteKV store, bool want, FreeRecordPool pool = default) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - pool ??= store.RevivificationManager.FreeRecordPool; + pool ??= store.RevivificationManager.freeRecordPool; // Wait until CheckEmptyWorker or TryAdd() has set the bin counters. var sw = new Stopwatch(); @@ -181,14 +205,14 @@ internal static void WaitForRecords(T } } - internal static unsafe int GetFreeRecordCount(TsavoriteKV store) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => GetFreeRecordCount(store.RevivificationManager.FreeRecordPool); + internal static int GetFreeRecordCount(TsavoriteKV store) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => GetFreeRecordCount(store.RevivificationManager.freeRecordPool); - internal static unsafe int GetFreeRecordCount(FreeRecordPool pool) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static unsafe int GetFreeRecordCount(FreeRecordPool pool) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { // This returns the count of all records, not just the free ones. var count = 0; @@ -206,37 +230,30 @@ internal static unsafe int GetFreeRecordCount(TsavoriteKV store, TKey key) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => AssertElidable(store, ref key); - - internal static void AssertElidable(TsavoriteKV store, ref TKey key) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static void AssertElidable(TsavoriteKV store, TestSpanByteKey key) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - OperationStackContext stackCtx = new(store.storeFunctions.GetKeyHashCode64(ref key)); - ClassicAssert.IsTrue(store.FindTag(ref stackCtx.hei), $"AssertElidable: Cannot find key {key}"); - var recordInfo = store.hlog.GetInfo(store.hlog.GetPhysicalAddress(stackCtx.hei.Address)); - ClassicAssert.Less(recordInfo.PreviousAddress, store.hlogBase.BeginAddress, "AssertElidable: expected elidable key"); + OperationStackContext stackCtx = new(store.storeFunctions.GetKeyHashCode64(key)); + ClassicAssert.IsTrue(store.FindTag(ref stackCtx.hei), $"AssertElidable: Cannot find ii {key.KeyBytes.ToShortString()}"); + var recordInfo = LogRecord.GetInfo(store.hlogBase.GetPhysicalAddress(stackCtx.hei.Address)); + ClassicAssert.Less(recordInfo.PreviousAddress, store.hlogBase.BeginAddress, "AssertElidable: expected elidable ii"); } - internal static int GetRevivifiableRecordCount(TsavoriteKV store, int numRecords) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator - => (int)(numRecords * store.RevivificationManager.revivifiableFraction); + internal static int GetRevivifiableRecordCount(TsavoriteKV store, int numRecords) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator + => (int)(numRecords * store.RevivificationManager.revivifiableFraction * store.Log.allocatorBase.logMutableFraction); - internal static int GetMinRevivifiableKey(TsavoriteKV store, int numRecords) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static int GetMinRevivifiableKey(TsavoriteKV store, int numRecords) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator => numRecords - GetRevivifiableRecordCount(store, numRecords); } - - [AllureNUnit] [TestFixture] - class RevivificationFixedLenTests : AllureTestBase + class RevivificationFixedLenTests : TestBase { - internal class RevivificationFixedLenFunctions : SimpleSimpleFunctions + internal class RevivificationFixedLenFunctions : SimpleLongSimpleFunctions { } @@ -245,17 +262,22 @@ internal class RevivificationFixedLenFunctions : SimpleSimpleFunctions RevivificationFixedLenFunctions functions; - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; + private int recordSize; + [SetUp] public void Setup() { DeleteDirectory(MethodTestDir, wait: true); log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.log"), deleteOnClose: true); + // Records all have a Span corresponding to a 'long' ii and value, which means one length byte. + recordSize = RoundUp(RecordInfo.Size + RecordDataHeader.NumIndicatorBytes + 2 + sizeof(long) * 2, Constants.kRecordAlignment); + double? revivifiableFraction = default; RecordElision? recordElision = default; foreach (var arg in TestContext.CurrentContext.Test.Arguments) @@ -272,7 +294,7 @@ public void Setup() } } - var revivificationSettings = RevivificationSettings.DefaultFixedLength.Clone(); + var revivificationSettings = RevivificationTestUtils.FixedLengthBins.Clone(); if (revivifiableFraction.HasValue) revivificationSettings.RevivifiableFraction = revivifiableFraction.Value; if (recordElision.HasValue) @@ -282,12 +304,12 @@ public void Setup() IndexSize = 1L << 24, LogDevice = log, PageSize = 1L << 12, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, RevivificationSettings = revivificationSettings - }, StoreFunctions.Create(IntKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); functions = new RevivificationFixedLenFunctions(); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -306,9 +328,10 @@ public void TearDown() void Populate() { - for (int key = 0; key < NumRecords; key++) + for (long keyNum = 0; keyNum < NumRecords; keyNum++) { - var status = bContext.Upsert(key, key * ValueMult); + long valueNum = keyNum * ValueMult; + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref valueNum)); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } } @@ -316,6 +339,7 @@ void Populate() [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void SimpleFixedLenTest([Values] DeleteDest deleteDest, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { Populate(); @@ -324,7 +348,8 @@ public void SimpleFixedLenTest([Values] DeleteDest deleteDest, [Values(UpdateOp. if (stayInChain) _ = RevivificationTestUtils.SwapFreeRecordPool(store, default); - var deleteKey = RevivificationTestUtils.GetMinRevivifiableKey(store, NumRecords); + long deleteKeyNum = RevivificationTestUtils.GetMinRevivifiableKey(store, NumRecords) + 2; // +2 to allow for page headers and rounding + var deleteKey = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref deleteKeyNum)); if (!stayInChain) RevivificationTestUtils.AssertElidable(store, deleteKey); var tailAddress = store.Log.TailAddress; @@ -332,8 +357,10 @@ public void SimpleFixedLenTest([Values] DeleteDest deleteDest, [Values(UpdateOp. _ = bContext.Delete(deleteKey); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); - var updateKey = deleteDest == DeleteDest.InChain ? deleteKey : NumRecords + 1; - var updateValue = updateKey + ValueMult; + long updateKeyNum = deleteDest == DeleteDest.InChain ? deleteKeyNum : NumRecords + 1; + var updateValueNum = updateKeyNum + ValueMult; + var updateKey = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref updateKeyNum)); + var updateValue = SpanByte.FromPinnedVariable(ref updateValueNum); if (!stayInChain) { @@ -341,16 +368,17 @@ public void SimpleFixedLenTest([Values] DeleteDest deleteDest, [Values(UpdateOp. RevivificationTestUtils.WaitForRecords(store, want: true); } - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(updateKey, updateValue) : bContext.RMW(updateKey, updateValue); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(updateKey, updateValue) : bContext.RMW(updateKey, ref updateValueNum); if (!stayInChain) RevivificationTestUtils.WaitForRecords(store, want: false); - ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, "Expected tail address not to grow (record was revivified)"); + ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, "Expected tail address not to grow (recordPtr was revivified)"); } [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void UnelideTest([Values] RecordElision elision, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { Populate(); @@ -358,31 +386,33 @@ public void UnelideTest([Values] RecordElision elision, [Values(UpdateOp.Upsert, var tailAddress = store.Log.TailAddress; // First delete all keys. This will overflow the bin. - for (var key = 0; key < NumRecords; ++key) + for (long keyNum = 0; keyNum < NumRecords; ++keyNum) { - _ = bContext.Delete(key); + _ = bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum))); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } - ClassicAssert.AreEqual(RevivificationBin.DefaultRecordsPerBin, RevivificationTestUtils.GetFreeRecordCount(store)); + // The NumberOfRecords will be adjusted upward so the partition is cache-line aligned, so this may be higher than specified. + ClassicAssert.LessOrEqual(RevivificationBin.DefaultRecordsPerBin, RevivificationTestUtils.GetFreeRecordCount(store)); RevivificationTestUtils.WaitForRecords(store, want: true); // Now re-add the keys. - for (var key = 0; key < NumRecords; ++key) + for (long keyNum = 0; keyNum < NumRecords; ++keyNum) { - var value = key + ValueMult; - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(key, value) : bContext.RMW(key, value); + long valueNum = keyNum + ValueMult; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(key, value) : bContext.RMW(key, ref valueNum); } // Now re-add the keys. For the elision case, we should see tailAddress grow sharply as only the records in the bin are available // for revivification. For In-Chain, we will revivify records that were unelided after the bin overflowed. But we have some records // ineligible for revivification due to revivifiableFraction. - var recordSize = RecordInfo.GetLength() + sizeof(int) * 2; var numIneligibleRecords = NumRecords - RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords); var noElisionExpectedTailAddress = tailAddress + numIneligibleRecords * recordSize; - if (elision == RecordElision.NoElide) - ClassicAssert.AreEqual(noElisionExpectedTailAddress, store.Log.TailAddress, "Expected tail address not to grow (records were revivified)"); + if (elision == RecordElision.NoElide) // Add 4 to account for page headers and rounding + ClassicAssert.GreaterOrEqual(noElisionExpectedTailAddress + 4 * recordSize, store.Log.TailAddress, "Expected tail address not to grow (records were revivified)"); else ClassicAssert.Less(noElisionExpectedTailAddress, store.Log.TailAddress, "Expected tail address to grow (records were not revivified)"); } @@ -390,6 +420,7 @@ public void UnelideTest([Values] RecordElision elision, [Values(UpdateOp.Upsert, [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] #pragma warning disable IDE0060 // Remove unused parameter (used by setup) public void SimpleMinAddressAddTest([Values] RevivifiableFraction revivifiableFraction) #pragma warning restore IDE0060 // Remove unused parameter @@ -397,11 +428,13 @@ public void SimpleMinAddressAddTest([Values] RevivifiableFraction revivifiableFr Populate(); // This should not go to FreeList because it's below the RevivifiableFraction - ClassicAssert.IsTrue(bContext.Delete(2).Found); + long keyNum = 2; + ClassicAssert.IsTrue(bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum))).Found); ClassicAssert.AreEqual(0, RevivificationTestUtils.GetFreeRecordCount(store)); // This should go to FreeList because it's above the RevivifiableFraction - ClassicAssert.IsTrue(bContext.Delete(NumRecords - 1).Found); + keyNum = NumRecords - 1; + ClassicAssert.IsTrue(bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum))).Found); ClassicAssert.AreEqual(1, RevivificationTestUtils.GetFreeRecordCount(store)); } @@ -415,7 +448,8 @@ public void SimpleMinAddressTakeTest([Values] RevivifiableFraction revivifiableF Populate(); // This should go to FreeList because it's above the RevivifiableFraction - ClassicAssert.IsTrue(bContext.Delete(NumRecords - 1).Found); + long keyNum = NumRecords - 1; + ClassicAssert.IsTrue(bContext.Delete(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum))).Found); ClassicAssert.AreEqual(1, RevivificationTestUtils.GetFreeRecordCount(store)); RevivificationTestUtils.WaitForRecords(store, want: true); @@ -423,10 +457,11 @@ public void SimpleMinAddressTakeTest([Values] RevivifiableFraction revivifiableF var pool = RevivificationTestUtils.SwapFreeRecordPool(store, default); // Now add a bunch of records to drop the FreeListed address below the RevivifiableFraction - int maxRecord = NumRecords * 2; - for (int key = NumRecords; key < maxRecord; key++) + long maxRecord = NumRecords * 2, valueNum; + for (keyNum = NumRecords; keyNum < maxRecord; keyNum++) { - var status = bContext.Upsert(key, key * ValueMult); + valueNum = keyNum * ValueMult; + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref valueNum)); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } @@ -434,15 +469,14 @@ public void SimpleMinAddressTakeTest([Values] RevivifiableFraction revivifiableF _ = RevivificationTestUtils.SwapFreeRecordPool(store, pool); var tailAddress = store.Log.TailAddress; - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(maxRecord, maxRecord * ValueMult) : bContext.RMW(maxRecord, maxRecord * ValueMult); + valueNum = maxRecord * ValueMult; + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref maxRecord)), SpanByte.FromPinnedVariable(ref valueNum)) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref maxRecord)), ref valueNum); - ClassicAssert.Less(tailAddress, store.Log.TailAddress, "Expected tail address to grow (record was not revivified)"); + ClassicAssert.Less(tailAddress, store.Log.TailAddress, "Expected tail address to grow (recordPtr was not revivified)"); } } - - [AllureNUnit] [TestFixture] - class RevivificationSpanByteTests : AllureTestBase + class RevivificationSpanByteTests : TestBase { const int KeyLength = 10; const int InitialLength = 50; @@ -453,15 +487,11 @@ class RevivificationSpanByteTests : AllureTestBase internal class RevivificationSpanByteFunctions : SpanByteFunctions { - private readonly TsavoriteKV> store; + private readonly TsavoriteKV> store; // Must be set after session is created - internal ClientSession> session; + internal ClientSession> session; - internal int expectedConcurrentDestLength = InitialLength; - internal int expectedSingleDestLength = InitialLength; - internal int expectedConcurrentFullValueLength = -1; - internal int expectedSingleFullValueLength = -1; internal int expectedInputLength = InitialLength; // used to configurably change RMW behavior to test tombstoning via RMW route. @@ -470,13 +500,13 @@ internal class RevivificationSpanByteFunctions : SpanByteFunctions internal bool deleteInCU = false; internal bool forceSkipIpu = false; - // This is a queue rather than a single value because there may be calls to, for example, ConcurrentWriter with one length - // followed by SingleWriter with another. - internal Queue expectedUsedValueLengths = new(); + // This is a queue rather than a single value because there may be calls to, for example, InPlaceWriter with one length + // followed by InitialWriter with another. + internal Queue expectedValueLengths = new(); internal bool readCcCalled, rmwCcCalled; - internal RevivificationSpanByteFunctions(TsavoriteKV> store) + internal RevivificationSpanByteFunctions(TsavoriteKV> store) { this.store = store; } @@ -494,70 +524,73 @@ private void AssertInfoValid(ref DeleteInfo deleteInfo) ClassicAssert.AreEqual(session.ctx.version, deleteInfo.Version); } - private static void VerifyKeyAndValue(ref SpanByte functionsKey, ref SpanByte functionsValue) + private static void VerifyKeyAndValue(ReadOnlySpan functionsKey, ReadOnlySpan functionsValue) { int valueOffset = 0, valueLengthRemaining = functionsValue.Length; ClassicAssert.Less(functionsKey.Length, valueLengthRemaining); while (valueLengthRemaining > 0) { var compareLength = Math.Min(functionsKey.Length, valueLengthRemaining); - Span valueSpan = functionsValue.AsSpan().Slice(valueOffset, compareLength); - Span keySpan = functionsKey.AsSpan()[..compareLength]; - ClassicAssert.IsTrue(valueSpan.SequenceEqual(keySpan), $"functionsValue (offset {valueOffset}, len {compareLength}: {SpanByte.FromPinnedSpan(valueSpan)}) does not match functionsKey ({SpanByte.FromPinnedSpan(keySpan)})"); + var valueSpan = functionsValue.Slice(valueOffset, compareLength); + var keySpan = functionsKey[..compareLength]; + ClassicAssert.IsTrue(valueSpan.SequenceEqual(keySpan), $"functionsValue (offset {valueOffset}, len {compareLength}: {valueSpan.ToShortString()}) does not match functionsKey ({keySpan.ToShortString()})"); valueLengthRemaining -= compareLength; } } - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + unsafe void CheckExpectedLengthsBefore(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, long recordAddress, bool isIPU = false) { - var rmwInfo = RevivificationTestUtils.CopyToRMWInfo(ref upsertInfo); - var result = InitialUpdater(ref key, ref input, ref dst, ref output, ref rmwInfo, ref recordInfo); - upsertInfo.UsedValueLength = rmwInfo.UsedValueLength; - return result; - } - - public override bool ConcurrentWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - var rmwInfo = RevivificationTestUtils.CopyToRMWInfo(ref upsertInfo); - var result = InPlaceUpdater(ref key, ref input, ref dst, ref output, ref rmwInfo, ref recordInfo); - upsertInfo.UsedValueLength = rmwInfo.UsedValueLength; - return result; - } - - public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - AssertInfoValid(ref rmwInfo); - ClassicAssert.AreEqual(expectedInputLength, input.Length); + var expectedValueLength = expectedValueLengths.Dequeue(); - var expectedUsedValueLength = expectedUsedValueLengths.Dequeue(); - - if (value.Length == 0) + // If an overflow logRecord is from new recordPtr creation it has not had its overflow set yet; it has just been initialized to inline length of ObjectIdMap.ObjectIdSize, + // and we'll call LogField.ConvertToOverflow later in this ISessionFunctions call to do the actual overflow allocation. + if (!logRecord.Info.ValueIsInline || (sizeInfo.IsSet && !sizeInfo.ValueIsInline)) { - ClassicAssert.AreEqual(expectedUsedValueLength, rmwInfo.UsedValueLength); // for the length header - ClassicAssert.AreEqual(Constants.kRecordAlignment, rmwInfo.FullValueLength); // This should be the "added record for Delete" case, so a "default" value + var (valueLength, _ /*valueAddress*/) = new RecordDataHeader((byte*)logRecord.DataHeaderAddress).GetValueFieldInfo(logRecord.Info); + ClassicAssert.AreEqual(ObjectIdMap.ObjectIdSize, (int)valueLength); } + if (sizeInfo.ValueIsInline) + ClassicAssert.AreEqual(expectedValueLength, logRecord.ValueSpan.Length); else + ClassicAssert.AreEqual(logRecord.Info.ValueIsInline ? expectedValueLength : ObjectIdMap.ObjectIdSize, logRecord.ValueSpan.Length); + + ClassicAssert.GreaterOrEqual(recordAddress, store.hlogBase.ReadOnlyAddress); + + // !IsSet means it is from Delete which does not receive a RecordSizeInfo. isIPU is an in-place update and thus the new value may legitimately be larger than the recordPtr. + if (sizeInfo.IsSet && !isIPU) { - ClassicAssert.AreEqual(expectedSingleDestLength, value.Length); - ClassicAssert.AreEqual(expectedSingleFullValueLength, rmwInfo.FullValueLength); - ClassicAssert.AreEqual(expectedUsedValueLength, rmwInfo.UsedValueLength); - ClassicAssert.GreaterOrEqual(rmwInfo.Address, store.hlogBase.ReadOnlyAddress); + var allocated = logRecord.AllocatedSize; + var actual = logRecord.ActualSize; + ClassicAssert.AreEqual(sizeInfo.ActualInlineRecordSize, actual); + ClassicAssert.AreEqual(sizeInfo.AllocatedInlineRecordSize, allocated); } - return base.InitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); } - public override bool NeedCopyUpdate(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + CheckExpectedLengthsBefore(ref logRecord, in sizeInfo, upsertInfo.Address); + return base.InitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); + } + + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) + { + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + CheckExpectedLengthsBefore(ref logRecord, in sizeInfo, upsertInfo.Address, isIPU: true); + return base.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo); + } + + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { if (deleteInNCU) { rmwInfo.Action = RMWAction.ExpireAndStop; return false; } - - return base.NeedCopyUpdate(ref key, ref input, ref oldValue, ref output, ref rmwInfo); + return base.NeedCopyUpdate(in srcLogRecord, ref input, ref output, ref rmwInfo); } - public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { if (deleteInCU) { @@ -566,28 +599,13 @@ public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanB } AssertInfoValid(ref rmwInfo); - - ClassicAssert.AreEqual(expectedInputLength, input.Length); - var expectedUsedValueLength = expectedUsedValueLengths.Dequeue(); - - if (newValue.Length == 0) - { - ClassicAssert.AreEqual(sizeof(int), rmwInfo.UsedValueLength); // for the length header - ClassicAssert.AreEqual(Constants.kRecordAlignment, rmwInfo.FullValueLength); // This should be the "added record for Delete" case, so a "default" value - } - else - { - ClassicAssert.AreEqual(expectedSingleDestLength, newValue.Length); - ClassicAssert.AreEqual(expectedSingleFullValueLength, rmwInfo.FullValueLength); - ClassicAssert.AreEqual(expectedUsedValueLength, rmwInfo.UsedValueLength); - ClassicAssert.GreaterOrEqual(rmwInfo.Address, store.hlogBase.ReadOnlyAddress); - } - return base.CopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo, ref recordInfo); + CheckExpectedLengthsBefore(ref dstLogRecord, in sizeInfo, rmwInfo.Address); + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(input.ReadOnlySpan, in sizeInfo); } - public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { AssertInfoValid(ref rmwInfo); @@ -601,105 +619,91 @@ public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref Sp } ClassicAssert.AreEqual(expectedInputLength, input.Length); - ClassicAssert.AreEqual(expectedConcurrentDestLength, value.Length); - ClassicAssert.AreEqual(expectedConcurrentFullValueLength, rmwInfo.FullValueLength); - - VerifyKeyAndValue(ref key, ref value); - - var expectedUsedValueLength = expectedUsedValueLengths.Dequeue(); - ClassicAssert.AreEqual(expectedUsedValueLength, rmwInfo.UsedValueLength); - ClassicAssert.GreaterOrEqual(rmwInfo.Address, store.hlogBase.ReadOnlyAddress); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + CheckExpectedLengthsBefore(ref logRecord, in sizeInfo, rmwInfo.Address, isIPU: true); + VerifyKeyAndValue(logRecord.Key, logRecord.ValueSpan); - return base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + return logRecord.TrySetValueSpanAndPrepareOptionals(input.ReadOnlySpan, in sizeInfo); } - // Override the default SpanByteFunctions impelementation; for these tests, we always want the input length. - public override int GetRMWModifiedValueLength(ref SpanByte value, ref SpanByte input) => input.TotalSize; - - public override bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public override bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { AssertInfoValid(ref deleteInfo); - ClassicAssert.AreEqual(expectedSingleDestLength, value.Length); - ClassicAssert.AreEqual(expectedSingleFullValueLength, deleteInfo.FullValueLength); - var expectedUsedValueLength = expectedUsedValueLengths.Dequeue(); - ClassicAssert.AreEqual(expectedUsedValueLength, deleteInfo.UsedValueLength); + RecordSizeInfo sizeInfo = default; + CheckExpectedLengthsBefore(ref logRecord, in sizeInfo, deleteInfo.Address); - ClassicAssert.GreaterOrEqual(deleteInfo.Address, store.hlogBase.ReadOnlyAddress); - - return base.SingleDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + return base.InitialDeleter(ref logRecord, ref deleteInfo); } - public override bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public override bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { AssertInfoValid(ref deleteInfo); - ClassicAssert.AreEqual(expectedConcurrentDestLength, value.Length); - ClassicAssert.AreEqual(expectedConcurrentFullValueLength, deleteInfo.FullValueLength); - - var expectedUsedValueLength = expectedUsedValueLengths.Dequeue(); - ClassicAssert.AreEqual(expectedUsedValueLength, deleteInfo.UsedValueLength); - ClassicAssert.GreaterOrEqual(deleteInfo.Address, store.hlogBase.ReadOnlyAddress); + RecordSizeInfo sizeInfo = default; + CheckExpectedLengthsBefore(ref logRecord, in sizeInfo, deleteInfo.Address); - return base.ConcurrentDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + return base.InPlaceDeleter(ref logRecord, ref deleteInfo); } - public override bool PostCopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public override bool PostCopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { AssertInfoValid(ref rmwInfo); - return base.PostCopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo); + return base.PostCopyUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, ref input, ref output, ref rmwInfo); } - public override void PostInitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) + public override void PostInitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { AssertInfoValid(ref rmwInfo); - base.PostInitialUpdater(ref key, ref input, ref value, ref output, ref rmwInfo); + base.PostInitialUpdater(ref logRecord, in sizeInfo, ref input, ref output, ref rmwInfo); } - public override void PostSingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason writeReason) + public override void PostInitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { AssertInfoValid(ref upsertInfo); - base.PostSingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, writeReason); + base.PostInitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); } - public override void PostSingleDeleter(ref SpanByte key, ref DeleteInfo deleteInfo) + public override void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { AssertInfoValid(ref deleteInfo); - base.PostSingleDeleter(ref key, ref deleteInfo); + base.PostInitialDeleter(ref logRecord, ref deleteInfo); } - public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, Empty ctx, Status status, RecordMetadata recordMetadata) { readCcCalled = true; - base.ReadCompletionCallback(ref key, ref input, ref output, ctx, status, recordMetadata); + base.ReadCompletionCallback(ref diskLogRecord, ref input, ref output, ctx, status, recordMetadata); } - public override void RMWCompletionCallback(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, Empty ctx, Status status, RecordMetadata recordMetadata) { rmwCcCalled = true; - base.RMWCompletionCallback(ref key, ref input, ref output, ctx, status, recordMetadata); + base.RMWCompletionCallback(ref diskLogRecord, ref input, ref output, ctx, status, recordMetadata); } - } - - static int RoundUpSpanByteFullValueLength(SpanByte input) => RoundupTotalSizeFullValue(input.TotalSize); - - static int RoundUpSpanByteFullValueLength(int dataLength) => RoundupTotalSizeFullValue(sizeof(int) + dataLength); - internal static int RoundupTotalSizeFullValue(int length) => (length + Constants.kRecordAlignment - 1) & (~(Constants.kRecordAlignment - 1)); - - static int RoundUpSpanByteUsedLength(int dataLength) => RoundUp(SpanByteTotalSize(dataLength), sizeof(int)); - - static int SpanByteTotalSize(int dataLength) => sizeof(int) + dataLength; + // Override the default SpanByteFunctions impelementation; for these tests, we always want the input length. + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = input.Length }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref PinnedSpanByte input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = input.Length }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref PinnedSpanByte input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = input.Length }; + } const int NumRecords = 200; RevivificationSpanByteFunctions functions; RevivificationSpanByteComparer comparer; - private TsavoriteKV> store; - private ClientSession> session; - private BasicContext> bContext; + private TsavoriteKV> store; + private ClientSession> session; + private BasicContext> bContext; private IDevice log; [SetUp] @@ -710,12 +714,13 @@ public void Setup() CollisionRange collisionRange = CollisionRange.None; - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = 1L << 24, LogDevice = log, PageSize = 1L << 17, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, + MaxInlineValueSize = 1024, RevivificationSettings = RevivificationSettings.PowerOf2Bins }; @@ -741,12 +746,12 @@ public void Setup() comparer = new RevivificationSpanByteComparer(collisionRange); store = new(kvSettings - , StoreFunctions.Create(comparer, SpanByteRecordDisposer.Instance) + , StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); functions = new RevivificationSpanByteFunctions(store); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; functions.session = session; } @@ -768,24 +773,20 @@ public void TearDown() void Populate(int from, int to) { - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(input); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory output = new(); for (int ii = from; ii < to; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(input.TotalSize); - var status = bContext.Upsert(ref key, ref input, ref input, ref output); + key.Fill((byte)ii); + input.Fill((byte)ii); + functions.expectedValueLengths.Enqueue(pinnedInputSpan.Length); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); - ClassicAssert.IsEmpty(functions.expectedUsedValueLengths); + ClassicAssert.IsEmpty(functions.expectedValueLengths); } } @@ -798,10 +799,9 @@ public void SpanByteNoRevivLengthTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] Up { Populate(); - Span keyVec = stackalloc byte[KeyLength]; + Span key = stackalloc byte[KeyLength]; byte fillByte = 42; - keyVec.Fill(fillByte); - var key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); // Do NOT delete; this is a no-reviv test of lengths @@ -813,42 +813,30 @@ public void SpanByteNoRevivLengthTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] Up _ => -1 }; - functions.expectedSingleDestLength = functions.expectedInputLength; - functions.expectedConcurrentDestLength = InitialLength; // This is from the initial Populate() - functions.expectedSingleFullValueLength = RoundUpSpanByteFullValueLength(functions.expectedInputLength); - functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - - Span inputVec = stackalloc byte[functions.expectedInputLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(fillByte); + Span input = stackalloc byte[functions.expectedInputLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(fillByte); // For Grow, we won't be able to satisfy the request with a revivification, and the new value length will be GrowLength - functions.expectedUsedValueLengths.Enqueue(sizeof(int) + InitialLength); + functions.expectedValueLengths.Enqueue(InitialLength); if (growth == Growth.Grow) - functions.expectedUsedValueLengths.Enqueue(sizeof(int) + GrowLength); + functions.expectedValueLengths.Enqueue(GrowLength); SpanByteAndMemory output = new(); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); - ClassicAssert.IsEmpty(functions.expectedUsedValueLengths); + ClassicAssert.IsEmpty(functions.expectedValueLengths); if (growth == Growth.Shrink) { - // What's there now will be what is passed to ConcurrentWriter/IPU (if Shrink, we kept the same value we allocated initially) - functions.expectedConcurrentFullValueLength = growth == Growth.Shrink ? RoundUpSpanByteFullValueLength(InitialLength) : functions.expectedSingleFullValueLength; - - // Now let's see if we have the correct expected extra length in the destination. - inputVec = stackalloc byte[InitialLength / 2]; // Grow this from ShrinkLength to InitialLength - input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(fillByte); + input = stackalloc byte[InitialLength / 2]; // Shrink this from InitialLength to ShrinkLength + input.Fill(fillByte); - functions.expectedInputLength = InitialLength / 2; - functions.expectedConcurrentDestLength = InitialLength / 2; - functions.expectedSingleFullValueLength = RoundUpSpanByteFullValueLength(functions.expectedInputLength); - functions.expectedUsedValueLengths.Enqueue(input.TotalSize); + functions.expectedInputLength = input.Length; + functions.expectedValueLengths.Enqueue(input.Length); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); - ClassicAssert.IsEmpty(functions.expectedUsedValueLengths); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); + ClassicAssert.IsEmpty(functions.expectedValueLengths); } } @@ -860,36 +848,36 @@ internal enum DeletionRoutes RMW_CU } - private Status DeleteViaRMW(ref SpanByte key, Span mockInputVec, byte fillByte) + private Status DeleteViaRMW(TestSpanByteKey key, Span mockInputVec, byte fillByte) { - var mockInput = SpanByte.FromPinnedSpan(mockInputVec); + var mockInput = PinnedSpanByte.FromPinnedSpan(mockInputVec); mockInputVec.Fill(fillByte); - return bContext.RMW(ref key, ref mockInput); + return bContext.RMW(key, ref mockInput); } - private Status PerformDeletion(DeletionRoutes deletionRoute, ref SpanByte key, byte fillByte) + private Status PerformDeletion(DeletionRoutes deletionRoute, TestSpanByteKey key, byte fillByte) { Status status; switch (deletionRoute) { case DeletionRoutes.DELETE: - return bContext.Delete(ref key); + return bContext.Delete(key); case DeletionRoutes.RMW_IPU: functions.deleteInIpu = true; Span mockInputVec = stackalloc byte[InitialLength]; - status = DeleteViaRMW(ref key, mockInputVec, fillByte); + status = DeleteViaRMW(key, mockInputVec, fillByte); functions.deleteInIpu = false; break; case DeletionRoutes.RMW_NCU: functions.deleteInNCU = true; mockInputVec = stackalloc byte[InitialLength]; - status = DeleteViaRMW(ref key, mockInputVec, fillByte); + status = DeleteViaRMW(key, mockInputVec, fillByte); functions.deleteInNCU = false; break; case DeletionRoutes.RMW_CU: functions.deleteInCU = true; mockInputVec = stackalloc byte[InitialLength]; - status = DeleteViaRMW(ref key, mockInputVec, fillByte); + status = DeleteViaRMW(key, mockInputVec, fillByte); functions.deleteInCU = false; break; default: @@ -908,34 +896,32 @@ public void SpanByteSimpleTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp var tailAddress = store.Log.TailAddress; - Span keyVec = stackalloc byte[KeyLength]; + Span key = stackalloc byte[KeyLength]; byte fillByte = 42; - keyVec.Fill(fillByte); - var key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - - Status status = PerformDeletion(deletionRoute, ref key, fillByte); + functions.expectedValueLengths.Enqueue(InitialLength); + Status status = PerformDeletion(deletionRoute, TestSpanByteKey.FromPinnedSpan(key), fillByte); + //if (deletionRoute == DeletionRoutes.DELETE) ClassicAssert.IsTrue(status.Found, status.ToString()); + //else + // ClassicAssert.IsTrue(status.NotFound && status.ShouldExpire, status.ToString()); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(fillByte); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(fillByte); SpanByteAndMemory output = new(); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); RevivificationTestUtils.WaitForRecords(store, want: true); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } @@ -947,44 +933,39 @@ public void SpanByteDeletionViaRMWRCURevivifiesOriginalRecordAfterTombstoning( { Populate(); - Span keyVec = stackalloc byte[KeyLength]; + Span key = stackalloc byte[KeyLength]; byte fillByte = 42; - keyVec.Fill(fillByte); - var key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); functions.forceSkipIpu = true; - var status = PerformDeletion(deletionRoute, ref key, fillByte); + var status = PerformDeletion(deletionRoute, TestSpanByteKey.FromPinnedSpan(key), fillByte); functions.forceSkipIpu = false; RevivificationTestUtils.WaitForRecords(store, want: true); ClassicAssert.AreEqual(1, RevivificationTestUtils.GetFreeRecordCount(store)); - ClassicAssert.IsTrue(status.Found, status.ToString()); + //ClassicAssert.IsTrue(status.NotFound && status.ShouldExpire, status.ToString()); + ClassicAssert.IsTrue(status.Found && status.IsExpired, status.ToString()); var tailAddress = store.Log.TailAddress; Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + var input = PinnedSpanByte.FromPinnedSpan(inputVec); inputVec.Fill(fillByte); SpanByteAndMemory output = new(); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(input); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); - // brand new value so we try to use a record out of free list - keyVec = stackalloc byte[KeyLength]; + // brand new value so we try to use a recordPtr out of free list fillByte = 255; - keyVec.Fill(fillByte); - key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref input, input.ReadOnlySpan, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref input); // since above would use revivification free list we should see no change of tail address. ClassicAssert.AreEqual(store.Log.TailAddress, tailAddress); @@ -1002,35 +983,30 @@ public void SpanByteIPUGrowAndRevivifyTest([Values(UpdateOp.Upsert, UpdateOp.RMW var tailAddress = store.Log.TailAddress; - Span keyVec = stackalloc byte[KeyLength]; + Span key = stackalloc byte[KeyLength]; byte fillByte = 42; - keyVec.Fill(fillByte); - var key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); - Span inputVec = stackalloc byte[GrowLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(fillByte); + Span input = stackalloc byte[GrowLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(fillByte); SpanByteAndMemory output = new(); functions.expectedInputLength = GrowLength; - functions.expectedSingleDestLength = GrowLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = RoundUpSpanByteFullValueLength(GrowLength); - functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(GrowLength)); + functions.expectedValueLengths.Enqueue(InitialLength); + functions.expectedValueLengths.Enqueue(GrowLength); - // Get a free record from a failed IPU. + // Get a free recordPtr from a failed IPU. if (updateOp == UpdateOp.Upsert) { - var status = bContext.Upsert(ref key, ref input, ref input, ref output); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } else if (updateOp == UpdateOp.RMW) { - var status = bContext.RMW(ref key, ref input); + var status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.IsTrue(status.Record.CopyUpdated, status.ToString()); } @@ -1042,24 +1018,22 @@ public void SpanByteIPUGrowAndRevivifyTest([Values(UpdateOp.Upsert, UpdateOp.RMW RevivificationTestUtils.WaitForRecords(store, want: true); - // Get a new key and shrink the requested length so we revivify the free record from the failed IPU. - keyVec.Fill(NumRecords + 1); - input = SpanByte.FromPinnedSpan(inputVec.Slice(0, InitialLength)); + // Get a new ii and shrink the requested length so we revivify the free recordPtr from the failed IPU. + key.Fill(NumRecords + 1); + input = input.Slice(0, InitialLength); + pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); if (updateOp == UpdateOp.Upsert) { - var status = bContext.Upsert(ref key, ref input, ref input, ref output); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } else if (updateOp == UpdateOp.RMW) { - var status = bContext.RMW(ref key, ref input); + var status = bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } @@ -1077,31 +1051,27 @@ public void SpanByteReadOnlyMinAddressTest([Values(UpdateOp.Upsert, UpdateOp.RMW var tailAddress = store.Log.TailAddress; - Span keyVec = stackalloc byte[KeyLength]; + Span key = stackalloc byte[KeyLength]; byte fillByte = 42; - keyVec.Fill(fillByte); - var key = SpanByte.FromPinnedSpan(keyVec); + key.Fill(fillByte); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(fillByte); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(fillByte); SpanByteAndMemory output = new(); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(input); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.Greater(store.Log.TailAddress, tailAddress); } @@ -1117,13 +1087,12 @@ private long PrepareDeletes(bool stayInChain, byte delAboveRO, FlushMode flushMo var pool = stayInChain ? RevivificationTestUtils.SwapFreeRecordPool(store, null) : null; - // Delete key below (what will be) the readonly line. This is for a target for the test; the record should not be revivified. - Span keyVecDelBelowRO = stackalloc byte[KeyLength]; - keyVecDelBelowRO.Fill(DelBelowRO); - var delKeyBelowRO = SpanByte.FromPinnedSpan(keyVecDelBelowRO); + // Delete ii below (what will be) the readonly line. This is for a target for the test; the recordPtr should not be revivified. + Span delKeyBelowRO = stackalloc byte[KeyLength]; + delKeyBelowRO.Fill(DelBelowRO); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref delKeyBelowRO); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(delKeyBelowRO)); ClassicAssert.IsTrue(status.Found, status.ToString()); if (flushMode == FlushMode.ReadOnly) @@ -1135,17 +1104,16 @@ private long PrepareDeletes(bool stayInChain, byte delAboveRO, FlushMode flushMo var tailAddress = store.Log.TailAddress; - // Delete key above the readonly line. This is the record that will be revivified. + // Delete ii above the readonly line. This is the recordPtr that will be revivified. // If not stayInChain, this also puts two elements in the free list; one should be skipped over on Take() as it is below readonly. - Span keyVecDelAboveRO = stackalloc byte[KeyLength]; - keyVecDelAboveRO.Fill(delAboveRO); - var delKeyAboveRO = SpanByte.FromPinnedSpan(keyVecDelAboveRO); + Span delKeyAboveRO = stackalloc byte[KeyLength]; + delKeyAboveRO.Fill(delAboveRO); if (!stayInChain && collisionRange == CollisionRange.None) // CollisionRange.Ten has a valid .PreviousAddress so won't be moved to FreeList - RevivificationTestUtils.AssertElidable(store, ref delKeyAboveRO); + RevivificationTestUtils.AssertElidable(store, TestSpanByteKey.FromPinnedSpan(delKeyAboveRO)); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - status = bContext.Delete(ref delKeyAboveRO); + functions.expectedValueLengths.Enqueue(InitialLength); + status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(delKeyAboveRO)); ClassicAssert.IsTrue(status.Found, status.ToString()); if (stayInChain) @@ -1159,11 +1127,6 @@ private long PrepareDeletes(bool stayInChain, byte delAboveRO, FlushMode flushMo } ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); - - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - return tailAddress; } @@ -1174,10 +1137,7 @@ private long PrepareDeletes(bool stayInChain, byte delAboveRO, FlushMode flushMo public void SpanByteUpdateRevivifyTest([Values] DeleteDest deleteDest, [Values] UpdateKey updateKey, [Values] CollisionRange collisionRange, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - - bool stayInChain = deleteDest == DeleteDest.InChain || collisionRange != CollisionRange.None; // Collisions make the key inelidable + bool stayInChain = deleteDest == DeleteDest.InChain || collisionRange != CollisionRange.None; // Collisions make the ii inelidable byte delAboveRO = (byte)(NumRecords - (stayInChain ? (int)CollisionRange.Ten + 3 // Will remain in chain @@ -1185,41 +1145,40 @@ public void SpanByteUpdateRevivifyTest([Values] DeleteDest deleteDest, [Values] long tailAddress = PrepareDeletes(stayInChain, delAboveRO, FlushMode.ReadOnly, collisionRange); - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory output = new(); - Span keyVecToTest = stackalloc byte[KeyLength]; - var keyToTest = SpanByte.FromPinnedSpan(keyVecToTest); + Span keyToTest = stackalloc byte[KeyLength]; bool expectReviv; if (updateKey is UpdateKey.Unfound or UpdateKey.CopiedBelowRO) { - // Unfound key should be satisfied from the freelist if !stayInChain, else will allocate a new record as it does not match the key chain. - // CopiedBelowRO should be satisfied from the freelist if !stayInChain, else will allocate a new record as it does not match the key chain + // Unfound ii should be satisfied from the freelist if !stayInChain, else will allocate a new recordPtr as it does not match the ii chain. + // CopiedBelowRO should be satisfied from the freelist if !stayInChain, else will allocate a new recordPtr as it does not match the ii chain // (but exercises a different code path than Unfound). // CollisionRange.Ten has a valid PreviousAddress so it is not elided from the cache. byte fillByte = updateKey == UpdateKey.Unfound ? Unfound : CopiedBelowRO; - keyVecToTest.Fill(fillByte); - inputVec.Fill(fillByte); + keyToTest.Fill(fillByte); + input.Fill(fillByte); expectReviv = !stayInChain && collisionRange != CollisionRange.Ten; } else if (updateKey == UpdateKey.DeletedBelowRO) { - // DeletedBelowRO will not match the key for the in-chain above-RO slot, and we cannot reviv below RO or retrieve below-RO from the - // freelist, so we will always allocate a new record unless we're using the freelist. + // DeletedBelowRO will not match the ii for the in-chain above-RO slot, and we cannot reviv below RO or retrieve below-RO from the + // freelist, so we will always allocate a new recordPtr unless we're using the freelist. byte fillByte = DelBelowRO; - keyVecToTest.Fill(fillByte); - inputVec.Fill(fillByte); + keyToTest.Fill(fillByte); + input.Fill(fillByte); expectReviv = !stayInChain && collisionRange != CollisionRange.Ten; } else if (updateKey == UpdateKey.DeletedAboveRO) { - // DeletedAboveRO means we will reuse an in-chain record, or will get it from the freelist if deleteDest is FreeList. + // DeletedAboveRO means we will reuse an in-chain recordPtr, or will get it from the freelist if deleteDest is FreeList. byte fillByte = delAboveRO; - keyVecToTest.Fill(fillByte); - inputVec.Fill(fillByte); + keyToTest.Fill(fillByte); + input.Fill(fillByte); expectReviv = true; } else @@ -1229,13 +1188,9 @@ public void SpanByteUpdateRevivifyTest([Values] DeleteDest deleteDest, [Values] } functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - - if (!expectReviv) - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(input); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref keyToTest, ref input, ref input, ref output) : bContext.RMW(ref keyToTest, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(keyToTest), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(keyToTest), ref pinnedInputSpan); if (expectReviv) ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); @@ -1254,23 +1209,22 @@ public void SimpleRevivifyTest([Values] DeleteDest deleteDest, [Values(UpdateOp. if (stayInChain) _ = RevivificationTestUtils.SwapFreeRecordPool(store, default); - // This freed record stays in the hash chain. + // This freed recordPtr stays in the hash chain. byte chainKey = NumRecords / 2 - 1; - Span keyVec = stackalloc byte[KeyLength]; - keyVec.Fill(chainKey); - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; + key.Fill(chainKey); if (!stayInChain) - RevivificationTestUtils.AssertElidable(store, ref key); + RevivificationTestUtils.AssertElidable(store, TestSpanByteKey.FromPinnedSpan(key)); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); var tailAddress = store.Log.TailAddress; - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(chainKey); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(chainKey); SpanByteAndMemory output = new(); @@ -1278,8 +1232,8 @@ public void SimpleRevivifyTest([Values] DeleteDest deleteDest, [Values(UpdateOp. RevivificationTestUtils.WaitForRecords(store, want: true); // Revivify in the chain. Because this stays in the chain, the expectedFullValueLength is roundup(InitialLength) - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + functions.expectedValueLengths.Enqueue(InitialLength); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } @@ -1293,20 +1247,19 @@ public void DeleteEntireChainAndRevivifyTest([Values(CollisionRange.Ten)] Collis // These freed records stay in the hash chain; we even skip the first one to ensure nothing goes into the free list. byte chainKey = 5; - Span keyVec = stackalloc byte[KeyLength]; - keyVec.Fill(chainKey); - var key = SpanByte.FromPinnedSpan(keyVec); - var hash = comparer.GetHashCode64(ref key); + Span key = stackalloc byte[KeyLength]; + key.Fill(chainKey); + var hash = comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(key)); List deletedSlots = []; for (int ii = chainKey + 1; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - if (comparer.GetHashCode64(ref key) != hash) + key.Fill((byte)ii); + if (comparer.GetHashCode64(TestSpanByteKey.FromPinnedSpan(key)) != hash) continue; - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); if (ii > RevivificationTestUtils.GetMinRevivifiableKey(store, NumRecords)) deletedSlots.Add((byte)ii); @@ -1319,19 +1272,19 @@ public void DeleteEntireChainAndRevivifyTest([Values(CollisionRange.Ten)] Collis ClassicAssert.Greater(deletedSlots.Count, 5); // should be about Ten var tailAddress = store.Log.TailAddress; - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); - inputVec.Fill(chainKey); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); + input.Fill(chainKey); SpanByteAndMemory output = new(); // Revivify in the chain. Because this stays in the chain, the expectedFullValueLength is roundup(InitialLength) for (int ii = 0; ii < deletedSlots.Count; ++ii) { - keyVec.Fill(deletedSlots[ii]); + key.Fill(deletedSlots[ii]); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + functions.expectedValueLengths.Enqueue(InitialLength); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } } @@ -1339,58 +1292,51 @@ public void DeleteEntireChainAndRevivifyTest([Values(CollisionRange.Ten)] Collis [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void DeleteAllRecordsAndRevivifyTest([Values(CollisionRange.None)] CollisionRange collisionRange, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { Populate(); long tailAddress = store.Log.TailAddress; - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - // "sizeof(int) +" because SpanByte has an int length prefix - var recordSize = RecordInfo.GetLength() + RoundUp(sizeof(int) + keyVec.Length, 8) + RoundUp(sizeof(int) + InitialLength, 8); + Span key = stackalloc byte[KeyLength]; // Delete for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); + key.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); } ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); ClassicAssert.AreEqual(RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords), RevivificationTestUtils.GetFreeRecordCount(store), $"Expected numRecords ({NumRecords}) free records"); - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory output = new(); - functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - // These come from the existing initial allocation so keep the full length - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); + functions.expectedInputLength = InitialLength; // Revivify var revivifiableKeyCount = RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords); for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + functions.expectedValueLengths.Enqueue(InitialLength); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); if (ii < revivifiableKeyCount) - ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, $"unexpected new record for key {ii}"); + ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, $"unexpected new recordPtr for ii {ii}"); else - ClassicAssert.Less(tailAddress, store.Log.TailAddress, $"unexpected revivified record for key {ii}"); + ClassicAssert.Less(tailAddress, store.Log.TailAddress, $"unexpected revivified recordPtr for ii {ii}"); - var status = bContext.Read(ref key, ref output); - ClassicAssert.IsTrue(status.Found, $"Expected to find key {ii}; status == {status}"); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(key), ref output); + ClassicAssert.IsTrue(status.Found, $"Expected to find ii {ii}; status == {status}"); } ClassicAssert.AreEqual(0, RevivificationTestUtils.GetFreeRecordCount(store), "expected no free records remaining"); @@ -1399,61 +1345,63 @@ public void DeleteAllRecordsAndRevivifyTest([Values(CollisionRange.None)] Collis // Confirm for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - var status = bContext.Read(ref key, ref output); - ClassicAssert.IsTrue(status.Found, $"Expected to find key {ii}; status == {status}"); + key.Fill((byte)ii); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(key), ref output); + ClassicAssert.IsTrue(status.Found, $"Expected to find ii {ii}; status == {status}"); } } [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void DeleteAllRecordsAndTakeSnapshotTest() { Populate(); - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; // Delete for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); + key.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); } ClassicAssert.AreEqual(RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords), RevivificationTestUtils.GetFreeRecordCount(store), $"Expected numRecords ({NumRecords}) free records"); +#pragma warning disable CA2012 // Use ValueTasks correctly _ = store.TakeHybridLogCheckpointAsync(CheckpointType.Snapshot).GetAwaiter().GetResult(); +#pragma warning restore CA2012 // Use ValueTasks correctly } [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void DeleteAllRecordsAndIterateTest() { Populate(); - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; // Delete for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); + key.Fill((byte)ii); - RevivificationTestUtils.AssertElidable(store, ref key); + RevivificationTestUtils.AssertElidable(store, TestSpanByteKey.FromPinnedSpan(key)); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); } ClassicAssert.AreEqual(RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords), RevivificationTestUtils.GetFreeRecordCount(store), $"Expected numRecords ({NumRecords}) free records"); using var iterator = session.Iterate(); - while (iterator.GetNext(out _)) + while (iterator.GetNext()) ; } @@ -1462,7 +1410,7 @@ public void DeleteAllRecordsAndIterateTest() [Category(SmokeTestCategory)] public void BinSelectionTest() { - var pool = store.RevivificationManager.FreeRecordPool; + var pool = store.RevivificationManager.freeRecordPool; int expectedBin = 0, recordSize = RevivificationTestUtils.GetMaxRecordSize(pool, expectedBin); while (true) { @@ -1486,115 +1434,59 @@ public void BinSelectionTest() [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] - //[Repeat(30)] - public unsafe void ArtificialBinWrappingTest() - { - var pool = store.RevivificationManager.FreeRecordPool; - - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - - Populate(); - - const int recordSize = 42; - - ClassicAssert.IsTrue(pool.GetBinIndex(recordSize, out int binIndex)); - ClassicAssert.AreEqual(2, binIndex); - - const int minAddress = 1_000; - int logicalAddress = 1_000_000; - - RevivificationStats revivStats = new(); - - // Fill the bin, including wrapping around at the end. - var recordCount = RevivificationTestUtils.GetRecordCount(pool, binIndex); - for (var ii = 0; ii < recordCount; ++ii) - ClassicAssert.IsTrue(store.RevivificationManager.TryAdd(logicalAddress + ii, recordSize, ref revivStats), "ArtificialBinWrappingTest: Failed to Add free record, pt 1"); - - // Try to add to a full bin; this should fail. - revivStats.Reset(); - ClassicAssert.IsFalse(store.RevivificationManager.TryAdd(logicalAddress + recordCount, recordSize, ref revivStats), "ArtificialBinWrappingTest: Expected to fail Adding free record"); - - RevivificationTestUtils.WaitForRecords(store, want: true); - - for (var ii = 0; ii < recordCount; ++ii) - ClassicAssert.IsTrue(RevivificationTestUtils.IsSet(pool, binIndex, ii), "expected bin to be set at ii == {ii}"); - - // Take() one to open up a space in the bin, then add one - revivStats.Reset(); - ClassicAssert.IsTrue(RevivificationTestUtils.TryTakeFromBin(pool, binIndex, recordSize, minAddress, store, out _, ref revivStats)); - revivStats.Reset(); - ClassicAssert.IsTrue(store.RevivificationManager.TryAdd(logicalAddress + recordCount + 1, recordSize, ref revivStats), "ArtificialBinWrappingTest: Failed to Add free record, pt 2"); - - // Take() all records in the bin. - revivStats.Reset(); - for (var ii = 0; ii < recordCount; ++ii) - ClassicAssert.IsTrue(RevivificationTestUtils.TryTakeFromBin(pool, binIndex, recordSize, minAddress, store, out _, ref revivStats), $"ArtificialBinWrappingTest: failed to Take at ii == {ii}"); - _ = revivStats.Dump(); - } - - [Test] - [Category(RevivificationCategory)] - [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] //[Repeat(3000)] - public unsafe void LiveBinWrappingTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp, [Values] WaitMode waitMode, [Values] DeleteDest deleteDest) + public void LiveBinWrappingTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp, [Values] WaitMode waitMode, [Values] DeleteDest deleteDest) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - Populate(); // Note: this test assumes no collisions (every delete goes to the FreeList) - var pool = store.RevivificationManager.FreeRecordPool; - - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + var pool = store.RevivificationManager.freeRecordPool; - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); // "sizeof(int) +" because SpanByte has an int length prefix. - var recordSize = RecordInfo.GetLength() + RoundUp(sizeof(int) + keyVec.Length, 8) + RoundUp(sizeof(int) + InitialLength, 8); + var recordSize = RecordInfo.Size + RoundUp(sizeof(int) + key.Length, Constants.kRecordAlignment) + RoundUp(sizeof(int) + InitialLength, Constants.kRecordAlignment); ClassicAssert.IsTrue(pool.GetBinIndex(recordSize, out int binIndex)); ClassicAssert.AreEqual(3, binIndex); - // We should have a recordSize > min size record in the bin, to test wrapping. + // We should have a addRecordSize > min smallSize recordPtr in the bin, to test wrapping. ClassicAssert.AreNotEqual(0, RevivificationTestUtils.GetSegmentStart(pool, binIndex, recordSize), "SegmentStart should not be 0, to test wrapping"); // Delete functions.expectedInputLength = InitialLength; for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Delete(ref key); - ClassicAssert.IsTrue(status.Found, $"{status} for key {ii}"); - //ClassicAssert.AreEqual(ii + 1, RevivificationTestUtils.GetFreeRecordCount(store), $"mismatched free record count for key {ii}, pt 1"); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); + ClassicAssert.IsTrue(status.Found, $"{status} for ii {ii}"); + //ClassicAssert.AreEqual(ii + 1, RevivificationTestUtils.GetFreeRecordCount(store), $"mismatched free recordPtr count for ii {ii}, pt 1"); } if (deleteDest == DeleteDest.FreeList && waitMode == WaitMode.Wait) { var actualNumRecords = RevivificationTestUtils.GetFreeRecordCount(store); - ClassicAssert.AreEqual(RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords), actualNumRecords, $"mismatched free record count"); + ClassicAssert.AreEqual(RevivificationTestUtils.GetRevivifiableRecordCount(store, NumRecords), actualNumRecords, $"mismatched free recordPtr count"); } // Revivify functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); long tailAddress = store.Log.TailAddress; SpanByteAndMemory output = new(); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); output.Memory?.Dispose(); if (deleteDest == DeleteDest.FreeList && waitMode == WaitMode.Wait && tailAddress != store.Log.TailAddress) @@ -1604,9 +1496,9 @@ public unsafe void LiveBinWrappingTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] U { var freeRecs = RevivificationTestUtils.GetFreeRecordCount(store); if (expectedReviv) - ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, $"failed to revivify record for key {ii}, freeRecs {freeRecs}"); + ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, $"failed to revivify recordPtr for ii {ii}, freeRecs {freeRecs}"); else - ClassicAssert.Less(tailAddress, store.Log.TailAddress, $"Unexpectedly revivified record for key {ii}, freeRecs {freeRecs}"); + ClassicAssert.Less(tailAddress, store.Log.TailAddress, $"Unexpectedly revivified recordPtr for ii {ii}, freeRecs {freeRecs}"); } } } @@ -1621,16 +1513,15 @@ public unsafe void LiveBinWrappingTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] U [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] + [Explicit("Revivifiable boundary has changed")] public void LiveBinWrappingNoRevivTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp, [Values(RevivificationEnabled.NoReviv)] RevivificationEnabled revivEnabled) { // For a comparison to the reviv version above. Populate(); - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); for (var iter = 0; iter < 100; ++iter) { @@ -1638,24 +1529,27 @@ public void LiveBinWrappingNoRevivTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] U functions.expectedInputLength = InitialLength; for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(iter == 0 ? InitialLength : InitialLength)); - var status = bContext.Delete(ref key); - ClassicAssert.IsTrue(status.Found, $"{status} for key {ii}, iter {iter}"); + functions.expectedValueLengths.Enqueue(iter == 0 ? InitialLength : InitialLength); + + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); + ClassicAssert.IsTrue(status.Found, $"{status} for ii {ii}, iter {iter}"); } for (var ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); SpanByteAndMemory output = new(); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); - output.Memory?.Dispose(); + _ = updateOp == UpdateOp.Upsert + ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) + : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); + output.Dispose(); } } } @@ -1663,8 +1557,9 @@ public void LiveBinWrappingNoRevivTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] U [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] - public void SimpleOversizeRevivifyTest([Values] DeleteDest deleteDest, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) + public void SimpleOverflowRevivifyTest([Values] DeleteDest deleteDest, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { + Assert.Ignore("Test ignored because SpanByteAllocatore currently does not support overflow."); Populate(); bool stayInChain = deleteDest == DeleteDest.InChain; @@ -1674,30 +1569,27 @@ public void SimpleOversizeRevivifyTest([Values] DeleteDest deleteDest, [Values(U _ = RevivificationTestUtils.SwapFreeRecordPool(store, default); byte chainKey = NumRecords + 1; - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; - Span inputVec = stackalloc byte[OversizeLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[OversizeLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory output = new(); - keyVec.Fill(chainKey); - inputVec.Fill(chainKey); + key.Fill(chainKey); + input.Fill(chainKey); // Oversize records in this test do not go to "next higher" bin (there is no next-higher bin in the default PowersOf2 bins we use) + // and they become an out-of-line pointer. functions.expectedInputLength = OversizeLength; - functions.expectedSingleDestLength = OversizeLength; - functions.expectedConcurrentDestLength = OversizeLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(OversizeLength); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(OversizeLength)); + functions.expectedValueLengths.Enqueue(ObjectIdMap.ObjectIdSize); - // Initial insert of the oversize record - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + // Initial insert of the oversize recordPtr + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); // Delete it - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(OversizeLength)); - var status = bContext.Delete(ref key); + functions.expectedValueLengths.Enqueue(OversizeLength); + var status = bContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); ClassicAssert.IsTrue(status.Found, status.ToString()); if (!stayInChain) RevivificationTestUtils.WaitForRecords(store, want: true); @@ -1705,8 +1597,8 @@ public void SimpleOversizeRevivifyTest([Values] DeleteDest deleteDest, [Values(U var tailAddress = store.Log.TailAddress; // Revivify in the chain. Because this is oversize, the expectedFullValueLength remains the same - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(OversizeLength)); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(ref key, ref input, ref input, ref output) : bContext.RMW(ref key, ref input); + functions.expectedValueLengths.Enqueue(OversizeLength); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output) : bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } @@ -1724,70 +1616,60 @@ public void SimplePendingOpsRevivifyTest([Values(CollisionRange.None)] Collision long tailAddress = PrepareDeletes(stayInChain: false, delAboveRO, FlushMode.OnDisk, collisionRange); // We always want freelist for this test. - var pool = store.RevivificationManager.FreeRecordPool; + var pool = store.RevivificationManager.freeRecordPool; ClassicAssert.IsTrue(RevivificationTestUtils.HasRecords(pool)); SpanByteAndMemory output = new(); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - functions.expectedSingleFullValueLength = functions.expectedConcurrentFullValueLength = RoundUpSpanByteFullValueLength(InitialLength); - // Use a different key below RO than we deleted; this will go pending to retrieve it - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + // Use a different ii below RO than we deleted; this will go pending to retrieve it + Span key = stackalloc byte[KeyLength]; if (pendingOp == PendingOp.Read) { - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[InitialLength]; - keyVec.Fill(targetRO); - inputVec.Fill(targetRO); + key.Fill(targetRO); + input.Fill(targetRO); functions.expectedInputLength = InitialLength; - functions.expectedSingleDestLength = InitialLength; - functions.expectedConcurrentDestLength = InitialLength; - var spanSlice = inputVec[..InitialLength]; - var inputSlice = SpanByte.FromPinnedSpan(spanSlice); + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input[..InitialLength]); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); - var status = bContext.Read(ref key, ref inputSlice, ref output); + functions.expectedValueLengths.Enqueue(InitialLength); + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, ref output); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePending(wait: true); ClassicAssert.IsTrue(functions.readCcCalled); } else if (pendingOp == PendingOp.RMW) { - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); - keyVec.Fill(targetRO); - inputVec.Fill(targetRO); + key.Fill(targetRO); + input.Fill(targetRO); - functions.expectedUsedValueLengths.Enqueue(SpanByteTotalSize(InitialLength)); + functions.expectedValueLengths.Enqueue(InitialLength); - _ = bContext.RMW(ref key, ref input); + _ = bContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); _ = bContext.CompletePending(wait: true); ClassicAssert.IsTrue(functions.rmwCcCalled); } ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); } } - - [AllureNUnit] [TestFixture] - class RevivificationObjectTests : AllureTestBase + class RevivificationObjectTests : TestBase { const int NumRecords = 1000; internal const int ValueMult = 1_000_000; - private MyFunctions functions; - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TestObjectFunctions functions; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; private IDevice objlog; @@ -1804,15 +1686,15 @@ public void Setup() LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 22, + LogMemorySize = 1L << 22, PageSize = 1L << 12, - RevivificationSettings = RevivificationSettings.DefaultFixedLength - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer()) + RevivificationSettings = RevivificationSettings.PowerOf2Bins + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - functions = new MyFunctions(); - session = store.NewSession(functions); + functions = new TestObjectFunctions(); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -1833,11 +1715,11 @@ public void TearDown() void Populate() { - for (int key = 0; key < NumRecords; key++) + for (int ii = 0; ii < NumRecords; ii++) { - var keyObj = new MyKey { key = key }; - var valueObj = new MyValue { value = key + ValueMult }; - var status = bContext.Upsert(keyObj, valueObj); + var key = new TestObjectKey { key = ii }; + var valueObj = new TestObjectValue { value = ii + ValueMult }; + var status = bContext.Upsert(key, valueObj); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } } @@ -1851,101 +1733,110 @@ public void SimpleObjectTest([Values] DeleteDest deleteDest, [Values(UpdateOp.Up var deleteKey = RevivificationTestUtils.GetMinRevivifiableKey(store, NumRecords); var tailAddress = store.Log.TailAddress; - _ = bContext.Delete(new MyKey { key = deleteKey }); + var key = new TestObjectKey { key = deleteKey }; + _ = bContext.Delete(key); ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress); var updateKey = deleteDest == DeleteDest.InChain ? deleteKey : NumRecords + 1; - var key = new MyKey { key = updateKey }; - var value = new MyValue { value = key.key + ValueMult }; - var input = new MyInput { value = value.value }; + key = new TestObjectKey { key = updateKey }; + var value = new TestObjectValue { value = key.key + ValueMult }; + var input = new TestObjectInput { value = value.value }; RevivificationTestUtils.WaitForRecords(store, want: true); - ClassicAssert.IsTrue(RevivificationTestUtils.HasRecords(store.RevivificationManager.FreeRecordPool), "Expected a free record after delete and WaitForRecords"); + ClassicAssert.IsTrue(RevivificationTestUtils.HasRecords(store.RevivificationManager.freeRecordPool), "Expected a free recordPtr after delete and WaitForRecords"); - _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(key, value) : bContext.RMW(key, input); + _ = updateOp == UpdateOp.Upsert ? bContext.Upsert(key, value) : bContext.RMW(key, ref input); RevivificationTestUtils.WaitForRecords(store, want: false); - ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, "Expected tail address not to grow (record was revivified)"); + ClassicAssert.AreEqual(tailAddress, store.Log.TailAddress, "Expected tail address not to grow (recordPtr was revivified)"); } } - - [AllureNUnit] [TestFixture] - class RevivificationSpanByteStressTests : AllureTestBase + class RevivificationSpanByteStressTests : TestBase { const int KeyLength = 10; const int InitialLength = 50; internal class RevivificationStressFunctions : SpanByteFunctions { - internal IKeyComparer keyComparer; // non-null if we are doing key comparisons (and thus expectedKey is non-default) - internal SpanByte expectedKey = default; // Set for each operation by the calling thread + internal IKeyComparer keyComparer; // non-null if we are doing ii comparisons (and thus expectedKey is non-default) + internal TestSpanByteKey expectedKey = default; // Set for each operation by the calling thread internal bool isFirstLap = true; // For first - internal RevivificationStressFunctions(IKeyComparer keyComparer) => this.keyComparer = keyComparer; + internal RevivificationStressFunctions(IKeyComparer keyComparer) => this.keyComparer = keyComparer; [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void VerifyKey(ref SpanByte functionsKey) + private void VerifyKey(TKey functionsKey) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { if (keyComparer is not null) - ClassicAssert.IsTrue(keyComparer.Equals(ref expectedKey, ref functionsKey)); + ClassicAssert.IsTrue(keyComparer.Equals(expectedKey, functionsKey)); } - private void VerifyKeyAndValue(ref SpanByte functionsKey, ref SpanByte functionsValue) + private void VerifyKeyAndValue(TKey functionsKey, ReadOnlySpan functionsValue) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { if (keyComparer is not null) - ClassicAssert.IsTrue(keyComparer.Equals(ref expectedKey, ref functionsKey), "functionsKey does not equal expectedKey"); + ClassicAssert.IsTrue(keyComparer.Equals(expectedKey, functionsKey), "functionsKey does not equal expectedKey"); - // Even in CompletePending(), we can verify internal consistency of key/value + // Even in CompletePending(), we can verify internal consistency of ii/value int valueOffset = 0, valueLengthRemaining = functionsValue.Length; - ClassicAssert.Less(functionsKey.Length, valueLengthRemaining); + ClassicAssert.Less(functionsKey.KeyBytes.Length, valueLengthRemaining); while (valueLengthRemaining > 0) { - var compareLength = Math.Min(functionsKey.Length, valueLengthRemaining); - Span valueSpan = functionsValue.AsSpan().Slice(valueOffset, compareLength); - Span keySpan = functionsKey.AsSpan()[..compareLength]; - ClassicAssert.IsTrue(valueSpan.SequenceEqual(keySpan), $"functionsValue (offset {valueOffset}, len {compareLength}: {SpanByte.FromPinnedSpan(valueSpan)}) does not match functionsKey ({SpanByte.FromPinnedSpan(keySpan)})"); + var compareLength = Math.Min(functionsKey.KeyBytes.Length, valueLengthRemaining); + var valueSpan = functionsValue.Slice(valueOffset, compareLength); + var keySpan = functionsKey.KeyBytes[..compareLength]; + ClassicAssert.IsTrue(valueSpan.SequenceEqual(keySpan), $"functionsValue (offset {valueOffset}, len {compareLength}: {valueSpan.ToShortString()}) does not match functionsKey ({keySpan.ToShortString()})"); valueOffset += compareLength; valueLengthRemaining -= compareLength; } } - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - VerifyKey(ref key); - return base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); + VerifyKey(logRecord); + return base.InitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); } - public override bool ConcurrentWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - VerifyKeyAndValue(ref key, ref dst); - return base.ConcurrentWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, ref recordInfo); + VerifyKeyAndValue(logRecord, srcValue); + return base.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo); } - public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - VerifyKey(ref key); - return base.InitialUpdater(ref key, ref input, ref newValue, ref output, ref rmwInfo, ref recordInfo); + VerifyKey(logRecord); + return logRecord.TrySetValueSpanAndPrepareOptionals(input.ReadOnlySpan, in sizeInfo); } - public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - VerifyKeyAndValue(ref key, ref oldValue); - return base.CopyUpdater(ref key, ref input, ref oldValue, ref newValue, ref output, ref rmwInfo, ref recordInfo); + VerifyKeyAndValue(srcLogRecord, srcLogRecord.ValueSpan); + return dstLogRecord.TrySetValueSpanAndPrepareOptionals(srcLogRecord.ValueSpan, in sizeInfo); } - public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - VerifyKeyAndValue(ref key, ref value); - return base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + VerifyKeyAndValue(logRecord, logRecord.ValueSpan); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return logRecord.TrySetValueSpanAndPrepareOptionals(input.ReadOnlySpan, in sizeInfo); } - public override bool SingleDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) - => base.SingleDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + public override bool InitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + => base.InitialDeleter(ref logRecord, ref deleteInfo); - public override unsafe bool ConcurrentDeleter(ref SpanByte key, ref SpanByte value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) - => base.ConcurrentDeleter(ref key, ref value, ref deleteInfo, ref recordInfo); + public override bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) + => base.InPlaceDeleter(ref logRecord, ref deleteInfo); } const int NumRecords = 200; @@ -1954,10 +1845,11 @@ public override unsafe bool ConcurrentDeleter(ref SpanByte key, ref SpanByte val RevivificationStressFunctions functions; RevivificationSpanByteComparer comparer; - private TsavoriteKV> store; - private ClientSession> session; - private BasicContext> bContext; + private TsavoriteKV> store; + private ClientSession> session; + private BasicContext> bContext; private IDevice log; + private ArtificialFreeBinAllocator artificialFreeBinAllocator; [SetUp] public void Setup() @@ -1975,20 +1867,23 @@ public void Setup() } } + if (TestContext.CurrentContext.Test.Name.StartsWith("Artificial")) + artificialFreeBinAllocator = new(maxRecords: 10); + comparer = new RevivificationSpanByteComparer(collisionRange); store = new(new() { IndexSize = 1L << 24, LogDevice = log, PageSize = 1L << 17, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, RevivificationSettings = RevivificationSettings.PowerOf2Bins - }, StoreFunctions.Create(comparer, SpanByteRecordDisposer.Instance) + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); functions = new RevivificationStressFunctions(keyComparer: null); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -2001,32 +1896,130 @@ public void TearDown() store = null; log?.Dispose(); log = null; + artificialFreeBinAllocator?.Dispose(); + artificialFreeBinAllocator = null; OnTearDown(); } - unsafe void Populate() + void Populate() { - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); SpanByteAndMemory output = new(); for (int ii = 0; ii < NumRecords; ++ii) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - var status = bContext.Upsert(ref key, ref input, ref input, ref output); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan, input, ref output); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } } + public enum WrapMode { Wrap, NoWrap }; + const int TakeRecordSize = 40; // Just below AddRecordSize + const int AddRecordSize = 48; // smallSize doesn't matter in this test, but must be a multiple of 8 + const int AddressIncrement = 1_000_000; // must be > ReadOnlyAddress + /// + /// For the artificial bin tests, the LogRecord needs only a RecordInfo and a RecordDataHeader; no actual data operations are done, + /// only "get allocated smallSize", so we can reuse LogRecords of the same "record smallSize". + /// + [StructLayout(LayoutKind.Sequential, Pack = 1)] + internal unsafe struct RecordStub + { + internal static int Size => sizeof(RecordStub); + + internal RecordInfo recordInfo; + + // All recordSizes in this test fit into a single byte, so the RecordDataHeader is less than a long. + readonly long headerWord; + + // All recordSizes in this test fit into a single byte, so the RecordDataHeader is less than a long and thus the key will cross the + // boundary into the first long. + private readonly long l1, l2, l3, l4, l5, l6, l7; + + /// Create an in-place initialization of a stubbed LogRecord + /// Record address of 'this', from the pinned array + /// Size of the recordPtr + internal static void Initialize(RecordStub* recordPtr, int recordSize) + { + recordPtr->recordInfo = RecordInfo.InitialValid; + + const int DefaultKeySize = sizeof(long); + var sizeInfo = new RecordSizeInfo() + { + FieldInfo = new() + { + KeySize = DefaultKeySize, + ValueSize = recordSize - DefaultKeySize - RecordInfo.Size - RecordDataHeader.MinHeaderBytes + } + }; + sizeInfo.SetKeyIsInline(); + sizeInfo.SetValueIsInline(); + + Assert.That(sizeInfo.InlineValueSize > 0, $"RecordSize {recordSize} is too small; sizeInfo.InlineValueSize {sizeInfo.InlineValueSize} must be greater than zero"); + sizeInfo.CalculateSizes(sizeInfo.FieldInfo.KeySize, sizeInfo.FieldInfo.ValueSize); + + // We don't use the key in these artificial bin tests, but verify we stored the address (it is a useful debugging tool). + long key = (long)recordPtr; + var logRecord = new LogRecord((long)recordPtr); + logRecord.InitializeRecord(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), in sizeInfo); + Assert.That(logRecord.Key.AsRef(), Is.EqualTo(key)); + + var dataHeader = new RecordDataHeader((byte*)&recordPtr->headerWord); + Assert.That(dataHeader.GetActualRecordSize(recordPtr->recordInfo), Is.EqualTo(sizeInfo.ActualInlineRecordSize)); + Assert.That(dataHeader.GetAllocatedRecordSize(), Is.EqualTo(sizeInfo.AllocatedInlineRecordSize)); + } + } + + unsafe class ArtificialFreeBinAllocator : IDisposable + { + RecordStub* records; + readonly int maxRecords; + int nextFreeRecord = 0; + + internal ArtificialFreeBinAllocator(int maxRecords) + { + this.maxRecords = maxRecords; + var bufferSizeInBytes = (nuint)(sizeof(RecordStub) * maxRecords); + records = (RecordStub*)NativeMemory.AlignedAlloc(bufferSizeInBytes, Constants.kCacheLineBytes); + NativeMemory.Clear(records, bufferSizeInBytes); + } + + internal RecordStub* AllocateRecordAndInitializeSize(int recordSize) + { + Assert.That(nextFreeRecord, Is.LessThan(maxRecords + 1), $"ArtificialFreeBinAllocator out of records (maxRecords {maxRecords})"); + Assert.That(recordSize, Is.LessThanOrEqualTo(RecordStub.Size), $"RecordSize {recordSize} exceeds RecordStub.Size {RecordStub.Size}"); + var recordPtr = records + nextFreeRecord++; + RecordStub.Initialize(recordPtr, recordSize); + return recordPtr; + } + + internal LogRecord AllocateLogRecord(int recordSize) + { + var recordInfoAndHeader = AllocateRecordAndInitializeSize(recordSize); + var logRecord = new LogRecord((long)recordInfoAndHeader); + Assert.That(logRecord.ActualSize, Is.EqualTo(recordSize), "Allocated LogRecord has unexpected smallSize"); + Assert.That(logRecord.AllocatedSize, Is.EqualTo(RoundUp(recordSize, Constants.kRecordAlignment)), "Allocated LogRecord has unexpected aligned smallSize"); + return logRecord; + } + + public void Dispose() + { + if (records != null) + { + NativeMemory.AlignedFree(records); + records = null; + } + } + } + [Test] [Category(RevivificationCategory)] [TestCase(20, 1, 1)] @@ -2036,10 +2029,9 @@ unsafe void Populate() //[Repeat(100)] public void ArtificialFreeBinThreadStressTest(int numIterations, int numAddThreads, int numTakeThreads) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); const int numRecordsPerThread = 1000; - const int recordSize = 48; // size doesn't matter in this test, but must be a multiple of 8 + + // Set up the fake sizeInfo and logRecord; they only have to return lengths. int maxRecords = numRecordsPerThread * numAddThreads; int numTotalThreads = numAddThreads + numTakeThreads; const long Unadded = 0; @@ -2049,7 +2041,7 @@ public void ArtificialFreeBinThreadStressTest(int numIterations, int numAddThrea // For this test we are bypassing the FreeRecordPool in store. var binDef = new RevivificationBin() { - RecordSize = recordSize, + RecordSize = AddRecordSize, NumberOfRecords = maxRecords }; var flags = new long[maxRecords]; @@ -2097,6 +2089,8 @@ void endIteration() $"maxRec/taken {maxRecords}/{totalTaken}, strayflags {strayFlags.Count}, strayRecords {strayRecords.Count}, iteration {iteration}"); } + var logRecord_AddSize = artificialFreeBinAllocator.AllocateLogRecord(AddRecordSize); + void runAddThread(int tid) { RevivificationStats revivStats = new(); @@ -2106,7 +2100,7 @@ void runAddThread(int tid) var flag = flags[addressBase]; ClassicAssert.AreEqual(Unadded, flag, $"Invalid flag {flag} trying to add addressBase {addressBase}, tid {tid}, iteration {iteration}"); flags[addressBase] = 1; - ClassicAssert.IsTrue(freeRecordPool.TryAdd(addressBase + AddressIncrement, recordSize, ref revivStats), $"Failed to add addressBase {addressBase}, tid {tid}, iteration {iteration}"); + ClassicAssert.IsTrue(freeRecordPool.TryAdd(addressBase + AddressIncrement, ref logRecord_AddSize, ref revivStats), $"Failed to add addressBase {addressBase}, tid {tid}, iteration {iteration}"); } } @@ -2115,7 +2109,7 @@ void runTakeThread(int tid) RevivificationStats revivStats = new(); while (totalTaken < maxRecords) { - if (freeRecordPool.bins[0].TryTake(recordSize, 0, store, out long address, ref revivStats)) + if (freeRecordPool.bins[0].TryTake(AddRecordSize, 0, store, out long address, ref revivStats)) { var addressBase = address - AddressIncrement; var prevFlag = Interlocked.CompareExchange(ref flags[addressBase], RemovedBase + tid, Added); @@ -2147,7 +2141,7 @@ void runTakeThread(int tid) try { - var timeoutSec = 5; // 5s per iteration should be plenty + var timeoutSec = 10; // 10s per iteration to handle slower CI environments ClassicAssert.IsTrue(Task.WaitAll([.. tasks], TimeSpan.FromSeconds(timeoutSec)), $"Task timeout at {timeoutSec} sec, maxRec/taken {maxRecords}/{totalTaken}, iteration {iteration}"); endIteration(); } @@ -2169,19 +2163,21 @@ void runTakeThread(int tid) [Test] [Category(RevivificationCategory)] [Category(SmokeTestCategory)] - public unsafe void ArtificialSimpleTest() + public void ArtificialSimpleTest() { var binDef = new RevivificationBin() { - RecordSize = TakeSize + 8, + RecordSize = TakeRecordSize + Constants.kRecordAlignment, NumberOfRecords = 64, BestFitScanLimit = RevivificationBin.UseFirstFit }; var freeRecordPool = RevivificationTestUtils.CreateSingleBinFreeRecordPool(store, binDef); + var logRecord_TakeSize = artificialFreeBinAllocator.AllocateLogRecord(TakeRecordSize); + RevivificationStats revivStats = new(); - ClassicAssert.IsTrue(freeRecordPool.TryAdd(AddressIncrement + 1, TakeSize, ref revivStats)); - ClassicAssert.IsTrue(freeRecordPool.TryTake(TakeSize, minAddress: AddressIncrement, out var address, ref revivStats)); + ClassicAssert.IsTrue(freeRecordPool.TryAdd(AddressIncrement + 1, ref logRecord_TakeSize, ref revivStats)); + ClassicAssert.IsTrue(freeRecordPool.TryTake(TakeRecordSize, minAddress: AddressIncrement, out var address, ref revivStats)); ClassicAssert.AreEqual(AddressIncrement + 1, address, "out address"); ClassicAssert.AreEqual(1, revivStats.successfulAdds, "Successful Adds"); @@ -2189,27 +2185,33 @@ public unsafe void ArtificialSimpleTest() _ = revivStats.Dump(); } - public enum WrapMode { Wrap, NoWrap }; - const int TakeSize = 40; - - private FreeRecordPool> CreateBestFitTestPool(int scanLimit, WrapMode wrapMode, ref RevivificationStats revivStats) + private FreeRecordPool> CreateBestFitTestPool(int scanLimit, WrapMode wrapMode, ref RevivificationStats revivStats) { + // "Wrap Mode" means we are going to split the segment for our Take records across the end of the bin, wrapping around to the beginning. + // So e.g. for the 64-record bin, we'll put 62 "don't want" records in; our segment start for the Take records will be before that (currently 48) + // and will scan forward to (currently) slot 62. This means a small range of record sizes will let us test the wrap for both TakeSize + // and TakeSize + Constants.kRecordAlignment. + // For non-Wrap, our best-fit test requires that TakeSize and TakeSize + Constants.kRecordAlignment start on the same segment; + // and the bin record size range will be from 16 (see FreeRecordPool.cs) to the RecordSize we specify here. So make the RecordSize + // large, for coarse-grained chunks, so both TakeSize and TakeSize + Constants.kRecordAlignment map to the same segment. var binDef = new RevivificationBin() { - RecordSize = TakeSize + 8, + RecordSize = TakeRecordSize + (wrapMode == WrapMode.Wrap ? Constants.kRecordAlignment : 128), NumberOfRecords = 64, BestFitScanLimit = scanLimit }; var freeRecordPool = RevivificationTestUtils.CreateSingleBinFreeRecordPool(store, binDef); - const int minAddress = AddressIncrement - 10; int expectedAdds = 0, expectedTakes = 0; if (wrapMode == WrapMode.Wrap) { + int minAddress = AddressIncrement - freeRecordPool.bins[0].recordCount - 10; + // Add too-small records to wrap around the end of the bin records. Use lower addresses so we don't mix up the "real" results. - const int smallSize = TakeSize - 4; + const int smallSize = 22; // min required size for 8-byte key is 22 + var logRecord_smallSize = artificialFreeBinAllocator.AllocateLogRecord(smallSize); for (var ii = 0; ii < freeRecordPool.bins[0].recordCount - 2; ++ii, ++expectedAdds) - ClassicAssert.IsTrue(freeRecordPool.TryAdd(minAddress + ii + 1, smallSize, ref revivStats)); + ClassicAssert.IsTrue(freeRecordPool.TryAdd(minAddress + ii + 1, ref logRecord_smallSize, ref revivStats)); // Now take out the four at the beginning. for (var ii = 0; ii < 4; ++ii, ++expectedTakes) @@ -2217,12 +2219,16 @@ private FreeRecordPool tasks = []; // Task rather than Thread for propagation of exception. @@ -2337,7 +2346,7 @@ unsafe void runThread(int tid) } Task.WaitAll([.. tasks]); - ClassicAssert.IsTrue(counter == 0); + Assert.That(counter, Is.EqualTo(0)); } [Test] @@ -2345,46 +2354,41 @@ unsafe void runThread(int tid) //[Repeat(3000)] public void LiveThreadContentionOnOneRecordTest([Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - const int numIterations = 2000; const int numDeleteThreads = 5, numUpdateThreads = 5; const int keyRange = numDeleteThreads; - unsafe void runDeleteThread(int tid) + void runDeleteThread(int tid) { Random rng = new(tid * 101); - using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); + using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); var localbContext = localSession.BasicContext; - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; for (var iteration = 0; iteration < numIterations; ++iteration) { for (var ii = tid; ii < NumRecords; ii += numDeleteThreads) { var kk = rng.Next(keyRange); - keyVec.Fill((byte)kk); - _ = localbContext.Delete(key); + key.Fill((byte)kk); + _ = localbContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); } } } - unsafe void runUpdateThread(int tid) + void runUpdateThread(int tid) { - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); Random rng = new(tid * 101); RevivificationStressFunctions localFunctions = new(keyComparer: comparer); - using var localSession = store.NewSession(localFunctions); + using var localSession = store.NewSession(localFunctions); var localbContext = localSession.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) @@ -2392,11 +2396,11 @@ unsafe void runUpdateThread(int tid) for (var ii = tid; ii < NumRecords; ii += numUpdateThreads) { var kk = rng.Next(keyRange); - keyVec.Fill((byte)kk); - inputVec.Fill((byte)kk); + key.Fill((byte)kk); + input.Fill((byte)kk); - localSession.functions.expectedKey = key; - _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(key, input) : localbContext.RMW(key, input); + localSession.functions.expectedKey = TestSpanByteKey.FromPinnedSpan(PinnedSpanByte.FromPinnedSpan(key)); + _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), input) : localbContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); localSession.functions.expectedKey = default; } @@ -2429,45 +2433,39 @@ public enum ThreadingPattern { SameKeys, RandomKeys }; public void LiveFreeListThreadStressTest([Values] CollisionRange collisionRange, [Values] ThreadingPattern threadingPattern, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - int numIterations = 100; const int numDeleteThreads = 5, numUpdateThreads = 5; - unsafe void runDeleteThread(int tid) + void runDeleteThread(int tid) { Random rng = new(tid * 101); - using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); + using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); var localbContext = localSession.BasicContext; - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; for (var iteration = 0; iteration < numIterations; ++iteration) { for (var ii = tid; ii < NumRecords; ii += numDeleteThreads) { var kk = threadingPattern == ThreadingPattern.RandomKeys ? rng.Next(NumRecords) : ii; - keyVec.Fill((byte)kk); - _ = localbContext.Delete(key); + key.Fill((byte)kk); + _ = localbContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); } } } - unsafe void runUpdateThread(int tid) + void runUpdateThread(int tid) { - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); Random rng = new(tid * 101); RevivificationStressFunctions localFunctions = new(keyComparer: comparer); - using var localSession = store.NewSession(localFunctions); + using var localSession = store.NewSession(localFunctions); var localbContext = localSession.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) @@ -2475,11 +2473,11 @@ unsafe void runUpdateThread(int tid) for (var ii = tid; ii < NumRecords; ii += numUpdateThreads) { var kk = threadingPattern == ThreadingPattern.RandomKeys ? rng.Next(NumRecords) : ii; - keyVec.Fill((byte)kk); - inputVec.Fill((byte)kk); + key.Fill((byte)kk); + input.Fill((byte)kk); - localSession.functions.expectedKey = key; - _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(key, input) : localbContext.RMW(key, input); + localSession.functions.expectedKey = TestSpanByteKey.FromPinnedSpan(PinnedSpanByte.FromPinnedSpan(key)); + _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), input) : localbContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); localSession.functions.expectedKey = default; } @@ -2509,54 +2507,48 @@ unsafe void runUpdateThread(int tid) //[Repeat(30)] public void LiveInChainThreadStressTest([Values(CollisionRange.Ten)] CollisionRange collisionRange, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - // Turn off freelist. _ = RevivificationTestUtils.SwapFreeRecordPool(store, default); const int numIterations = 500; const int numDeleteThreads = 5, numUpdateThreads = 5; - unsafe void runDeleteThread(int tid) + void runDeleteThread(int tid) { - using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); + using var localSession = store.NewSession(new RevivificationStressFunctions(keyComparer: null)); var localbContext = localSession.BasicContext; - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[KeyLength]; for (var iteration = 0; iteration < numIterations; ++iteration) { for (var ii = tid; ii < NumRecords; ii += numDeleteThreads) { - keyVec.Fill((byte)ii); - _ = localbContext.Delete(key); + key.Fill((byte)ii); + _ = localbContext.Delete(TestSpanByteKey.FromPinnedSpan(key)); } } } - unsafe void runUpdateThread(int tid) + void runUpdateThread(int tid) { - Span keyVec = stackalloc byte[KeyLength]; - var key = SpanByte.FromPinnedSpan(keyVec); - - Span inputVec = stackalloc byte[InitialLength]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[KeyLength]; + Span input = stackalloc byte[InitialLength]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); RevivificationStressFunctions localFunctions = new RevivificationStressFunctions(keyComparer: null); - using var localSession = store.NewSession(localFunctions); + using var localSession = store.NewSession(localFunctions); var localbContext = localSession.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) { for (var ii = tid; ii < NumRecords; ii += numUpdateThreads) { - keyVec.Fill((byte)ii); - inputVec.Fill((byte)ii); + key.Fill((byte)ii); + input.Fill((byte)ii); - localSession.functions.expectedKey = key; - _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(key, input) : localbContext.RMW(key, input); + localSession.functions.expectedKey = TestSpanByteKey.FromPinnedSpan(PinnedSpanByte.FromPinnedSpan(key)); + _ = updateOp == UpdateOp.Upsert ? localbContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), input) : localbContext.RMW(TestSpanByteKey.FromPinnedSpan(key), ref pinnedInputSpan); localSession.functions.expectedKey = default; } diff --git a/libs/storage/Tsavorite/cs/test/test.recordops/Tsavorite.test.recordops.csproj b/libs/storage/Tsavorite/cs/test/test.recordops/Tsavorite.test.recordops.csproj new file mode 100644 index 00000000000..493794ca137 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recordops/Tsavorite.test.recordops.csproj @@ -0,0 +1,32 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + false + + + diff --git a/libs/storage/Tsavorite/cs/test/CheckpointManagerTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/CheckpointManagerTests.cs similarity index 88% rename from libs/storage/Tsavorite/cs/test/CheckpointManagerTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/CheckpointManagerTests.cs index b0bc5059fee..fa7cda7100d 100644 --- a/libs/storage/Tsavorite/cs/test/CheckpointManagerTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/CheckpointManagerTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -16,12 +15,10 @@ namespace Tsavorite.test { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; [TestFixture] - public class CheckpointManagerTests : AllureTestBase + public class CheckpointManagerTests : TestBase { private readonly Random random = new(0); @@ -49,19 +46,19 @@ public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode) { TestUtils.RecreateDirectory(TestUtils.MethodTestDir); - using var store = new TsavoriteKV( + using var store = new TsavoriteKV( new() { IndexSize = 1L << 16, LogDevice = log, MutableFraction = 1, PageSize = 1L << 10, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, CheckpointManager = checkpointManager - }, StoreFunctions.Create(LongKeyComparer.Instance) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var s = store.NewSession>(new SimpleSimpleFunctions()); + using var s = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = s.BasicContext; var logCheckpoints = new Dictionary(); @@ -71,7 +68,9 @@ public async Task CheckpointManagerPurgeCheck([Values] DeviceMode deviceMode) for (var i = 0; i < 10; i++) { // Do some dummy update - _ = bContext.Upsert(0, random.Next()); + var key = 0L; + var value = (long)random.Next(); + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)); var checkpointType = random.Next(5); Guid result = default; diff --git a/libs/storage/Tsavorite/cs/test/ComponentRecoveryTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ComponentRecoveryTests.cs similarity index 98% rename from libs/storage/Tsavorite/cs/test/ComponentRecoveryTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ComponentRecoveryTests.cs index 0cb4de11d3e..13e2a08d4c0 100644 --- a/libs/storage/Tsavorite/cs/test/ComponentRecoveryTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ComponentRecoveryTests.cs @@ -4,7 +4,6 @@ using System; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,9 +11,8 @@ namespace Tsavorite.test.recovery { - [AllureNUnit] [TestFixture] - public class ComponentRecoveryTests : AllureTestBase + public class ComponentRecoveryTests : TestBase { private static unsafe void Setup_MallocFixedPageSizeRecoveryTest(out int seed, out IDevice device, out int numBucketsToAdd, out long[] logicalAddresses, out ulong numBytesWritten) { diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/LargeObjectTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/LargeObjectTests.cs new file mode 100644 index 00000000000..e01ee15c67a --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/LargeObjectTests.cs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.LargeObjects +{ + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + internal class LargeObjectTests : TestBase + { + [SetUp] + public void Setup() => RecreateDirectory(MethodTestDir); + + [TearDown] + public void TearDown() => OnTearDown(); + + [Test] + [Category("TsavoriteKV")] + [TestCase(CheckpointType.Snapshot, RandomMode.Rng, 10, 400)] // in-mem log with 1 objectlog segment + [TestCase(CheckpointType.Snapshot, RandomMode.NoRng, 10, 200)] // in-mem log with 1 objectlog segment + [TestCase(CheckpointType.Snapshot, RandomMode.Rng, 2000, 18000)] // Eviction to disk with about 4 objectlog segments + [TestCase(CheckpointType.Snapshot, RandomMode.NoRng, 2000, 9000)] // Eviction to disk with 4 objectlog segments + [TestCase(CheckpointType.FoldOver, RandomMode.Rng, 10, 400)] // in-mem log with 1 objectlog segment + [TestCase(CheckpointType.FoldOver, RandomMode.NoRng, 10, 200)] // in-mem log with 1 objectlog segment + [TestCase(CheckpointType.FoldOver, RandomMode.Rng, 2000, 18000)] // Eviction to disk with about 4 objectlog segments + [TestCase(CheckpointType.FoldOver, RandomMode.NoRng, 2000, 9000)] // Eviction to disk with 4 objectlog segments + public async ValueTask LargeObjectTest([Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, RandomMode rngMode, int numObjects, int numItems) + { + Guid token = default; + + // Step 1: Create and populate store. + using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.log"))) + using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.obj.log"))) + using (var store = new TsavoriteKV(CreateKVSettings(log, objlog) + , StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestLargeObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) + using (var session = store.NewSession(new TestLargeObjectFunctions())) + { + var bContext = session.BasicContext; + Random rng = new Random(33); + + for (int key = 0; key < numObjects; key++) + { + var mykey = new TestObjectKey { key = key }; + var value = new TestLargeObjectValue(rngMode == RandomMode.Rng ? 1 + rng.Next(numItems) : numItems); + _ = bContext.Upsert(mykey, value, Empty.Default); + } + + // Validate read before checkpoint + DoRead(session, numObjects, store); + + _ = store.TryInitiateFullCheckpoint(out token, checkpointType); + await store.CompleteCheckpointAsync().ConfigureAwait(false); + } + + // Step 1: Create and recover store. + using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.log"))) + using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "LargeObjectTest.obj.log"))) + using (var store = new TsavoriteKV(CreateKVSettings(log, objlog) + , StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestLargeObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) + { + _ = store.Recover(token); + + using (var session = store.NewSession(new TestLargeObjectFunctions())) + DoRead(session, numObjects, store); + } + + static void DoRead(ClientSession session, + int numObjects, TsavoriteKV store) + { + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Object }; + TestLargeObjectOutput output = new(); + var bContext = session.BasicContext; + + for (int keycnt = 0; keycnt < numObjects; keycnt++) + { + var key = new TestObjectKey { key = keycnt }; + var status = bContext.Read(key, ref input, ref output, Empty.Default); + + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + Assert.That(status.Found, Is.True, $"Read failed for key {keycnt} with status {status}"); + + // Sample every 5th item so it's not quite so slow as this test isn't [Explicit] + for (int i = 0; i < output.valueObject.value.Length; i += 5) + Assert.That(output.valueObject.value[i], Is.EqualTo((byte)(output.valueObject.value.Length + i))); + + // Make sure we test the last item. + Assert.That(output.valueObject.value[^1], Is.EqualTo((byte)(output.valueObject.value.Length * 2 - 1))); + } + } + + static KVSettings CreateKVSettings(IDevice log, IDevice objlog) => new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + PageSize = 1L << 13, // 8 KB + LogMemorySize = 1L << 16, // 64 KB + SegmentSize = 1L << 17, // 128 KB + ObjectLogSegmentSize = 1L << 22, // 4 MB + CheckpointDir = MethodTestDir + }; + } + + [Test] + [Category("TsavoriteKV")] + [TestCase(CheckpointType.Snapshot, RandomMode.Rng, 10, 1_000, 10_000)] // 10 x 40 MB: 10 objects x 1,000 lists x 5,000 (average) 8-byte items = 10 * 1,000 * 40,000 bytes = 400MB (1 segment) + [TestCase(CheckpointType.Snapshot, RandomMode.NoRng, 10, 1_000, 10_000)] // 10 x 40 MB: 10 objects x 1,000 lists x 5,000 8-byte items = 10 * 1,000 * 40,000 bytes = 400MB (1 segment) + [TestCase(CheckpointType.Snapshot, RandomMode.Rng, 1, 20_000, 70_000)] // 1 x 4 GB: 1 object x 20,000 lists x 35,000 (average) 8-byte items = 1 * 20,000 * 280,000 bytes = over 5GB (4 or 5 segments, tests high length byte) + [TestCase(CheckpointType.Snapshot, RandomMode.NoRng, 1, 20_000, 25_000)] // 1 x 4 GB: 1 object x 20,000 lists x 25,000 8-byte items = 1 * 20,000 * 200,000 bytes = just under 4GB (3 segments) + [Explicit("Long running, high-memory test")] + public async ValueTask MultiListObjectTest([Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, RandomMode rngMode, int numObjects, int numLists, int numItems) + { + Guid token = default; + + // Step 1: Create and populate store. + using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MultiListObjectTest.log"))) + using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MultiListObjectTest.obj.log"))) + using (var store = new TsavoriteKV(CreateKVSettings(log, objlog) + , StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestMultiListObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) + using (var session = store.NewSession(new TestMultiListObjectFunctions())) + { + var bContext = session.BasicContext; + Random rng = new Random(33); + + // Create the objects + for (int key = 0; key < numObjects; key++) + { + var mykey = new TestObjectKey { key = key }; + var value = new TestMultiListObjectValue(key, numLists, numItems, rngMode == RandomMode.Rng ? rng : null); + _ = bContext.Upsert(mykey, value, Empty.Default); + } + + // Validate read before checkpoint + DoRead(session, numObjects, store); + + _ = store.TryInitiateFullCheckpoint(out token, checkpointType); + await store.CompleteCheckpointAsync(); + } + + // Step 1: Create and recover store. + using (var log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MultiListObjectTest.log"))) + using (var objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "MultiListObjectTest.obj.log"))) + using (var store = new TsavoriteKV(CreateKVSettings(log, objlog) + , StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestMultiListObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions))) + { + _ = store.Recover(token); + + using (var session = store.NewSession(new TestMultiListObjectFunctions())) + DoRead(session, numObjects, store); + } + + static void DoRead(ClientSession session, + int numOps, TsavoriteKV store) + { + var bContext = session.BasicContext; + + for (int keycnt = 0; keycnt < numOps; keycnt++) + { + TestMultiListObjectInput input = new() + { + objectIndex = keycnt, + listIndex = 12, // TODO: vary listIndex by rng and have multiple. + itemIndex = 24 + }; + TestMultiListObjectOutput output = new(); + + var key = new TestObjectKey { key = keycnt }; + var status = bContext.Read(key, ref input, ref output, Empty.Default); + + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + Assert.That(status.Found, Is.True, $"Read failed for key {keycnt} with status {status}"); + Assert.That(output.oldValue, Is.EqualTo(input.ExpectedOutputValue)); + + for (int i = 0; i < output.valueObject.lists.Length; i++) + { + // Sample every 50th item so it's not too slow + for (var j = 0; j < output.valueObject.lists[i].Count; j += 50) + Assert.That(output.valueObject.lists[i][j], Is.EqualTo(TestMultiListObjectValue.CreateValue(keycnt, i, j))); + + // Make sure we test the last item. + Assert.That(output.valueObject.lists[i][^1], Is.EqualTo(TestMultiListObjectValue.CreateValue(keycnt, i, output.valueObject.lists[i].Count - 1))); + } + } + } + + static KVSettings CreateKVSettings(IDevice log, IDevice objlog) => new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + PageSize = 1L << 13, // 8 KB + LogMemorySize = 1L << 16, // 64 KB + SegmentSize = 1L << 17, // 128 KB + ObjectLogSegmentSize = 1L << 30, // 1 GB + CheckpointDir = MethodTestDir + }; + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/LogResumeTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/LogResumeTests.cs similarity index 98% rename from libs/storage/Tsavorite/cs/test/LogResumeTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/LogResumeTests.cs index d3de8ea4ce9..b9cd0303031 100644 --- a/libs/storage/Tsavorite/cs/test/LogResumeTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/LogResumeTests.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class LogResumeTests : AllureTestBase + internal class LogResumeTests : TestBase { private IDevice device; @@ -25,7 +23,7 @@ public void Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "Tsavoritelog.log"), deleteOnClose: true); + device = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "TsavoriteLog.log"), deleteOnClose: true); } [TearDown] diff --git a/libs/storage/Tsavorite/cs/test/GenericDiskDeleteTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectDiskDeleteTests.cs similarity index 52% rename from libs/storage/Tsavorite/cs/test/GenericDiskDeleteTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ObjectDiskDeleteTests.cs index 374772d5085..50c430b2f3f 100644 --- a/libs/storage/Tsavorite/cs/test/GenericDiskDeleteTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectDiskDeleteTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,24 +10,22 @@ namespace Tsavorite.test { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; [TestFixture] - internal class GenericDiskDeleteTests : AllureTestBase + internal class ObjectDiskDeleteTests : TestBase { - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log, objlog; [SetUp] public void Setup() { DeleteDirectory(MethodTestDir, wait: true); - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericDiskDeleteTests.log"), deleteOnClose: true); - objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericDiskDeleteTests.obj.log"), deleteOnClose: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectDiskDeleteTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectDiskDeleteTests.obj.log"), deleteOnClose: true); store = new(new() { @@ -36,12 +33,12 @@ public void Setup() LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 14, + LogMemorySize = 1L << 14, PageSize = 1L << 9 - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer()) + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new MyFunctionsDelete()); + session = store.NewSession(new TestObjectFunctionsDelete()); bContext = session.BasicContext; } @@ -70,19 +67,19 @@ public void DiskDeleteBasicTest1() var start = store.Log.TailAddress; for (int i = 0; i < totalRecords; i++) { - var _key = new MyKey { key = i }; - var _value = new MyValue { value = i }; - _ = bContext.Upsert(ref _key, ref _value, 0); + var _key = new TestObjectKey { key = i }; + var _value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, _value, 0); } for (int i = 0; i < totalRecords; i++) { - var input = new MyInput(); - var output = new MyOutput(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - if (bContext.Read(ref key1, ref input, ref output, 0).IsPending) + if (bContext.Read(key1, ref input, ref output, 0).IsPending) _ = bContext.CompletePending(true); else ClassicAssert.AreEqual(value.value, output.value.value); @@ -90,17 +87,17 @@ public void DiskDeleteBasicTest1() for (int i = 0; i < totalRecords; i++) { - var key1 = new MyKey { key = i }; - _ = bContext.Delete(ref key1); + var key1 = new TestObjectKey { key = i }; + _ = bContext.Delete(key1); } for (int i = 0; i < totalRecords; i++) { - var input = new MyInput(); - var output = new MyOutput(); - var key1 = new MyKey { key = i }; + var input = new TestObjectInput(); + var output = new TestObjectOutput(); + var key1 = new TestObjectKey { key = i }; - var status = bContext.Read(ref key1, ref input, ref output, 1); + var status = bContext.Read(key1, ref input, ref output, 1); if (status.IsPending) { @@ -111,11 +108,11 @@ public void DiskDeleteBasicTest1() } - using var iter = store.Log.Scan(start, store.Log.TailAddress, ScanBufferingMode.SinglePageBuffering); + using var iter = store.Log.Scan(start, store.Log.TailAddress, DiskScanBufferingMode.SinglePageBuffering); int val = 0; - while (iter.GetNext(out RecordInfo recordInfo, out MyKey key, out MyValue value)) + while (iter.GetNext()) { - if (recordInfo.Tombstone) + if (iter.Info.Tombstone) val++; } ClassicAssert.AreEqual(val, totalRecords); @@ -129,60 +126,60 @@ public void DiskDeleteBasicTest2() const int totalRecords = 2000; for (int i = 0; i < totalRecords; i++) { - var _key = new MyKey { key = i }; - var _value = new MyValue { value = i }; - _ = bContext.Upsert(ref _key, ref _value, 0); + var _key = new TestObjectKey { key = i }; + var _value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, _value, 0); } - var key100 = new MyKey { key = 100 }; - var value100 = new MyValue { value = 100 }; - var key200 = new MyKey { key = 200 }; + var key100 = new TestObjectKey { key = 100 }; + var value100 = new TestObjectValue { value = 100 }; + var key200 = new TestObjectKey { key = 200 }; - _ = bContext.Delete(ref key100); + _ = bContext.Delete(key100); - var input = new MyInput { value = 1000 }; - var output = new MyOutput(); - var status = bContext.Read(ref key100, ref input, ref output, 1); + var input = new TestObjectInput { value = 1000 }; + var output = new TestObjectOutput(); + var status = bContext.Read(key100, ref input, ref output, 1); ClassicAssert.IsFalse(status.Found, status.ToString()); - status = bContext.Upsert(ref key100, ref value100, 0); + status = bContext.Upsert(key100, value100, 0); ClassicAssert.IsTrue(!status.Found, status.ToString()); - status = bContext.Read(ref key100, ref input, ref output, 0); + status = bContext.Read(key100, ref input, ref output, 0); ClassicAssert.IsTrue(status.Found, status.ToString()); ClassicAssert.AreEqual(value100.value, output.value.value); - _ = bContext.Delete(ref key100); - _ = bContext.Delete(ref key200); + _ = bContext.Delete(key100); + _ = bContext.Delete(key200); // This RMW should create new initial value, since item is deleted - status = bContext.RMW(ref key200, ref input, 1); + status = bContext.RMW(key200, ref input, 1); ClassicAssert.IsFalse(status.Found); - status = bContext.Read(ref key200, ref input, ref output, 0); + status = bContext.Read(key200, ref input, ref output, 0); ClassicAssert.IsTrue(status.Found, status.ToString()); ClassicAssert.AreEqual(input.value, output.value.value); // Delete key 200 again - _ = bContext.Delete(ref key200); + _ = bContext.Delete(key200); // Eliminate all records from memory for (int i = 201; i < 2000; i++) { - var _key = new MyKey { key = i }; - var _value = new MyValue { value = i }; - _ = bContext.Upsert(ref _key, ref _value, 0); + var _key = new TestObjectKey { key = i }; + var _value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, _value, 0); } - status = bContext.Read(ref key100, ref input, ref output, 1); + status = bContext.Read(key100, ref input, ref output, 1); ClassicAssert.IsTrue(status.IsPending); _ = bContext.CompletePending(true); // This RMW should create new initial value, since item is deleted - status = bContext.RMW(ref key200, ref input, 1); + status = bContext.RMW(key200, ref input, 1); ClassicAssert.IsTrue(status.IsPending); _ = bContext.CompletePending(true); - status = bContext.Read(ref key200, ref input, ref output, 0); + status = bContext.Read(key200, ref input, ref output, 0); ClassicAssert.IsTrue(status.Found, status.ToString()); ClassicAssert.AreEqual(input.value, output.value.value); } diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIdMapTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIdMapTests.cs new file mode 100644 index 00000000000..2fe20c80641 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIdMapTests.cs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.Objects +{ + /// + /// This also tests and , + /// which in turn tests . + /// + [TestFixture] + class ObjectIdMapTests : TestBase + { + ObjectIdMap map; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir); + map = new(); + } + + [TearDown] + public void TearDown() + { + DeleteDirectory(MethodTestDir); + } + + private void LoadMap(int numThreads, int chaptersPerThread, int pagesPerChapter) + { + Assert.That(map.objectArray.IsInitialized, Is.False); + + void runLoadThread(int tid) + { + // Reduce memory stress by reusing the same object because we are not doing operations on it; it's just a null/not-null indicator in the slot. + var valueObject = new TestObjectValue(); + + for (var chapter = 0; chapter < chaptersPerThread; ++chapter) + { + for (int page = 0; page < pagesPerChapter; ++page) + { + // Assert.That() does reflection and allocates a ConstraintResult class instance, which is slow, so use a bare test to filter for it in inner loops. + var objectId = map.Allocate(); + if (objectId >= map.Count) + Assert.Fail("objectId should be < map.Count"); + map.Set(objectId, valueObject); + } + } + } + + Task[] tasks = new Task[numThreads]; // Task rather than Thread for propagation of exceptions. + for (int t = 0; t < numThreads; t++) + { + var tid = t; + tasks[t] = Task.Factory.StartNew(() => runLoadThread(tid)); + } + Task.WaitAll(tasks); + } + + [Test] + [Category(ObjectIdMapCategory), Category(MultiLevelPageArrayCategory), Category(SmokeTestCategory)] + [Repeat(1000)] // Repeat is an intended part of the test due to thread-timing non-determinism; writing the iteration count would slow things so we don't + public void ObjectIdMapTestStressInitialAllocs([Values(1, 8)] int numThreads) + { + // Focus on the initial allocation which lazily creates the book array. + LoadMap(numThreads, chaptersPerThread: 1, pagesPerChapter: 1); + + Assert.That(map.Count, Is.EqualTo(numThreads)); + + // We don't run this test with enough threads to go beyond this. + Assert.That(map.objectArray.book.Length, Is.EqualTo(MultiLevelPageArray.InitialBookSize)); + } + + [Test] + [Category(ObjectIdMapCategory), Category(MultiLevelPageArrayCategory), Category(SmokeTestCategory)] + //[Repeat(100)] // Repeat is an intended part of the test due to thread-timing non-determinism; writing the iteration count would slow things so we don't + public void ObjectIdMapTestStressAllocAndFree([Values(1, 8)] int numThreads) + { + Assert.That(map.objectArray.IsInitialized, Is.False); + + // Allocate enough to fill past the first MultiLevelPageArray.InitialBookSize chapters. + var chaptersPerThread = MultiLevelPageArray.InitialBookSize / numThreads + 1; + LoadMap(numThreads, chaptersPerThread, pagesPerChapter: MultiLevelPageArray.ChapterSize); + + var allocatedCount = map.Count; + Assert.That(allocatedCount, Is.EqualTo(chaptersPerThread * MultiLevelPageArray.ChapterSize * numThreads)); + Assert.That(map.objectArray.book.Length, Is.GreaterThan(MultiLevelPageArray.InitialBookSize)); + + // Now test the freelist loading. + void runLoadFreeListThread(int tid) + { + Random rng = new(tid); + + // Free() from a thread-specific chapter to threads aren't freeing the same objectId; in actual use, + // we'd Allocate() which does per-thread ownership instead. + for (var page = 0; page < MultiLevelPageArray.ChapterSize; ++page) + { + // After being freed, the slot in the objectVector should be cleared (so objects are freed as early as possible). + var objectId = tid * MultiLevelPageArray.ChapterSize + page; + map.Free(objectId); + Assert.That(map.GetHeapObject(objectId), Is.Null, "map.GetHeapObject(objectId) should be null after Free() pt 1"); + } + } + + Task[] tasks = new Task[numThreads]; // Task rather than Thread for propagation of exceptions. + for (int t = 0; t < numThreads; t++) + { + var tid = t; + tasks[t] = Task.Factory.StartNew(() => runLoadFreeListThread(tid)); + } + Task.WaitAll(tasks); + + Assert.That(map.freeSlots.MaxCount, Is.EqualTo(MultiLevelPageArray.ChapterSize * numThreads), "All freed items should have been added to the the freeList elementArray"); + Assert.That(map.freeSlots.stack.IsNil, Is.False, "All freed items should be in the stack"); + Assert.That(map.freeSlots.freeNodes.IsNil, Is.True, "No freed items should be in the freeList"); + + // Finally, test the freelist allocation. + void runAllocateFromFreeListThread(int tid) + { + Random rng = new(tid); + + // Free() from a thread-specific chapter to threads aren't freeing the same objectId; in actual use, + // we'd Allocate() which does per-thread ownership instead. + for (var page = 0; page < MultiLevelPageArray.ChapterSize; ++page) + { + var objectId = map.Allocate(); + + // The request should have been satisfied from the freeList, not another allocation. + if (objectId >= allocatedCount) + Assert.Fail("objectId should be less than allocatedCount"); + + // Make sure the slot in the objectVector is still cleared. + Assert.That(map.GetHeapObject(objectId), Is.Null, "map.GetHeapObject(objectId) should be null after Free() pt 2"); + } + } + + Array.Clear(tasks); + for (int t = 0; t < numThreads; t++) + { + var tid = t; + tasks[t] = Task.Factory.StartNew(() => runAllocateFromFreeListThread(tid)); + } + Task.WaitAll(tasks); + + Assert.That(map.Count, Is.EqualTo(allocatedCount)); + + Assert.That(map.freeSlots.stack.IsNil, Is.True, "No freed items should be in the stack"); + Assert.That(map.freeSlots.freeNodes.IsNil, Is.False, "All freed items should be in the freeList"); + Assert.That(map.freeSlots.elementArray.Count, Is.EqualTo(MultiLevelPageArray.ChapterSize * numThreads), "No freed items should have been added to the the freeList elementArray"); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectInlineTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectInlineTests.cs new file mode 100644 index 00000000000..d0cd9c43649 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectInlineTests.cs @@ -0,0 +1,457 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using Garnet.test; +using NUnit.Framework; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.Objects +{ + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + internal class ObjectInlineTests : TestBase + { + private TsavoriteKV store; + private IDevice log, objlog; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.obj.log"), deleteOnClose: true); + + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << 15, + PageSize = 1L << 10 + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + objlog?.Dispose(); + objlog = null; + DeleteDirectory(MethodTestDir); + } + + [Test, Category(TsavoriteKVTestCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectAsInlineStructUpsertTest() + { + using var session = store.NewSession(new TestInlineObjectFunctions()); + var bContext = session.BasicContext; + + TestObjectKey key = new() { key = 9999999 }; + TestObjectInput input = new() { value = 23 }; + TestObjectOutput output = default; + + // Overflow<->Inline conversions + + // Start with an inline value. + input.wantValueStyle = TestValueStyle.Inline; + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 24; + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 25; + input.wantValueStyle = TestValueStyle.Overflow; + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 26; + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + // Overflow<->Object conversions + + input.value = 30; + input.wantValueStyle = TestValueStyle.Object; // Overflow -> Object + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 31; + input.wantValueStyle = TestValueStyle.Overflow; // Object -> Overflow + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 32; + input.wantValueStyle = TestValueStyle.Object; // Overflow -> Object again + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + // Object<->Inline conversions + + input.value = 40; + input.wantValueStyle = TestValueStyle.Inline; // Object -> Inline + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 41; + input.wantValueStyle = TestValueStyle.Object; // Inline -> Object + _ = bContext.Upsert(key, ref input, desiredValue: (IHeapObject)null, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + } + + [Test, Category(TsavoriteKVTestCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectAsInlineStructRMWTest() + { + using var session = store.NewSession(new TestInlineObjectFunctions()); + var bContext = session.BasicContext; + + TestObjectKey key = new() { key = 9999999 }; + TestObjectInput input = new() { value = 23 }; + TestObjectOutput output = default; + + // Overflow<->Inline conversions + + // Start with an inline value. + input.wantValueStyle = TestValueStyle.Inline; + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.None)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + var priorSum = input.value; + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(input.value)); + + input.value = 24; + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + input.value = 25; + input.wantValueStyle = TestValueStyle.Overflow; + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + input.value = 26; + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + // Overflow<->Object conversions + + input.value = 30; + input.wantValueStyle = TestValueStyle.Object; // Overflow -> Object + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + input.value = 31; + input.wantValueStyle = TestValueStyle.Overflow; // Object -> Overflow + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + input.value = 32; + input.wantValueStyle = TestValueStyle.Object; // Overflow -> Object again + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Overflow)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + // Object<->Inline conversions + + input.value = 40; + input.wantValueStyle = TestValueStyle.Inline; // Object -> Inline + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + priorSum += input.value; + + input.value = 41; + input.wantValueStyle = TestValueStyle.Object; // Inline -> Object + _ = bContext.RMW(key, ref input, ref output); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Inline)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + + _ = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(output.srcValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.destValueStyle, Is.EqualTo(TestValueStyle.Object)); + Assert.That(output.value.value, Is.EqualTo(priorSum + input.value)); + } + + public class TestInlineObjectFunctions : TestObjectFunctions + { + // Force test of overflow values + const int OverflowValueSize = 1 << (LogSettings.kDefaultMaxInlineValueSizeBits + 1); + byte[] pinnedValueOverflowBytes = GC.AllocateArray(OverflowValueSize, pinned: true); + Span GetOverflowValueSpanByte() => new(pinnedValueOverflowBytes); + + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + Assert.That(sizeInfo.ValueIsInline, Is.EqualTo(logRecord.Info.ValueIsInline), $"Non-IPU mismatch in sizeInfo ({sizeInfo.ValueIsInline}) and dstLogRecord ({logRecord.Info.ValueIsInline}) ValueIsInline in {Utility.GetCurrentMethodName()}"); + return DoWriter(ref logRecord, in sizeInfo, ref input, srcValue: null, ref output); + } + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + // Use the same record for source and dest; DoUpdater does not modify dest until all source info is processed. + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return DoUpdater(in logRecord, ref logRecord, in sizeInfo, input, ref output); + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, ref TestObjectOutput output, ref RMWInfo rmwInfo) + { + Assert.That(sizeInfo.ValueIsInline, Is.EqualTo(dstLogRecord.Info.ValueIsInline), $"Non-IPU mismatch in sizeInfo ({sizeInfo.ValueIsInline}) and dstLogRecord ({dstLogRecord.Info.ValueIsInline}) ValueIsInline in {Utility.GetCurrentMethodName()}"); + return DoUpdater(in srcLogRecord, ref dstLogRecord, in sizeInfo, input, ref output); + } + + private bool DoUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord logRecord, in RecordSizeInfo sizeInfo, TestObjectInput input, ref TestObjectOutput output) + where TSourceLogRecord : ISourceLogRecord + { + Set(ref output.srcValueStyle, srcLogRecord.Info); + SetAndVerify(ref input, ref output.destValueStyle, sizeInfo.ValueIsInline, sizeInfo.ValueIsOverflow); + + // If the value is inline it is a ValueStruct; if it is overflow it is a buffer with the first long set to the desired value. + long srcValue; + if (srcLogRecord.Info.ValueIsInline) + srcValue = (int)srcLogRecord.ValueSpan.AsRef().vfield1; + else if (srcLogRecord.Info.ValueIsOverflow) + { + Assert.That(srcLogRecord.ValueSpan.Length, Is.EqualTo(OverflowValueSize)); + srcValue = (int)srcLogRecord.ValueSpan.AsRef(); + } + else + srcValue = ((TestObjectValue)srcLogRecord.ValueObject).value; + + output.value = srcLogRecord.Info.ValueIsObject ? (TestObjectValue)srcLogRecord.ValueObject : new TestObjectValue { value = (int)srcValue }; + + var result = false; + switch (output.destValueStyle) + { + case TestValueStyle.Inline: + ValueStruct valueStruct = new() { vfield1 = srcValue + input.value, vfield2 = (srcValue + input.value) * 100 }; + result = logRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref valueStruct), in sizeInfo); + break; + case TestValueStyle.Overflow: + Span overflowValue = GetOverflowValueSpanByte(); + overflowValue.AsRef() = srcValue + input.value; + result = logRecord.TrySetValueSpanAndPrepareOptionals(overflowValue, in sizeInfo); + break; + case TestValueStyle.Object: + result = logRecord.TrySetValueObjectAndPrepareOptionals(output.value, in sizeInfo); + break; + default: + Assert.Fail("Unknown value style"); + return false; + } + + if (result) + output.value.value += input.value; + return result; + } + + public override bool InPlaceWriter(ref LogRecord logRecord, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) + { + Set(ref output.srcValueStyle, logRecord.Info); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetUpsertFieldInfo(logRecord, srcValue, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + return DoWriter(ref logRecord, in sizeInfo, ref input, (TestObjectValue)srcValue, ref output); + } + + public override bool Reader(in TSourceLogRecord srcLogRecord, ref TestObjectInput input, ref TestObjectOutput output, ref ReadInfo readInfo) + { + Set(ref output.srcValueStyle, srcLogRecord.Info); + + // If the value is inline it is a ValueStruct; if it is overflow it is a buffer with the first long set to the desired value. + if (srcLogRecord.Info.ValueIsInline) + output.value = new TestObjectValue() { value = (int)srcLogRecord.ValueSpan.AsRef().vfield1 }; + else if (srcLogRecord.Info.ValueIsOverflow) + { + Assert.That(srcLogRecord.ValueSpan.Length, Is.EqualTo(OverflowValueSize)); + unsafe { output.value = new TestObjectValue() { value = (int)srcLogRecord.ValueSpan.AsRef() }; } + } + else + output.value = (TestObjectValue)srcLogRecord.ValueObject; + return true; + } + + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) + { + Assert.That(sizeInfo.ValueIsInline, Is.EqualTo(logRecord.Info.ValueIsInline), $"Non-IPU mismatch in sizeInfo ({sizeInfo.ValueIsInline}) and dstLogRecord ({logRecord.Info.ValueIsInline}) ValueIsInline in {Utility.GetCurrentMethodName()}"); + return DoWriter(ref logRecord, in sizeInfo, ref input, (TestObjectValue)srcValue, ref output); + } + + private bool DoWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, TestObjectValue srcValue, ref TestObjectOutput output) + { + Assert.That(srcValue, Is.Null, "srcValue should be null for these upsert tests; use Input instead"); + output.srcValueStyle = TestValueStyle.None; + SetAndVerify(ref input, ref output.destValueStyle, sizeInfo.ValueIsInline, sizeInfo.ValueIsOverflow); + + output.value = new TestObjectValue { value = input.value }; + switch (output.destValueStyle) + { + case TestValueStyle.Inline: + ValueStruct valueStruct = new() { vfield1 = input.value, vfield2 = input.value * 100 }; + return logRecord.TrySetValueSpanAndPrepareOptionals(SpanByte.FromPinnedVariable(ref valueStruct), in sizeInfo); + case TestValueStyle.Overflow: + Span overflowValue = GetOverflowValueSpanByte(); + overflowValue.AsRef() = input.value; + return logRecord.TrySetValueSpanAndPrepareOptionals(overflowValue, in sizeInfo); + case TestValueStyle.Object: + return logRecord.TrySetValueObjectAndPrepareOptionals(output.value, in sizeInfo); + default: + Assert.Fail("Unknown value style"); + return false; + } + } + + static void Set(ref TestValueStyle style, RecordInfo recordInfo) => Set(ref style, recordInfo.ValueIsInline, recordInfo.ValueIsOverflow); + + static void Set(ref TestValueStyle style, bool isInline, bool isOverflow) + { + style = (isInline, isOverflow) switch + { + (true, false) => TestValueStyle.Inline, + (false, true) => TestValueStyle.Overflow, + _ => TestValueStyle.Object + }; + } + static void SetAndVerify(ref TestObjectInput input, ref TestValueStyle style, bool isInline, bool isOverflow) + { + Set(ref style, isInline, isOverflow); + Assert.That(style, Is.EqualTo(input.wantValueStyle)); + } + + static RecordFieldInfo GetFieldInfo(TKey key, ref TestObjectInput input) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => new() + { + KeySize = key.KeyBytes.Length, + ValueSize = input.wantValueStyle switch + { + TestValueStyle.Inline => ValueStruct.AsSpanByteDataSize, + TestValueStyle.Overflow => OverflowValueSize, + TestValueStyle.Object => ObjectIdMap.ObjectIdSize, + _ => int.MaxValue + }, + ValueIsObject = input.wantValueStyle == TestValueStyle.Object + }; + + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref TestObjectInput input) + => GetFieldInfo(srcLogRecord, ref input); + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref TestObjectInput input) + => GetFieldInfo(key, ref input); + public override unsafe RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref TestObjectInput input) + => GetFieldInfo(key, ref input); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIterationTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIterationTests.cs new file mode 100644 index 00000000000..b9b70927e8a --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectIterationTests.cs @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test +{ + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + internal class ObjectIterationTests : TestBase + { + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; + private IDevice log, objlog; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + // Tests call InternalSetup() + } + + private void InternalSetup(bool largeMemory) + { + // Broke this out as we have different requirements by test. + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectIterationTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectIterationTests.obj.log"), deleteOnClose: true); + + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = 0.1, + LogMemorySize = 1L << (largeMemory ? 25 : 14), + PageSize = 1L << (largeMemory ? 20 : 9) + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + session = store.NewSession(new TestObjectFunctionsDelete()); + bContext = session.BasicContext; + } + + [TearDown] + public void TearDown() + { + session?.Dispose(); + session = null; + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + objlog?.Dispose(); + objlog = null; + + OnTearDown(); + } + + internal struct ObjectPushIterationTestFunctions : IScanIteratorFunctions + { + internal int keyMultToValue; + internal long numRecords; + internal int stopAt; + + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord + { + cursorRecordResult = CursorRecordResult.Accept; // default; not used here + if (keyMultToValue > 0) + ClassicAssert.AreEqual(logRecord.Key.AsRef().key * keyMultToValue, ((TestObjectValue)logRecord.ValueObject).value); + return stopAt != ++numRecords; + } + + public readonly bool OnStart(long beginAddress, long endAddress) => true; + public readonly void OnException(Exception exception, long numberOfRecords) { } + public readonly void OnStop(bool completed, long numberOfRecords) { } + } + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + + public void ObjectIterationBasicTest([Values] ScanIteratorType scanIteratorType) + { + InternalSetup(largeMemory: false); + ObjectPushIterationTestFunctions scanIteratorFunctions = new(); + + const int totalRecords = 2000; + + void iterateAndVerify(int keyMultToValue, int expectedRecs) + { + scanIteratorFunctions.keyMultToValue = keyMultToValue; + scanIteratorFunctions.numRecords = 0; + + if (scanIteratorType == ScanIteratorType.Pull) + { + using var iter = session.Iterate(); + while (iter.GetNext()) + _ = scanIteratorFunctions.Reader(in iter, default, default, out _); + } + else + ClassicAssert.IsTrue(session.Iterate(ref scanIteratorFunctions), $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}"); + + ClassicAssert.AreEqual(expectedRecs, scanIteratorFunctions.numRecords); + } + + // Initial population + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); + } + iterateAndVerify(1, totalRecords); + + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = 2 * i }; + _ = bContext.Upsert(key1, value); + } + iterateAndVerify(2, totalRecords); + + for (int i = totalRecords / 2; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); + } + iterateAndVerify(0, totalRecords); + + for (int i = 0; i < totalRecords; i += 2) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); + } + iterateAndVerify(0, totalRecords); + + for (int i = 0; i < totalRecords; i += 2) + { + var key1 = new TestObjectKey { key = i }; + _ = bContext.Delete(key1); + } + iterateAndVerify(0, totalRecords / 2); + + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = 3 * i }; + _ = bContext.Upsert(key1, value); + } + iterateAndVerify(3, totalRecords); + + store.Log.FlushAndEvict(wait: true); + iterateAndVerify(3, totalRecords); + } + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + + public void ObjectIterationPushStopTest() + { + InternalSetup(largeMemory: false); + ObjectPushIterationTestFunctions scanIteratorFunctions = new(); + + const int totalRecords = 2000; + var start = store.Log.TailAddress; + + void scanAndVerify(int stopAt, bool useScan) + { + scanIteratorFunctions.numRecords = 0; + scanIteratorFunctions.stopAt = stopAt; + if (useScan) + ClassicAssert.IsFalse(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); + else + ClassicAssert.IsFalse(session.Iterate(ref scanIteratorFunctions), $"Failed to terminate push iteration early; numRecords = {scanIteratorFunctions.numRecords}"); + ClassicAssert.AreEqual(stopAt, scanIteratorFunctions.numRecords); + } + + // Initial population + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); + } + + scanAndVerify(42, useScan: true); + scanAndVerify(42, useScan: false); + } + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + //[Repeat(3000)] + //[Explicit("Temporary: accessing a disposed object")] + public void ObjectIterationPushLockTest([Values(1, 2, 4, 8)] int scanThreads, [Values(0, 1, 4)] int updateThreads, [Values] ScanMode scanMode, [Values] bool largeMemory) + { + InternalSetup(largeMemory); + + const int totalRecords = 2000; + var start = store.Log.TailAddress; + + void LocalScan(int i) + { + using var session = store.NewSession(new TestObjectFunctionsDelete()); + ObjectPushIterationTestFunctions scanIteratorFunctions = new(); + + var end = store.Log.TailAddress; + if (scanMode == ScanMode.Scan) + Assert.That(store.Log.Scan(ref scanIteratorFunctions, start, end), Is.True, $"Failed to complete push scan; numRecords = {scanIteratorFunctions.numRecords}, start = {start}, end = {end}"); + else + Assert.That(session.Iterate(ref scanIteratorFunctions), Is.True, $"Failed to complete push iteration; numRecords = {scanIteratorFunctions.numRecords}, start = {start}, end = {end}"); + + // If we are doing Scan with updates and without largeMemory, there will be records appended at the log tail due to not + // being able to do IPU, so the scan count may be > totalRecords. + if (scanMode == ScanMode.Scan && !largeMemory && updateThreads > 0) + Assert.That(scanIteratorFunctions.numRecords, Is.GreaterThanOrEqualTo(totalRecords)); + else + Assert.That(scanIteratorFunctions.numRecords, Is.EqualTo(totalRecords)); + } + + const int keyTag = 0x420000; + + void LocalUpdate(int tid) + { + using var localSession = store.NewSession(new TestObjectFunctionsDelete()); + var localBContext = localSession.BasicContext; + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i + keyTag }; + var value = new TestObjectValue { value = (tid + 1) * i }; + var status = localBContext.Upsert(key1, value); + Assert.That(status.IsPending, Is.False, "Upsert should not go pending"); + } + } + + { // Initial population + for (int i = 0; i < totalRecords; i++) + { + var key1 = new TestObjectKey { key = i + keyTag }; + var value = new TestObjectValue { value = i + 340000 }; + var status = bContext.Upsert(key1, value); + Assert.That(status.IsPending, Is.False, "Upsert should not go pending"); + } + } + + List tasks = []; // Task rather than Thread for propagation of exception. + var numThreads = scanThreads + updateThreads; + for (int t = 0; t < numThreads; t++) + { + var tid = t; + if (t < scanThreads) + tasks.Add(Task.Factory.StartNew(() => LocalScan(tid))); + else + tasks.Add(Task.Factory.StartNew(() => LocalUpdate(tid))); + } + Task.WaitAll([.. tasks]); + } + } + [TestFixture] + internal class ObjectIterationTests2 : TestBase + { + // Per issue #1630, handle 4 pages worth of records with an InsertAll-DeleteAll-ReInsertAll pattern. + public class InsDelIns_ScanIteratorFunctions : IScanIteratorFunctions + { + public const int MaxCount = 64; + public readonly (int Key, int Value)[] ExpectedItems = new (int Key, int Value)[MaxCount]; + public readonly List<(int Key, int Value)> UnexpectedItems = []; + public int Count; + + public bool Reader(in TSourceLogRecord sourceLogRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord + { + ref readonly var key = ref sourceLogRecord.Key.AsRef(); + ref readonly var value = ref sourceLogRecord.ValueSpan.AsRef(); + if (Count < ExpectedItems.Length) + ExpectedItems[Count] = (key, value); + else + UnexpectedItems.Add((key, value)); + + Count++; + if (Count == MaxCount) + { + cursorRecordResult = CursorRecordResult.Accept | CursorRecordResult.EndBatch; + WriteLine($"EndBatch {key},{value}"); + } + else + { + // If this happens, it is likely because we had to go to pending IO and had some pending when we triggered the EndBatch at MaxCount; + // the pending operations are completed which call back to here. + cursorRecordResult = CursorRecordResult.Accept; + WriteLine($"Accept {key},{value}"); + } + return true; + } + public bool OnStart(long beginAddress, long endAddress) + { + Count = 0; + WriteLine("OnStart"); + return true; + } + public void OnException(Exception exception, long numberOfRecords) => WriteLine($"{exception.GetType().Name}: {exception.Message}; numRec {numberOfRecords}"); + public void OnStop(bool completed, long numberOfRecords) => WriteLine("OnStop"); + + // For debugging only + internal static void WriteLine(string s) { } // => Debug.WriteLine(s); + } + + private const int Count = 249; // In the Issue this is 253, but in V2 the presence of PageHeader uses some space and that causes 253 to require another page. + private const long PageSize = 1L << 12; + private const long SegmentSize = PageSize; + + private static void RunTest(IDevice LogDevice) + { + var Settings = new KVSettings + { + IndexSize = 1L << 6, + LogMemorySize = 1L << 13, + PageSize = PageSize, + SegmentSize = SegmentSize, + MutableFraction = 1, + LogDevice = LogDevice, + }; + var StoreFunctions = new StoreFunctions(new SpanByteComparer(), () => null, new DefaultRecordTriggers()); + using var Store = new TsavoriteKV, ObjectAllocator>>( + Settings, StoreFunctions, + static (AllocSettings, StoreFuncs) => new ObjectAllocator>(AllocSettings, StoreFuncs)); + using var ReadAddSession = Store.NewSession>(new SpanByteFunctions(System.Buffers.MemoryPool.Shared)); + + Span keySpan = stackalloc int[1]; + Span valueSpan = stackalloc int[1]; + var key = TestSpanByteKey.FromPinnedSpan(MemoryMarshal.AsBytes(keySpan)); + var value = MemoryMarshal.AsBytes(valueSpan); + + // Insert all + for (var Key = 0; Key < Count; Key++) + { + keySpan[0] = Key; + valueSpan[0] = Key; + _ = ReadAddSession.BasicContext.Upsert(key, value); + } + + // Delete all + for (var Key = 0; Key < Count; Key++) + { + keySpan[0] = Key; + var Status = ReadAddSession.BasicContext.Delete(key, default); + } + + // ReInsert all + for (var Key = 0; Key < Count; Key++) + { + keySpan[0] = Key; + valueSpan[0] = Key; + _ = ReadAddSession.BasicContext.Upsert(key, value); + } + + var ScanIteratorFunctions = new InsDelIns_ScanIteratorFunctions(); + long TotalCount = 0, Cursor = 0; + while (true) + { + _ = ReadAddSession.IterateLookup(ref ScanIteratorFunctions, ref Cursor, resetCursor: false); + TotalCount += ScanIteratorFunctions.Count; + InsDelIns_ScanIteratorFunctions.WriteLine($"while: {TotalCount}"); + + // If we did not get a full batch, we're done + if (ScanIteratorFunctions.Count < InsDelIns_ScanIteratorFunctions.MaxCount) + break; + } + Assert.That(TotalCount, Is.EqualTo(Count)); + Assert.That(ScanIteratorFunctions.UnexpectedItems, Is.Empty, "Unexpected items were sent to Reader(), probably pending items that were in-flight when EndBatch was received"); + } + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + public void InsDelIns_LocalMemory() + { + using var LogDevice = new LocalMemoryDevice(capacity: SegmentSize, sz_segment: SegmentSize, parallelism: 1, sector_size: 64U, latencyMs: 0, fileName: Path.Combine(MethodTestDir, "test.log")); + RunTest(LogDevice); + } + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(SmokeTestCategory)] + public void InsDelIns_MLSD() + { + using var LogDevice = new ManagedLocalStorageDevice(filename: Path.Combine(MethodTestDir, "test.log"), + capacity: SegmentSize, + deleteOnClose: true + ); + RunTest(LogDevice); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/GenericLogCompactionTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogCompactionTests.cs similarity index 52% rename from libs/storage/Tsavorite/cs/test/GenericLogCompactionTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogCompactionTests.cs index 3024a73a4c0..9fe9ebcba71 100644 --- a/libs/storage/Tsavorite/cs/test/GenericLogCompactionTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogCompactionTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,16 +10,14 @@ namespace Tsavorite.test { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; [TestFixture] - internal class GenericLogCompactionTests : AllureTestBase + internal class ObjectLogCompactionTests : TestBase { - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log, objlog; [SetUp] @@ -29,41 +26,30 @@ public void Setup() // Clean up log files from previous test runs in case they weren't cleaned up DeleteDirectory(MethodTestDir, wait: true); - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = 1L << 13, MutableFraction = 0.1, - MemorySize = 1L << 14, + LogMemorySize = 1L << 14, PageSize = 1L << 9 }; - if (TestContext.CurrentContext.Test.Arguments.Length == 0) - { - // Default log creation - log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericLogCompactionTests.log"), deleteOnClose: true); - objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "GenericLogCompactionTests.obj.log"), deleteOnClose: true); - } - else - { - // For this class, deviceType is the only parameter. Using this to illustrate the approach; NUnit doesn't provide metadata for arguments, - // so for multi-parameter tests it is probably better to stay with the "separate SetUp method" approach. - var deviceType = (TestDeviceType)TestContext.CurrentContext.Test.Arguments[0]; - - log = CreateTestDevice(deviceType, Path.Join(MethodTestDir, $"LogCompactBasicTest_{deviceType}.log")); - objlog = CreateTestDevice(deviceType, Path.Join(MethodTestDir, $"LogCompactBasicTest_{deviceType}.obj.log")); - - kvSettings.SegmentSize = 1L << 22; - } + // For this class, compactionType is the first (and currently only) parameter. Using this to illustrate the approach; NUnit doesn't provide metadata for arguments, + // so for multi-parameter tests it is probably better to stay with the "separate SetUp method" approach. + Assert.That(TestContext.CurrentContext.Test.Arguments[0].GetType(), Is.EqualTo(typeof(CompactionType))); + var compactionType = (CompactionType)TestContext.CurrentContext.Test.Arguments[0]; + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, $"LogCompactBasicTest_{compactionType}.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, $"LogCompactBasicTest_{compactionType}.obj.log"), deleteOnClose: true); kvSettings.LogDevice = log; kvSettings.ObjectLogDevice = objlog; store = new(kvSettings - , StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer(), DefaultRecordDisposer.Instance) + , StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(new MyFunctionsDelete()); + session = store.NewSession(new TestObjectFunctionsDelete()); bContext = session.BasicContext; } @@ -88,7 +74,7 @@ public void TearDown() [Category("Smoke")] public void LogCompactBasicTest([Values] CompactionType compactionType) { - MyInput input = new(); + TestObjectInput input = new(); const int totalRecords = 500; long compactUntil = 0; @@ -98,9 +84,9 @@ public void LogCompactBasicTest([Values] CompactionType compactionType) if (i == 250) compactUntil = store.Log.TailAddress; - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, 0); } compactUntil = session.Compact(compactUntil, compactionType); @@ -110,20 +96,16 @@ public void LogCompactBasicTest([Values] CompactionType compactionType) // Read all keys - all should be present for (int i = 0; i < totalRecords; i++) { - MyOutput output = new(); + TestObjectOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, 0); + var status = bContext.Read(key1, ref input, ref output, 0); if (status.IsPending) { _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); - ClassicAssert.IsTrue(completedOutputs.Next()); - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found); - output = completedOutputs.Current.Output; - ClassicAssert.IsFalse(completedOutputs.Next()); - completedOutputs.Dispose(); + (status, output) = GetSinglePendingResult(completedOutputs); } ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.value, output.value.value); @@ -135,7 +117,7 @@ public void LogCompactBasicTest([Values] CompactionType compactionType) [Category("Compaction")] public void LogCompactTestNewEntries([Values] CompactionType compactionType) { - MyInput input = new(); + TestObjectInput input = new(); const int totalRecords = 2000; long compactUntil = 0; @@ -145,17 +127,17 @@ public void LogCompactTestNewEntries([Values] CompactionType compactionType) if (i == 1000) compactUntil = store.Log.TailAddress; - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, 0); } // Put fresh entries for 1000 records for (int i = 0; i < 1000; i++) { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, 0); } store.Log.Flush(true); @@ -169,18 +151,18 @@ public void LogCompactTestNewEntries([Values] CompactionType compactionType) // Read 2000 keys - all should be present for (int i = 0; i < totalRecords; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, 0); + var status = bContext.Read(key1, ref input, ref output, 0); if (status.IsPending) - _ = bContext.CompletePending(true); - else { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value.value, output.value.value); + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + (status, output) = GetSinglePendingResult(completedOutputs); } + ClassicAssert.IsTrue(status.Found); + ClassicAssert.AreEqual(value.value, output.value.value); } } @@ -190,7 +172,7 @@ public void LogCompactTestNewEntries([Values] CompactionType compactionType) [Category("Smoke")] public void LogCompactAfterDeleteTest([Values] CompactionType compactionType) { - MyInput input = new(); + TestObjectInput input = new(); const int totalRecords = 2000; long compactUntil = 0; @@ -200,15 +182,23 @@ public void LogCompactAfterDeleteTest([Values] CompactionType compactionType) if (i == totalRecords / 2) compactUntil = store.Log.TailAddress; - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + var status = bContext.Upsert(key1, value, 0); + if (status.IsPending) + { + _ = bContext.CompletePending(wait: true); + } if (i % 8 == 0) { int j = i / 4; - key1 = new MyKey { key = j }; - _ = bContext.Delete(ref key1); + key1 = new TestObjectKey { key = j }; + var delStatus = bContext.Delete(key1); + if (delStatus.IsPending) + { + _ = bContext.CompletePending(wait: true); + } } } @@ -219,27 +209,26 @@ public void LogCompactAfterDeleteTest([Values] CompactionType compactionType) // Read keys - all should be present for (int i = 0; i < totalRecords; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; int ctx = ((i < 500) && (i % 2 == 0)) ? 1 : 0; - var status = bContext.Read(ref key1, ref input, ref output, ctx); + var status = bContext.Read(key1, ref input, ref output, ctx); if (status.IsPending) - _ = bContext.CompletePending(true); - else { - if (ctx == 0) - { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value.value, output.value.value); - } - else - { - ClassicAssert.IsFalse(status.Found); - } + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + (status, output) = GetSinglePendingResult(completedOutputs); + } + + if (ctx == 0) + { + ClassicAssert.IsTrue(status.Found); + ClassicAssert.AreEqual(value.value, output.value.value); } + else + ClassicAssert.IsFalse(status.Found); } } @@ -249,7 +238,7 @@ public void LogCompactAfterDeleteTest([Values] CompactionType compactionType) public void LogCompactBasicCustomFctnTest([Values] CompactionType compactionType) { - MyInput input = new(); + TestObjectInput input = new(); const int totalRecords = 2000; var compactUntil = 0L; @@ -259,9 +248,9 @@ public void LogCompactBasicCustomFctnTest([Values] CompactionType compactionType if (i == totalRecords / 2) compactUntil = store.Log.TailAddress; - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value, 0); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, 0); } compactUntil = session.Compact(compactUntil, compactionType, default(EvenCompactionFunctions)); @@ -271,29 +260,26 @@ public void LogCompactBasicCustomFctnTest([Values] CompactionType compactionType // Read 2000 keys - all should be present for (var i = 0; i < totalRecords; i++) { - var output = new MyOutput(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + var output = new TestObjectOutput(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; var ctx = (i < (totalRecords / 2) && (i % 2 != 0)) ? 1 : 0; - var status = bContext.Read(ref key1, ref input, ref output, ctx); + var status = bContext.Read(key1, ref input, ref output, ctx); if (status.IsPending) { - _ = bContext.CompletePending(true); + _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); + (status, output) = GetSinglePendingResult(completedOutputs); } - else + + if (ctx == 0) { - if (ctx == 0) - { - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(value.value, output.value.value); - } - else - { - ClassicAssert.IsFalse(status.Found); - } + ClassicAssert.IsTrue(status.Found); + ClassicAssert.AreEqual(value.value, output.value.value); } + else + ClassicAssert.IsFalse(status.Found); } } @@ -306,17 +292,17 @@ public void LogCompactCopyInPlaceCustomFctnTest([Values] CompactionType compacti // Update: irrelevant as session compaction no longer uses Copy/CopyInPlace // This test checks if CopyInPlace returning false triggers call to Copy - using var session = store.NewSession(new MyFunctionsDelete()); + using var session = store.NewSession(new TestObjectFunctionsDelete()); - var key = new MyKey { key = 100 }; - var value = new MyValue { value = 20 }; + var key = new TestObjectKey { key = 100 }; + var value = new TestObjectValue { value = 20 }; - _ = bContext.Upsert(ref key, ref value, 0); + _ = bContext.Upsert(key, value, 0); store.Log.Flush(true); - value = new MyValue { value = 21 }; - _ = bContext.Upsert(ref key, ref value, 0); + value = new TestObjectValue { value = 21 }; + _ = bContext.Upsert(key, value, 0); store.Log.Flush(true); @@ -324,9 +310,9 @@ public void LogCompactCopyInPlaceCustomFctnTest([Values] CompactionType compacti var compactUntil = session.Compact(store.Log.TailAddress, compactionType, compactionFunctions); store.Log.Truncate(); - var input = default(MyInput); - var output = default(MyOutput); - var status = bContext.Read(ref key, ref input, ref output); + var input = default(TestObjectInput); + var output = default(TestObjectOutput); + var status = bContext.Read(key, ref input, ref output); if (status.IsPending) { ClassicAssert.IsTrue(bContext.CompletePendingWithOutputs(out var outputs, wait: true)); @@ -336,14 +322,18 @@ public void LogCompactCopyInPlaceCustomFctnTest([Values] CompactionType compacti ClassicAssert.AreEqual(value.value, output.value.value); } - private class Test2CompactionFunctions : ICompactionFunctions + private class Test2CompactionFunctions : ICompactionFunctions { - public bool IsDeleted(ref MyKey key, ref MyValue value) => false; + public bool IsDeleted(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => false; } - private struct EvenCompactionFunctions : ICompactionFunctions + private struct EvenCompactionFunctions : ICompactionFunctions { - public readonly bool IsDeleted(ref MyKey key, ref MyValue value) => value.value % 2 != 0; + public readonly bool IsDeleted(in TSourceLogRecord logRecord) + where TSourceLogRecord : ISourceLogRecord + => ((TestObjectValue)logRecord.ValueObject).value % 2 != 0; } } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/GenericLogScanTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogScanTests.cs similarity index 65% rename from libs/storage/Tsavorite/cs/test/GenericLogScanTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogScanTests.cs index 2b04bd8129f..4eee51242fc 100644 --- a/libs/storage/Tsavorite/cs/test/GenericLogScanTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectLogScanTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,18 +12,31 @@ namespace Tsavorite.test { // Must be in a separate block so the "using ClassStoreFunctions" is the first line in its namespace declaration. - public class MyObjectComparerModulo : IKeyComparer + public class TestObjectValueComparerModulo : IKeyComparer { readonly long mod; - internal MyObjectComparerModulo(long mod) => this.mod = mod; + internal TestObjectValueComparerModulo(long mod) => this.mod = mod; - public bool Equals(ref MyKey k1, ref MyKey k2) => k1.key == k2.key; + public bool Equals(TFirstKey k1, TSecondKey k2) + where TFirstKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TSecondKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef().key == k2.KeyBytes.AsRef().key; // Force collisions to create a chain - public long GetHashCode64(ref MyKey key) + public long GetHashCode64(TKey key) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { - long hash = Utility.GetHashCode(key.key); + long hash = Utility.GetHashCode(key.KeyBytes.AsRef().key); return mod > 0 ? hash % mod : hash; } } @@ -32,18 +44,16 @@ public long GetHashCode64(ref MyKey key) namespace Tsavorite.test { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; [TestFixture] - internal class GenericLogScanTests : AllureTestBase + internal class ObjectLogScanTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log, objlog; const int TotalRecords = 250; - MyObjectComparerModulo comparer; + TestObjectValueComparerModulo comparer; [SetUp] public void Setup() @@ -56,7 +66,7 @@ public void Setup() { if (arg is HashModulo mod && mod == HashModulo.Hundred) { - comparer = new MyObjectComparerModulo(100); + comparer = new TestObjectValueComparerModulo(100); continue; } } @@ -76,20 +86,18 @@ public void TearDown() OnTearDown(); } - internal struct GenericPushScanTestFunctions : IScanIteratorFunctions + internal struct ObjectPushScanTestFunctions : IScanIteratorFunctions { internal long numRecords; public readonly bool OnStart(long beginAddress, long endAddress) => true; - public bool ConcurrentReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public bool SingleReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; // default; not used here - ClassicAssert.AreEqual(numRecords, key.key, $"log scan 1: key"); - ClassicAssert.AreEqual(numRecords, value.value, $"log scan 1: value"); + ClassicAssert.AreEqual(numRecords, logRecord.Key.AsRef().key, $"log scan 1: key"); + ClassicAssert.AreEqual(numRecords, ((TestObjectValue)logRecord.ValueObject).value, $"log scan 1: value"); ++numRecords; return true; @@ -113,14 +121,15 @@ public void DiskWriteScanBasicTest([Values] TestDeviceType deviceType, [Values] LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 9, - SegmentSize = 1L << 22 - }, StoreFunctions.Create(comparer, () => new MyKeySerializer(), () => new MyValueSerializer()) + SegmentSize = 1L << 18, + ObjectLogSegmentSize = 1L << 22 + }, StoreFunctions.Create(comparer, () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession(new MyFunctions()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; using var s = store.Log.Subscribe(new LogObserver()); @@ -128,25 +137,29 @@ public void DiskWriteScanBasicTest([Values] TestDeviceType deviceType, [Values] var start = store.Log.TailAddress; for (int i = 0; i < TotalRecords; i++) { - var _key = new MyKey { key = i }; - var _value = new MyValue { value = i }; - _ = bContext.Upsert(ref _key, ref _value, Empty.Default); + var _key = new TestObjectKey { key = i }; + var _value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, _value, Empty.Default); if (i % 100 == 0) store.Log.FlushAndEvict(true); } + + // One pass to verify in-memory scan + scanAndVerify(DiskScanBufferingMode.SinglePageBuffering); + store.Log.FlushAndEvict(true); - GenericPushScanTestFunctions scanIteratorFunctions = new(); + ObjectPushScanTestFunctions scanIteratorFunctions = new(); - void scanAndVerify(ScanBufferingMode sbm) + void scanAndVerify(DiskScanBufferingMode sbm) { scanIteratorFunctions.numRecords = 0; if (scanIteratorType == ScanIteratorType.Pull) { using var iter = store.Log.Scan(start, store.Log.TailAddress, sbm); - while (iter.GetNext(out var recordInfo)) - _ = scanIteratorFunctions.SingleReader(ref iter.GetKey(), ref iter.GetValue(), default, default, out _); + while (iter.GetNext()) + _ = scanIteratorFunctions.Reader(in iter, default, default, out _); } else ClassicAssert.IsTrue(store.Log.Scan(ref scanIteratorFunctions, start, store.Log.TailAddress, sbm), "Failed to complete push iteration"); @@ -154,11 +167,11 @@ void scanAndVerify(ScanBufferingMode sbm) ClassicAssert.AreEqual(TotalRecords, scanIteratorFunctions.numRecords); } - scanAndVerify(ScanBufferingMode.SinglePageBuffering); - scanAndVerify(ScanBufferingMode.DoublePageBuffering); + scanAndVerify(DiskScanBufferingMode.SinglePageBuffering); + scanAndVerify(DiskScanBufferingMode.DoublePageBuffering); } - class LogObserver : IObserver> + class LogObserver : IObserver { int val = 0; @@ -171,12 +184,12 @@ public void OnError(Exception error) { } - public void OnNext(ITsavoriteScanIterator iter) + public void OnNext(ITsavoriteScanIterator iter) { - while (iter.GetNext(out _, out MyKey key, out MyValue value)) + while (iter.GetNext()) { - ClassicAssert.AreEqual(val, key.key, $"LogObserver.OnNext: key"); - ClassicAssert.AreEqual(val, value.value, $"LogObserver.OnNext: value"); + ClassicAssert.AreEqual(val, iter.Key.AsRef().key, $"LogObserver.OnNext: key"); + ClassicAssert.AreEqual(val, ((TestObjectValue)iter.ValueObject).value, $"LogObserver.OnNext: value"); val++; } } @@ -196,14 +209,14 @@ public void BlittableScanJumpToBeginAddressTest() LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, PageSize = 1L << 15, SegmentSize = 1L << 18 - }, StoreFunctions.Create(comparer, () => new MyKeySerializer(), () => new MyValueSerializer()) + }, StoreFunctions.Create(comparer, () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession(new MyFunctions()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; const int numRecords = 200; @@ -217,42 +230,42 @@ public void BlittableScanJumpToBeginAddressTest() shiftBeginAddressTo = store.Log.TailAddress; shiftToKey = i; } - var key = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key, ref value, Empty.Default); + var key = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key, value, Empty.Default); } using var iter = store.Log.Scan(store.Log.HeadAddress, store.Log.TailAddress); for (int i = 0; i < 100; ++i) { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); - ClassicAssert.AreEqual(i, iter.GetKey().key); - ClassicAssert.AreEqual(i, iter.GetValue().value); + ClassicAssert.IsTrue(iter.GetNext()); + ClassicAssert.AreEqual(i, iter.Key.AsRef().key); + ClassicAssert.AreEqual(i, ((TestObjectValue)iter.ValueObject).value); } store.Log.ShiftBeginAddress(shiftBeginAddressTo); for (int i = 0; i < numTailRecords; ++i) { - ClassicAssert.IsTrue(iter.GetNext(out var recordInfo)); + ClassicAssert.IsTrue(iter.GetNext()); if (i == 0) ClassicAssert.AreEqual(store.Log.BeginAddress, iter.CurrentAddress); var expectedKey = numRecords - numTailRecords + i; - ClassicAssert.AreEqual(expectedKey, iter.GetKey().key); - ClassicAssert.AreEqual(expectedKey, iter.GetValue().value); + ClassicAssert.AreEqual(expectedKey, iter.Key.AsRef().key); + ClassicAssert.AreEqual(expectedKey, ((TestObjectValue)iter.ValueObject).value); } } - public class ScanFunctions : MyFunctions + public class ScanFunctions : TestObjectFunctions { // Right now this is unused but helped with debugging so I'm keeping it around. internal long insertedAddress; - public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyValue src, ref MyValue dst, ref MyOutput output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref TestObjectInput input, IHeapObject srcValue, ref TestObjectOutput output, ref UpsertInfo upsertInfo) { insertedAddress = upsertInfo.Address; - return base.SingleWriter(ref key, ref input, ref src, ref dst, ref output, ref upsertInfo, reason, ref recordInfo); + return base.InitialWriter(ref logRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo); } } @@ -260,11 +273,10 @@ public override bool SingleWriter(ref MyKey key, ref MyInput input, ref MyValue [Category("TsavoriteKV")] [Category("Smoke")] - public void GenericScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void ObjectScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { const int PageSizeBits = 9; const long PageSize = 1L << PageSizeBits; - var recordSize = GenericAllocatorImpl.RecordSize; log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.log")); objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.obj.log")); @@ -275,22 +287,31 @@ public void GenericScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, PageSize = 1L << 15, SegmentSize = 1L << 18 - }, StoreFunctions.Create(comparer, () => new MyKeySerializer(), () => new MyValueSerializer()) + }, StoreFunctions.Create(comparer, () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; + var startTailAddress = store.Log.TailAddress; + var recordSize = 0; for (int i = 0; i < TotalRecords; i++) { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); + + if (recordSize == 0) + { + // Verify the recordSize from the first record's tailAddress growth. + recordSize = (int)(store.Log.TailAddress - startTailAddress); + // Size should be RecordInfo, MinHeaderBytes, Key len 4, value size 4 (objectId), objectLogPosition ulong. + Assert.That(recordSize, Is.EqualTo(32), $"Expected record size of 32 but was {recordSize}"); + } } var scanCursorFuncs = new ScanCursorFuncs(); @@ -319,16 +340,16 @@ public void GenericScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] // Scan and verify we see them all scanCursorFuncs.Initialize(verifyKeys); - ClassicAssert.IsFalse(session.ScanCursor(ref cursor, long.MaxValue, scanCursorFuncs, long.MaxValue), "Expected scan to finish and return false, pt 1"); + ClassicAssert.IsFalse(session.ScanCursor(ref cursor, count: long.MaxValue, scanCursorFuncs, endAddress: long.MaxValue), "Expected scan to finish and return false, pt 1"); ClassicAssert.AreEqual(TotalRecords, scanCursorFuncs.numRecords, "Unexpected count for all on-disk"); ClassicAssert.AreEqual(0, cursor, "Expected cursor to be 0, pt 2"); // Add another totalRecords, with keys incremented by totalRecords to remain distinct, and verify we see all keys. for (int i = 0; i < TotalRecords; i++) { - var key1 = new MyKey { key = i + TotalRecords }; - var value = new MyValue { value = i + TotalRecords }; - _ = bContext.Upsert(ref key1, ref value); + var key1 = new TestObjectKey { key = i + TotalRecords }; + var value = new TestObjectValue { value = i + TotalRecords }; + _ = bContext.Upsert(key1, value); } scanCursorFuncs.Initialize(verifyKeys); ClassicAssert.IsFalse(session.ScanCursor(ref cursor, long.MaxValue, scanCursorFuncs, long.MaxValue), "Expected scan to finish and return false, pt 1"); @@ -346,8 +367,8 @@ public void GenericScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] } while (cursor < PageSize * 3); // Now try an invalid cursor in-memory. First we have to read what's at the target start address (let's use HeadAddress) to find what the value is. - MyInput input = new(); - MyOutput output = new(); + TestObjectInput input = new(); + TestObjectOutput output = new(); ReadOptions readOptions = default; var readStatus = bContext.ReadAtAddress(store.hlogBase.HeadAddress, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(readStatus.Found, $"Could not read at HeadAddress; {readStatus}"); @@ -366,7 +387,7 @@ public void GenericScanCursorTest([Values(HashModulo.NoMod, HashModulo.Hundred)] [Category("TsavoriteKV")] [Category("Smoke")] - public void GenericScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) + public void ObjectScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hundred)] HashModulo hashMod) { log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.log")); objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "test.obj.log")); @@ -377,21 +398,21 @@ public void GenericScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hun LogDevice = log, ObjectLogDevice = objlog, MutableFraction = 0.1, - MemorySize = 1L << 20, + LogMemorySize = 1L << 20, PageSize = 1L << 15, SegmentSize = 1L << 18 - }, StoreFunctions.Create(comparer, () => new MyKeySerializer(), () => new MyValueSerializer()) + }, StoreFunctions.Create(comparer, () => new TestObjectValue.Serializer()) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - using var session = store.NewSession(new ScanFunctions()); + using var session = store.NewSession(new ScanFunctions()); var bContext = session.BasicContext; for (int i = 0; i < TotalRecords; i++) { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; - _ = bContext.Upsert(ref key1, ref value); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value); } var scanCursorFuncs = new ScanCursorFuncs(); @@ -410,30 +431,31 @@ public void GenericScanCursorFilterTest([Values(HashModulo.NoMod, HashModulo.Hun ClassicAssert.Greater(cursor, 0, "Expected cursor to be > 0, pt 1"); } - internal sealed class ScanCursorFuncs : IScanIteratorFunctions + internal sealed class ScanCursorFuncs : IScanIteratorFunctions { internal int numRecords; internal long lastAddress; internal bool verifyKeys; - internal Func filter; + internal Func filter; internal void Initialize(bool verifyKeys) => Initialize(verifyKeys, k => true); - internal void Initialize(bool verifyKeys, Func filter) + internal void Initialize(bool verifyKeys, Func filter) { numRecords = 0; this.verifyKeys = verifyKeys; this.filter = filter; } - public bool ConcurrentReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { - cursorRecordResult = filter(key) ? CursorRecordResult.Accept : CursorRecordResult.Skip; + cursorRecordResult = filter(logRecord.Key.AsRef()) ? CursorRecordResult.Accept : CursorRecordResult.Skip; if (cursorRecordResult != CursorRecordResult.Accept) return true; if (verifyKeys) - ClassicAssert.AreEqual(numRecords, key.key, "Mismatched key field on Scan"); + ClassicAssert.AreEqual(numRecords, logRecord.Key.AsRef().key, "Mismatched key field on Scan"); ClassicAssert.Greater(recordMetadata.Address, 0); ++numRecords; lastAddress = recordMetadata.Address; @@ -446,10 +468,6 @@ public void OnException(Exception exception, long numberOfRecords) public bool OnStart(long beginAddress, long endAddress) => true; public void OnStop(bool completed, long numberOfRecords) { } - - public bool SingleReader(ref MyKey key, ref MyValue value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => ConcurrentReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); } - } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/ObjectReadCacheTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectReadCacheTests.cs similarity index 51% rename from libs/storage/Tsavorite/cs/test/ObjectReadCacheTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ObjectReadCacheTests.cs index 6f9c8fc73f5..2fdda5e3d35 100644 --- a/libs/storage/Tsavorite/cs/test/ObjectReadCacheTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectReadCacheTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,14 +9,12 @@ namespace Tsavorite.test.ReadCacheTests { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; [TestFixture] - internal class ObjectReadCacheTests : AllureTestBase + internal class ObjectReadCacheTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log, objlog; [SetUp] @@ -32,12 +29,12 @@ public void Setup() IndexSize = 1L << 13, LogDevice = log, ObjectLogDevice = objlog, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, ReadCacheMemorySize = 1L << 15, ReadCachePageSize = 1L << 10, ReadCacheEnabled = true - }, StoreFunctions.Create(new MyKey.Comparer(), () => new MyKeySerializer(), () => new MyValueSerializer(), DefaultRecordDisposer.Instance) + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -59,18 +56,18 @@ public void TearDown() [Category("Smoke")] public void ObjectDiskWriteReadCache() { - using var session = store.NewSession(new MyFunctions()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; - MyInput input = default; + TestObjectInput input = default; for (int i = 0; i < 2000; i++) { - var key = new MyKey { key = i }; - var value = new MyValue { value = i }; - bContext.Upsert(ref key, ref value, Empty.Default); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, Empty.Default); } - bContext.CompletePending(true); + _ = bContext.CompletePending(true); // Evict all records from main memory of hybrid log store.Log.FlushAndEvict(true); @@ -78,23 +75,23 @@ public void ObjectDiskWriteReadCache() // Read 2000 keys - all should be served from disk, populating and evicting the read cache FIFO for (int i = 0; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read last 100 keys - all should be served from cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.value, output.value.value); } @@ -105,54 +102,55 @@ public void ObjectDiskWriteReadCache() // Read 100 keys - all should be served from disk, populating cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read 100 keys - all should be served from cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.value, output.value.value); } + const int valueAdd = 100_000; // Upsert to overwrite the read cache for (int i = 1900; i < 1950; i++) { - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i + 1 }; - bContext.Upsert(ref key1, ref value, Empty.Default); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i + valueAdd }; + _ = bContext.Upsert(key1, value, Empty.Default); } // RMW to overwrite the read cache for (int i = 1950; i < 2000; i++) { - var key1 = new MyKey { key = i }; - input = new MyInput { value = 1 }; - var status = bContext.RMW(ref key1, ref input, Empty.Default); + var key1 = new TestObjectKey { key = i }; + input = new TestObjectInput { value = valueAdd }; + var status = bContext.RMW(key1, ref input, Empty.Default); if (status.IsPending) - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read the 100 keys for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i + 1 }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i + valueAdd }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found, $"key = {key1.key}"); ClassicAssert.AreEqual(value.value, output.value.value); } @@ -162,42 +160,41 @@ public void ObjectDiskWriteReadCache() [Category("TsavoriteKV")] public void ObjectDiskWriteReadCache2() { - using var session = store.NewSession(new MyFunctions()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; - MyInput input = default; + TestObjectInput input = default; for (int i = 0; i < 2000; i++) { - var key = new MyKey { key = i }; - var value = new MyValue { value = i }; - bContext.Upsert(ref key, ref value, Empty.Default); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(key1, value, Empty.Default); } - bContext.CompletePending(true); + _ = bContext.CompletePending(true); - // Dispose the hybrid log from memory entirely - store.Log.DisposeFromMemory(); + store.Log.FlushAndEvict(wait: true); // Read 2000 keys - all should be served from disk, populating and evicting the read cache FIFO for (int i = 0; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read last 100 keys - all should be served from cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.value, output.value.value); } @@ -208,23 +205,23 @@ public void ObjectDiskWriteReadCache2() // Read 100 keys - all should be served from disk, populating cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - var key1 = new MyKey { key = i }; - var value = new MyValue { value = i }; + TestObjectOutput output = new(); + var key1 = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read 100 keys - all should be served from cache for (int i = 1900; i < 2000; i++) { - MyOutput output = new(); - MyKey key1 = new() { key = i }; - MyValue value = new() { value = i }; + TestObjectOutput output = new(); + TestObjectKey key1 = new() { key = i }; + TestObjectValue value = new() { value = i }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.value, output.value.value); } diff --git a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest.cs similarity index 73% rename from libs/storage/Tsavorite/cs/test/ObjectRecoveryTest.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest.cs index b3d1134a458..a263af618fc 100644 --- a/libs/storage/Tsavorite/cs/test/ObjectRecoveryTest.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest.cs @@ -4,32 +4,31 @@ using System; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using Tsavorite.core; +using Tsavorite.test.recovery.sumstore; namespace Tsavorite.test.recovery.objects { - using ClassAllocator = GenericAllocator>>; - using ClassStoreFunctions = StoreFunctions>; + using static Tsavorite.test.TestUtils; + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; internal struct StructTuple { public T1 Item1; public T2 Item2; } - - [AllureNUnit] [TestFixture] - internal class ObjectRecoveryTests : AllureTestBase + internal class ObjectRecoveryTests : TestBase { const long NumUniqueKeys = 1L << 14; const long KeySpace = 1L << 14; const long NumOps = 1L << 19; const long CompletePendingInterval = 1L << 10; const long CheckpointInterval = 1L << 16; - private TsavoriteKV store; + private TsavoriteKV store; private Guid token; private IDevice log, objlog; @@ -50,7 +49,7 @@ public void Setup(bool deleteDir) LogDevice = log, ObjectLogDevice = objlog, CheckpointDir = TestUtils.MethodTestDir - }, StoreFunctions.Create(new AdIdObj.Comparer(), () => new AdIdObj.Serializer(), () => new NumClicksObj.Serializer()) + }, StoreFunctions.Create(new AdId.Comparer(), () => new NumClicksObj.Serializer(), DefaultRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -79,12 +78,12 @@ private void PrepareToRecover() [Test] [Category("TsavoriteKV"), Category("CheckpointRestore")] - public async ValueTask ObjectRecoveryTest1([Values] bool isAsync) + public async ValueTask ObjectRecoveryTest1([Values] CompletionSyncMode syncMode) { Populate(); PrepareToRecover(); - if (isAsync) + if (syncMode == CompletionSyncMode.Async) _ = await store.RecoverAsync(token, token).ConfigureAwait(false); else _ = store.Recover(token, token); @@ -95,25 +94,25 @@ public async ValueTask ObjectRecoveryTest1([Values] bool isAsync) public unsafe void Populate() { // Prepare the dataset - var inputArray = new StructTuple[NumOps]; + var inputArray = GC.AllocateArray>((int)NumOps); for (int i = 0; i < NumOps; i++) { - inputArray[i] = new StructTuple + inputArray[i] = new() { - Item1 = new AdIdObj { adId = i % NumUniqueKeys }, + Item1 = new AdId { adId = i % NumUniqueKeys }, Item2 = new Input { numClicks = new NumClicksObj { numClicks = 1 } } }; } // Register thread with Tsavorite - var session = store.NewSession(new Functions()); + var session = store.NewSession(new Functions()); var bContext = session.BasicContext; // Process the batch of input data bool first = true; for (int i = 0; i < NumOps; i++) { - _ = bContext.RMW(ref inputArray[i].Item1, ref inputArray[i].Item2, Empty.Default); + _ = bContext.RMW(inputArray[i].Item1, ref inputArray[i].Item2, Empty.Default); if ((i + 1) % CheckpointInterval == 0) { @@ -123,14 +122,11 @@ public unsafe void Populate() while (!store.TryInitiateFullCheckpoint(out _, CheckpointType.Snapshot)) ; store.CompleteCheckpointAsync().GetAwaiter().GetResult(); - first = false; } if (i % CompletePendingInterval == 0) - { _ = bContext.CompletePending(false, false); - } } @@ -142,17 +138,17 @@ public unsafe void Populate() public unsafe void Verify(Guid cprVersion, Guid indexVersion) { // Create array for reading - var inputArray = new StructTuple[NumUniqueKeys]; + var inputArray = GC.AllocateArray>((int)NumUniqueKeys); for (int i = 0; i < NumUniqueKeys; i++) { - inputArray[i] = new StructTuple + inputArray[i] = new() { - Item1 = new AdIdObj { adId = i }, + Item1 = new AdId { adId = i }, Item2 = new Input { numClicks = new NumClicksObj { numClicks = 0 } } }; } - var session = store.NewSession(new Functions()); + var session = store.NewSession(new Functions()); var bContext = session.BasicContext; Input input = default; @@ -160,7 +156,7 @@ public unsafe void Verify(Guid cprVersion, Guid indexVersion) for (var i = 0; i < NumUniqueKeys; i++) { Output output = new(); - _ = bContext.Read(ref inputArray[i].Item1, ref input, ref output, Empty.Default); + _ = bContext.Read(inputArray[i].Item1, ref input, ref output, Empty.Default); } // Complete all pending requests diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest2.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest2.cs new file mode 100644 index 00000000000..84e48fa32ce --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest2.cs @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.recovery.objects +{ + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + public class ObjectRecoveryTests2 : TestBase + { + int numberOfRecords; + + [SetUp] + public void Setup() + { + RecreateDirectory(MethodTestDir); + } + + [TearDown] + public void TearDown() + { + TestUtils.OnTearDown(); + } + + [Test] + [Category("TsavoriteKV")] + [Category("CheckpointRestore")] + [Category("Smoke")] + + public async ValueTask ObjectRecoveryTest2( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Range(300, 700, 300)] int numberOfRecords, + [Values] CompletionSyncMode syncMode) + { + this.numberOfRecords = numberOfRecords; + + // Populate and checkpoint + { + Prepare(out var log, out var objlog, out var store); + + var session = store.NewSession(new TestObjectFunctions()); + Write(session, store, checkpointType); + Read(session, delete: false); + session.Dispose(); + + _ = store.TryInitiateFullCheckpoint(out var guid, checkpointType); // guid is useful for debugging, but not otherwise used in this test + await store.CompleteCheckpointAsync(); + + Destroy(log, objlog, store); + } + + // Restore and verify + { + Prepare(out var log, out var objlog, out var store); + + if (syncMode == CompletionSyncMode.Async) + _ = await store.RecoverAsync().ConfigureAwait(false); + else + _ = store.Recover(); + + var session = store.NewSession(new TestObjectFunctions()); + Read(session, delete: true); + session.Dispose(); + + Destroy(log, objlog, store); + } + } + + private static void Prepare(out IDevice log, out IDevice objlog, out TsavoriteKV store) + { + log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "RecoverTests.log")); + objlog = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "RecoverTests_HEAP.log")); + store = new(new() + { + IndexSize = 1L << 26, + LogDevice = log, + ObjectLogDevice = objlog, + SegmentSize = 1L << 12, + LogMemorySize = 1L << 12, + PageSize = 1L << 9, + CheckpointDir = Path.Combine(MethodTestDir, "checkpoints") + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + } + + private static void Destroy(IDevice log, IDevice objlog, TsavoriteKV store) + { + // Dispose Tsavorite instance and log + store.Dispose(); + log.Dispose(); + objlog.Dispose(); + } + + private void Write(ClientSession session, + TsavoriteKV store, CheckpointType checkpointType) + { + var bContext = session.BasicContext; + + for (int i = 0; i < numberOfRecords; i++) + { + var _key = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, value); + if (i > 0 && i % 100 == 0) + { + _ = store.TryInitiateFullCheckpoint(out _, checkpointType); + store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); + } + } + } + + private void Read(ClientSession session, bool delete) + { + var bContext = session.BasicContext; + + for (int i = 0; i < numberOfRecords; i++) + { + TestObjectKey key = new() { key = i }; + TestObjectInput input = default; + TestObjectOutput output = new(); + + var status = bContext.Read(key, ref input, ref output); + bool wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + + ClassicAssert.IsTrue(status.Found, $"key: {key.key}; status {status}; wasPending {wasPending}"); + ClassicAssert.AreEqual(i, output.value.value); + } + + if (delete) + { + TestObjectKey key = new() { key = 1 }; + TestObjectInput input = default; + TestObjectOutput output = new(); + _ = bContext.Delete(key); + var status = bContext.Read(key, ref input, ref output); + + ClassicAssert.IsFalse(status.IsPending, $"key: {key.key}; status {status}"); + ClassicAssert.IsFalse(status.Found, $"key: {key.key}; status {status}"); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest3.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest3.cs new file mode 100644 index 00000000000..10950aadcbc --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectRecoveryTest3.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.recovery.objects +{ + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + public class ObjectRecoveryTests3 : TestBase + { + int iterations; + + [SetUp] + public void Setup() + { + RecreateDirectory(MethodTestDir); + } + + [TearDown] + public void TearDown() + { + TestUtils.OnTearDown(); + } + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + public async ValueTask ObjectRecoveryTest3( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values(1000)] int iterations, + [Values] CompletionSyncMode syncMode) + { + this.iterations = iterations; + Prepare(out IDevice log, out IDevice objlog, out var store); + + var session1 = store.NewSession(new TestObjectFunctions()); + var tokens = Write(session1, store, checkpointType); + Read(session1, false, iterations); + session1.Dispose(); + + _ = store.TryInitiateHybridLogCheckpoint(out Guid token, checkpointType); + store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); + tokens.Add((iterations, token)); + Destroy(log, objlog, store); + + foreach (var item in tokens) + { + Prepare(out log, out objlog, out store); + + if (syncMode == CompletionSyncMode.Async) + _ = await store.RecoverAsync(default, item.Item2).ConfigureAwait(false); + else + _ = store.Recover(default, item.Item2); + + var session2 = store.NewSession(new TestObjectFunctions()); + Read(session2, false, item.Item1); + session2.Dispose(); + + Destroy(log, objlog, store); + } + } + + private static void Prepare(out IDevice log, out IDevice objlog, out TsavoriteKV store) + { + log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "RecoverTests.log")); + objlog = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "RecoverTests_HEAP.log")); + store = new(new() + { + IndexSize = 1L << 26, + LogDevice = log, + ObjectLogDevice = objlog, + SegmentSize = 1L << 12, + LogMemorySize = 1L << 12, + PageSize = 1L << 9, + CheckpointDir = Path.Combine(MethodTestDir, "check-points") + }, StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + } + + private static void Destroy(IDevice log, IDevice objlog, TsavoriteKV store) + { + // Dispose Tsavorite instance and log + store.Dispose(); + log.Dispose(); + objlog.Dispose(); + } + + private List<(int, Guid)> Write(ClientSession session, + TsavoriteKV store, CheckpointType checkpointType) + { + var bContext = session.BasicContext; + + var tokens = new List<(int, Guid)>(); + for (int i = 0; i < iterations; i++) + { + var _key = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + _ = bContext.Upsert(_key, value); + + if (i % 1000 == 0 && i > 0) + { + _ = store.TryInitiateHybridLogCheckpoint(out Guid token, checkpointType); + store.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); + tokens.Add((i, token)); + } + } + return tokens; + } + + private static void Read(ClientSession session, bool delete, int iter) + { + var bContext = session.BasicContext; + + for (int i = 0; i < iter; i++) + { + var key = new TestObjectKey { key = i }; + TestObjectInput input = default; + TestObjectOutput output = new(); + var status = bContext.Read(key, ref input, ref output); + + if (status.IsPending) + { + Assert.That(bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + + ClassicAssert.IsTrue(status.Found, $"key: {key}"); + ClassicAssert.AreEqual(i, output.value.value, $"key: {key}"); + } + + if (delete) + { + var key = new TestObjectKey { key = 1 }; + var input = default(TestObjectInput); + var output = new TestObjectOutput(); + _ = bContext.Delete(key); + var status = bContext.Read(key, ref input, ref output); + + if (status.IsPending) + { + Assert.That(bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + + ClassicAssert.IsFalse(status.Found, $"key: {key}"); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/ObjectTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectTests.cs new file mode 100644 index 00000000000..8b3bbabc161 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/ObjectTests.cs @@ -0,0 +1,575 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using static Tsavorite.core.Utility; + +namespace Tsavorite.test.Objects +{ + using static TestUtils; + + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; + [TestFixture] + internal class ObjectTests : TestBase + { + private TsavoriteKV store; + private IDevice log, objlog; + const long LogMemorySize = 1L << 15; + const long PageSize = 1L << 10; + + [SetUp] + public void Setup() + { + DeleteDirectory(MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.log"), deleteOnClose: true); + objlog = Devices.CreateLogDevice(Path.Join(MethodTestDir, "ObjectTests.obj.log"), deleteOnClose: true); + var storeFunctions = TestContext.CurrentContext.Test.MethodName.StartsWith("LargeObject") + ? StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestLargeObjectValue.Serializer(), DefaultRecordTriggers.Instance) + : StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance); + + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + ObjectLogDevice = objlog, + MutableFraction = TestContext.CurrentContext.Test.MethodName.Equals(nameof(LargeObjectLinearizeFlushedPages)) ? 1.0 : 0.1, + LogMemorySize = LogMemorySize, + PageSize = PageSize + }, storeFunctions + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + objlog?.Dispose(); + objlog = null; + OnTearDown(); + } + + [Test, Category(TsavoriteKVTestCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectInMemWriteReadUpsert() + { + using var session = store.NewSession(new TestObjectFunctions()); + var bContext = session.BasicContext; + + TestObjectKey key = new() { key = 9999999 }; + TestObjectValue value = new() { value = 23 }; + + TestObjectInput input = default; + TestObjectOutput output = default; + + _ = bContext.Upsert(key, value, Empty.Default); + _ = bContext.Read(key, ref input, ref output, Empty.Default); + ClassicAssert.AreEqual(value.value, output.value.value); + } + + [Test, Category(TsavoriteKVTestCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectInMemWriteReadRMW() + { + using var session = store.NewSession(new TestObjectFunctions()); + var bContext = session.BasicContext; + + TestObjectKey key1 = new() { key = 8999998 }; + TestObjectInput input1 = new() { value = 23 }; + TestObjectOutput output = new(); + + _ = bContext.RMW(key1, ref input1, Empty.Default); + + TestObjectKey key2 = new() { key = 8999999 }; + TestObjectInput input2 = new() { value = 24 }; + _ = bContext.RMW(key2, ref input2, Empty.Default); + + _ = bContext.Read(key1, ref input1, ref output, Empty.Default); + + ClassicAssert.AreEqual(input1.value, output.value.value); + + _ = bContext.Read(key2, ref input2, ref output, Empty.Default); + ClassicAssert.AreEqual(input2.value, output.value.value); + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectDiskWriteReadSingle() + { + using var session = store.NewSession(new TestObjectFunctions()); + var bContext = session.BasicContext; + const int keyInt = 42; + + var key = new TestObjectKey { key = keyInt }; + var value = new TestObjectValue { value = keyInt }; + _ = bContext.Upsert(key, value, Empty.Default); + + TestObjectInput input = new(); + TestObjectOutput output = new(); + var status = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.False); + Assert.That(status.Found, Is.True); + Assert.That(output.value.value, Is.EqualTo(keyInt)); + + store.Log.FlushAndEvict(wait: true); + + status = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.True); + Assert.That(bContext.CompletePendingWithOutputs(out var outputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(outputs); + Assert.That(status.Found, Is.True); + Assert.That(output.value.value, Is.EqualTo(keyInt)); + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectDiskWriteRead() + { + using var session = store.NewSession(new TestObjectFunctions()); + var bContext = session.BasicContext; + + for (int i = 0; i < 2000; i++) + { + var key = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + if (i == 120) + i += 0; + _ = bContext.Upsert(key, value, Empty.Default); + } + + TestObjectKey key2 = new() { key = 23 }; + TestObjectInput input = new(); + TestObjectOutput g1 = new(); + var status = bContext.Read(key2, ref input, ref g1, Empty.Default); + + if (status.IsPending) + { + _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); + (status, g1) = GetSinglePendingResult(outputs); + } + + ClassicAssert.IsTrue(status.Found); + ClassicAssert.AreEqual(23, g1.value.value); + + key2.key = 99999; + status = bContext.Read(key2, ref input, ref g1, Empty.Default); + + if (status.IsPending) + (status, _) = bContext.GetSinglePendingResult(); + ClassicAssert.IsFalse(status.Found); + + // Update last 100 using RMW in memory + for (int i = 1900; i < 2000; i++) + { + var key = new TestObjectKey { key = i }; + input = new TestObjectInput { value = 1 }; + status = bContext.RMW(key, ref input, Empty.Default); + ClassicAssert.IsFalse(status.IsPending, "Expected RMW to complete in-memory"); + } + + // Update first 100 using RMW from storage + var numPendingUpdates = 0; + for (int i = 0; i < 100; i++) + { + var key = new TestObjectKey { key = i }; + input = new TestObjectInput { value = 1 }; + status = bContext.RMW(key, ref input, Empty.Default); + if (status.IsPending) + { + numPendingUpdates++; + _ = bContext.CompletePending(true); + } + } + Assert.That(numPendingUpdates, Is.EqualTo(100)); + + var numPendingReads = 0; + for (int i = 0; i < 2000; i++) + { + var output = new TestObjectOutput(); + var key = new TestObjectKey { key = i }; + var value = new TestObjectValue { value = i }; + + status = bContext.Read(key, ref input, ref output, Empty.Default); + if (status.IsPending) + { + numPendingReads++; + (status, output) = bContext.GetSinglePendingResult(); + } + + if (i is < 100 or >= 1900) + ClassicAssert.AreEqual(value.value + 1, output.value.value); + else + ClassicAssert.AreEqual(value.value, output.value.value); + } + Assert.That(numPendingReads, Is.GreaterThanOrEqualTo(numPendingUpdates)); + } + + /// Various sizes to test + public enum SerializeKeyValueSize + { + Thirty = 30, + OneK = 1024, + HalfBuffer = IStreamBuffer.BufferSize / 2, + OneBuffer = IStreamBuffer.BufferSize, + ThreeHalfBuffer = (IStreamBuffer.BufferSize / 2) * 3, + TwoBuffer = IStreamBuffer.BufferSize * 2 + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + //[Repeat(300)] + public void LargeObjectDiskWriteReadSmallKeyBigValue([Values] SerializeKeyValueSize serializeValueSize) + { + using var session = store.NewSession(new TestLargeObjectFunctions()); + var bContext = session.BasicContext; + + var input = new TestLargeObjectInput(); + var output = new TestLargeObjectOutput(); + var valueSize = (int)serializeValueSize; + const int numRec = 3; + for (int ii = 0; ii < numRec; ii++) + { + var key = new TestObjectKey { key = ii }; + var value = new TestLargeObjectValue(valueSize + (ii * 4096)); + new Span(value.value).Fill(0x42); + _ = bContext.Upsert(key, ref input, value, ref output); + } + + // Test before and after the flush + DoRead(onDisk: false); + store.Log.FlushAndEvict(wait: true); + DoRead(onDisk: true); + + void DoRead(bool onDisk) + { + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Object }; + for (int ii = 0; ii < numRec; ii++) + { + var output = new TestLargeObjectOutput(); + var key = new TestObjectKey { key = ii }; + + var status = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.EqualTo(onDisk), $"IsPending ({status.IsPending}) != onDisk"); + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + + Assert.That(output.valueObject.value.Length, Is.EqualTo(valueSize + (ii * 4096))); + var numLongs = output.valueObject.value.Length % 8; + var badIndex = new ReadOnlySpan(output.valueObject.value).IndexOfAnyExcept((byte)0x42); + if (badIndex != -1) + Assert.Fail($"Unexpected byte value at index {badIndex}, onDisk {onDisk}, record# {ii}: {output.valueObject.value[badIndex]}"); + } + } + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + //[Repeat(300)] + public void LargeObjectMultiFlushedPages([Values(SerializeKeyValueSize.Thirty, SerializeKeyValueSize.OneK)] SerializeKeyValueSize serializeValueSize) + { + // Ensure our size calculations are correct by validating the test parameters are what we expect + Assert.That(LogMemorySize, Is.EqualTo(1L << 15)); + Assert.That(PageSize, Is.EqualTo(1L << 10)); + Assert.That(store.hlogBase.BufferSize, Is.EqualTo(32)); // LogMemorySize is PageSize << 5 + Assert.That(store.hlogBase.MaxAllocatedPageCount, Is.EqualTo(32)); + const int RecordLength = 32; // LogRecord allocated size + const int ObjectsPerPage = (int)((PageSize - PageHeader.Size) / RecordLength); + Assert.That(ObjectsPerPage, Is.EqualTo(30)); // Make debugging easier by verifying the length we'll see in the IDE + + var functions = new TestLargeObjectFunctions { expectedRecordLength = RecordLength }; // ExpectedRecordLength controls how many objects per page + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + var input = new TestLargeObjectInput(); + var output = new TestLargeObjectOutput(); + var valueSize = (int)serializeValueSize; + + // Start with a full buffer plus two pages to overflow + int numRec = ObjectsPerPage * (store.hlogBase.BufferSize + 2); + int lastKey = 0; + for (; lastKey < numRec; lastKey++) + { + var key = new TestObjectKey { key = lastKey }; + var value = new TestLargeObjectValue(valueSize); + new Span(value.value).Fill((byte)key.key); + _ = bContext.Upsert(key, ref input, value, ref output); + } + + // The upsert loop implicitly advanced ReadOnlyAddress as pages were turned, which kicks off asynchronous page flushes. + // ROA is already at its final value, but the flush completions that drive FlushedUntilAddress arrive on background + // I/O threads and may not all have landed by the time we reach here. Synchronize by re-invoking ShiftReadOnlyAddress + // with wait:true; the shift itself is a no-op (MonotonicUpdate fails for the same address) but the wait loop blocks + // until FlushedUntilAddress catches up to the current ReadOnlyAddress. + store.Log.ShiftReadOnlyAddress(store.hlogBase.ReadOnlyAddress, wait: true); + + // Test that the expected number of pages were flushed and that we get the right results back. We've flushed in a multiple of full pages, + // so ReadOnlyAddress will be aligned to page size. With 0.1 mutable fraction, tail at 34 * PageSize, and head shifted to page 2 during + // the last two page-turns (each of which moves head by one page once MaxAllocatedPageCount is reached), CalculateReadOnlyAddress snaps + // ROA to page 29. That is: 2 closed pages (0, 1), 27 immutable (2..28) and 5 mutable (29..33), with FUA == ROA == 29 * PageSize. + // Note: This test does NOT use SizeTracker; therefore it does not evict except when page count is exceeded. + Assert.That(IsAligned(store.hlogBase.ReadOnlyAddress, store.hlogBase.PageSize), $"ReadOnlyAddress ({store.hlogBase.ReadOnlyAddress}) should be page-aligned"); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.PageSize * 29), $"FUA ({store.hlogBase.FlushedUntilAddress}) != PageSize * 29"); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.ReadOnlyAddress), $"FUA ({store.hlogBase.FlushedUntilAddress}) == ROA"); + DoRead(0, 2 * ObjectsPerPage - 1, onDisk: true); + DoRead(2 * ObjectsPerPage, lastKey, onDisk: false); + + // Now make ReadOnlyAddress be no longer aligned to page: add half a page, the SetReadOnlyToTail. + const int ObjectsPerHalfPage = ObjectsPerPage / 2; + for (var ii = 0; ii < ObjectsPerHalfPage; ii++) + { + var key = new TestObjectKey { key = lastKey + ii }; + var value = new TestLargeObjectValue(valueSize); + new Span(value.value).Fill((byte)key.key); + _ = bContext.Upsert(key, ref input, value, ref output); + } + + store.epoch.Resume(); + try + { + Assert.That(store.hlogBase.ShiftReadOnlyToTail(out _, out var sroTask), Is.True); + sroTask.Wait(); + } + finally + { + store.epoch.Suspend(); + } + + // This should have flushed 2.5 more pages and Closed one more. + Assert.That(!IsAligned(store.hlogBase.ReadOnlyAddress, store.hlogBase.PageSize), $"ReadOnlyAddress ({store.hlogBase.ReadOnlyAddress}) should not be page-aligned"); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.PageSize * 34 + PageHeader.Size + ObjectsPerHalfPage * RecordLength), $"FUA ({store.hlogBase.FlushedUntilAddress}) != PageSize * 34 + ObjectsPerHalfPage"); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.ReadOnlyAddress), $"FUA ({store.hlogBase.FlushedUntilAddress}) == ROA"); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.GetTailAddress()), $"FUA ({store.hlogBase.FlushedUntilAddress}) == TA"); + DoRead(0, ObjectsPerPage * 3 - 1, onDisk: true); + DoRead(ObjectsPerPage * 3, lastKey + ObjectsPerHalfPage, onDisk: false); + + void DoRead(int firstKey, int lastKey, bool onDisk) + { + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Object }; + for (int ii = firstKey; ii < lastKey; ii++) + { + var output = new TestLargeObjectOutput(); + var key = new TestObjectKey { key = ii }; + + var status = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.EqualTo(onDisk), $"status.IsPending ({status}) != onDisk for key {ii}"); + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + + Assert.That(output.valueObject.value.Length, Is.EqualTo(valueSize)); + var numLongs = output.valueObject.value.Length % 8; + var badIndex = new ReadOnlySpan(output.valueObject.value).IndexOfAnyExcept((byte)ii); + if (badIndex != -1) + Assert.Fail($"Unexpected byte value at index {badIndex}, onDisk {onDisk}, record# {ii}: {output.valueObject.value[badIndex]}"); + } + } + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + //[Repeat(300)] + // Note: This test name keys the mutableFraction + public async Task LargeObjectLinearizeFlushedPages([Values(SerializeKeyValueSize.Thirty, SerializeKeyValueSize.OneK)] SerializeKeyValueSize serializeValueSize) + { + // Ensure our size calculations are correct by validating the test parameters are what we expect + Assert.That(LogMemorySize, Is.EqualTo(1L << 15)); + Assert.That(PageSize, Is.EqualTo(1L << 10)); + Assert.That(store.hlogBase.BufferSize, Is.EqualTo(32)); // LogMemorySize is PageSize << 5 + Assert.That(store.hlogBase.MaxAllocatedPageCount, Is.EqualTo(32)); + const int RecordLength = 32; // LogRecord allocated size + const int ObjectsPerPage = (int)((PageSize - PageHeader.Size) / RecordLength); + Assert.That(ObjectsPerPage, Is.EqualTo(30)); // Make debugging easier by verifying the length we'll see in the IDE + + var functions = new TestLargeObjectFunctions { expectedRecordLength = RecordLength }; // ExpectedRecordLength controls how many objects per page + using var session = store.NewSession(functions); + var bContext = session.BasicContext; + + var input = new TestLargeObjectInput(); + var output = new TestLargeObjectOutput(); + var valueSize = (int)serializeValueSize; + + // Verify initial conditions + Assert.That(store.hlogBase.GetTailAddress(), Is.EqualTo(PageHeader.Size)); + Assert.That(store.hlogBase.HeadAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.hlogBase.ReadOnlyAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(PageHeader.Size)); + + int numRec = ObjectsPerPage * store.hlogBase.BufferSize; + int lastKey = 0; + for (; lastKey < numRec; lastKey++) + { + var key = new TestObjectKey { key = lastKey }; + var value = new TestLargeObjectValue(valueSize); + new Span(value.value).Fill((byte)key.key); + _ = bContext.Upsert(key, ref input, value, ref output); + } + + // Verify post-insert conditions. + Assert.That(store.hlogBase.GetTailAddress(), Is.EqualTo(store.hlogBase.PageSize * store.hlogBase.BufferSize)); + Assert.That(store.hlogBase.HeadAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.hlogBase.ReadOnlyAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(PageHeader.Size)); + + // Now test flushing in sequence so the second call has to wait to linearize. Use a semaphore to make sure we've + // launched the first one before calling the second, else the second call may be run first. + ManualResetEventSlim gate = new(false); + var task = Task.Run(() => DoFlush((store.hlogBase.BufferSize - 20) * store.hlogBase.PageSize, gate)); + + // We have to wait for this outside the epoch to avoid deadlock. + gate.Wait(); + + Task sroTask; + store.epoch.Resume(); + try + { + Assert.That(store.hlogBase.ShiftReadOnlyToTail(out _, out sroTask), Is.True); + } + finally + { + store.epoch.Suspend(); + } + gate.Dispose(); + + await Task.WhenAll(task, sroTask); + + // Test that the FlushedUntilAddress is correct and that we get the right results back; nothing has been evicted yet, so all records are in memory. + Assert.That(store.hlogBase.FlushedUntilAddress, Is.EqualTo(store.hlogBase.GetTailAddress())); + Assert.That(store.hlogBase.ReadOnlyAddress, Is.EqualTo(store.hlogBase.FlushedUntilAddress), $"FUA ({store.hlogBase.FlushedUntilAddress}) == ROA"); + DoRead(0, ObjectsPerPage * store.hlogBase.BufferSize, onDisk: false); + + void DoRead(int firstKey, int lastKey, bool onDisk) + { + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Object }; + for (int ii = firstKey; ii < lastKey; ii++) + { + var output = new TestLargeObjectOutput(); + var key = new TestObjectKey { key = ii }; + + var status = bContext.Read(key, ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.EqualTo(onDisk), $"status.IsPending ({status}) != onDisk for key {ii}"); + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + + Assert.That(output.valueObject.value.Length, Is.EqualTo(valueSize)); + var numLongs = output.valueObject.value.Length % 8; + var badIndex = new ReadOnlySpan(output.valueObject.value).IndexOfAnyExcept((byte)ii); + if (badIndex != -1) + Assert.Fail($"Unexpected byte value at index {badIndex}, onDisk {onDisk}, record# {ii}: {output.valueObject.value[badIndex]}"); + } + } + + void DoFlush(long newReadOnlyAddress, ManualResetEventSlim gate) + { + store.epoch.Resume(); + try + { + // Do this in two pieces so we signal the gate in between to give the second call above a chance to launch. + // The first shift must succeed because no concurrent shift has happened yet. + Assert.That(store.hlogBase.ShiftReadOnlyAddress(newReadOnlyAddress - store.hlogBase.PageSize * 10), Is.True); + gate?.Set(); + + // The second shift races with the main thread's ShiftReadOnlyToTail. Either ordering is valid and exercises + // the linearization path: if this shift lands first, ShiftReadOnlyToTail extends ROA from newReadOnlyAddress + // to the tail; if ShiftReadOnlyToTail lands first, ROA is already past newReadOnlyAddress and MonotonicUpdate + // (correctly) refuses to move ROA backwards, so this call returns false. Both outcomes converge to the same + // post-condition (ROA == FUA == TailAddress), which is what the caller asserts after awaiting both tasks. + _ = store.hlogBase.ShiftReadOnlyAddress(newReadOnlyAddress); + } + finally + { + store.epoch.Suspend(); + } + } + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + public void ObjectDiskWriteReadOverflowValue() + { + using var session = store.NewSession(new TestLargeObjectFunctions()); + var bContext = session.BasicContext; + + var valueSize = IStreamBuffer.BufferSize / 2; + const int numRec = 5; + var valueBuffer = new byte[valueSize * numRec]; + new Span(valueBuffer).Fill(0x42); + + for (int ii = 0; ii < numRec; ii++) + { + var key = new TestObjectKey { key = ii }; + var value = new ReadOnlySpan(valueBuffer).Slice(0, valueSize * (ii + 1)); + _ = bContext.Upsert(key, value, Empty.Default); + } + + store.Log.FlushAndEvict(wait: true); + + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Overflow }; + + for (int ii = 0; ii < numRec; ii++) + { + var output = new TestLargeObjectOutput(); + var key = new TestObjectKey { key = ii }; + + input.expectedSpanLength = valueSize * (ii + 1); + var status = bContext.Read(key, ref input, ref output, Empty.Default); + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + + Assert.That(output.valueArray.Length, Is.EqualTo(valueSize * (ii + 1))); + Assert.That(new ReadOnlySpan(output.valueArray).SequenceEqual(new ReadOnlySpan(valueBuffer).Slice(0, output.valueArray.Length))); + } + } + + [Test, Category(TsavoriteKVTestCategory), Category(LogRecordCategory), Category(SmokeTestCategory), Category(ObjectIdMapCategory)] + //[Repeat(300)] + public void LargeObjectDiskWriteReadBigKeyAndValue([Values] SerializeKeyValueSize serializeKeySize, [Values] SerializeKeyValueSize serializeValueSize) + { + using var session = store.NewSession(new TestLargeObjectFunctions()); + var bContext = session.BasicContext; + + var input = new TestLargeObjectInput(); + var output = new TestLargeObjectOutput(); + var keySize = (int)serializeKeySize; + var keyBuf = new byte[keySize]; + var valueSize = (int)serializeValueSize; + const int numRec = 3; + for (int ii = 0; ii < numRec; ii++) + { + var value = new TestLargeObjectValue(valueSize + (ii * 4096)); + var key = new Span(keyBuf); + key.Fill((byte)(ii + 100)); + new Span(value.value).Fill(0x42); + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), ref input, value, ref output); + } + + // Test before and after the flush + DoRead(onDisk: false); + store.Log.FlushAndEvict(wait: true); + DoRead(onDisk: true); + + void DoRead(bool onDisk) + { + TestLargeObjectInput input = new() { wantValueStyle = TestValueStyle.Object }; + for (int ii = 0; ii < numRec; ii++) + { + var output = new TestLargeObjectOutput(); + var key = new Span(keyBuf); + key.Fill((byte)(ii + 100)); + + var status = bContext.Read(TestSpanByteKey.FromPinnedSpan(key), ref input, ref output, Empty.Default); + Assert.That(status.IsPending, Is.EqualTo(onDisk)); + if (status.IsPending) + (status, output) = bContext.GetSinglePendingResult(); + Assert.That(status.Found, Is.True); + + Assert.That(output.valueObject.value.Length, Is.EqualTo(valueSize + (ii * 4096)), $"record# ii {ii}"); + var badIndex = new ReadOnlySpan(output.valueObject.value).IndexOfAnyExcept((byte)0x42); + if (badIndex != -1) + Assert.Fail($"Unexpected byte value at index {badIndex}, onDisk {onDisk}, record# {ii}: {output.valueObject.value[badIndex]}"); + } + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/RecoverReadOnlyTest.cs b/libs/storage/Tsavorite/cs/test/test.recovery/RecoverReadOnlyTest.cs similarity index 98% rename from libs/storage/Tsavorite/cs/test/RecoverReadOnlyTest.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/RecoverReadOnlyTest.cs index df93b8069ab..cdcb130ec3d 100644 --- a/libs/storage/Tsavorite/cs/test/RecoverReadOnlyTest.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/RecoverReadOnlyTest.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using Tsavorite.core; @@ -15,9 +14,8 @@ namespace Tsavorite.test { - [AllureNUnit] [TestFixture] - internal class BasicRecoverReadOnly : AllureTestBase + internal class BasicRecoverReadOnly : TestBase { private TsavoriteLog log; private IDevice device; diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/RecoveryCheckTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/RecoveryCheckTests.cs new file mode 100644 index 00000000000..38e8f724feb --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/RecoveryCheckTests.cs @@ -0,0 +1,1079 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; +using Tsavorite.test.recovery.sumstore; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.recovery +{ + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; + + public enum DeviceMode + { + Local, + Cloud + } + + public class RecoveryCheckBase : TestBase + { + protected IDevice log; + protected const int NumOps = 5000; + protected AdId[] inputArray; + + protected void BaseSetup() + { + inputArray = new AdId[NumOps]; + for (int i = 0; i < NumOps; i++) + inputArray[i].adId = i; + + log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "hlog.log"), deleteOnClose: false); + RecreateDirectory(MethodTestDir); + } + + protected void BaseTearDown() + { + log?.Dispose(); + log = null; + TestUtils.OnTearDown(); + } + + protected static void AssertEquivalentTailAddress(long tailAddress1, long tailAddress2, long pageSize, int iteration) + { + if (tailAddress1 != tailAddress2) + { + // We adjust TailAddress in recovery to start at PageHeader.Size offset within the page if it ended on a page boundary + // in the RecoveryInfo, so test for that case here. + Assert.That(tailAddress1 / pageSize == tailAddress2 / pageSize && tailAddress2 % pageSize == PageHeader.Size, Is.True, + $"iteration {iteration}: tailAddress1 != tailAddress2 even after adjusting for PageHeader.Size offset"); + } + } + + public class MyFunctions : SimpleLongSimpleFunctions + { + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref long input, ref long output, Empty ctx, Status status, RecordMetadata recordMetadata) + { + ClassicAssert.IsTrue(status.Found, $"status = {status}"); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef(), output, $"output = {output}"); + } + } + + public class MyFunctions2 : SimpleLongSimpleFunctions + { + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref long input, ref long output, Empty ctx, Status status, RecordMetadata recordMetadata) + { + Verify(status, diskLogRecord.Key.AsRef(), output); + } + + internal static void Verify(Status status, long key, long output) + { + ClassicAssert.IsTrue(status.Found); + if (key < 950) + ClassicAssert.AreEqual(key, output); + else + ClassicAssert.AreEqual(key + 1, output); + } + } + } + [TestFixture] + public class RecoveryCheck1Tests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + [Test] + [Category("TsavoriteKV")] + [Category("CheckpointRestore")] + [Category("Smoke")] + + public async ValueTask RecoveryCheck1( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values] CompletionSyncMode completionSyncMode, [Values] ReadCacheMode readCacheMode, [Values(1L << 13, 1L << 16)] long indexSize) + { + const long pageSize = 1L << 10; + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new MyFunctions()); + var bc1 = s1.BasicContext; + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + if (readCacheMode == ReadCacheMode.UseRC) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 0; key < 1000; key++) + { + keyLong = key; + long output = default; + + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + } + + var task = store1.TakeFullCheckpointAsync(checkpointType); + + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + if (completionSyncMode == CompletionSyncMode.Async) + { + var (status, token) = await task; + _ = await store2.RecoverAsync(default, token); + } + else + { + var (status, token) = task.AsTask().GetAwaiter().GetResult(); + _ = store2.Recover(default, token); + } + + ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); + ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); + AssertEquivalentTailAddress(store1.Log.TailAddress, store2.Log.TailAddress, pageSize, iteration: 0); + + using var s2 = store2.NewSession(new MyFunctions()); + var bc2 = s2.BasicContext; + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + } + } + [TestFixture] + public class RecoveryCheck2Tests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + //[Repeat(3000)] + public async ValueTask RecoveryCheck2( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values] CompletionSyncMode completionSyncMode, [Values] ReadCacheMode readCacheMode, [Values(1L << 13, 1L << 16)] long indexSize) + { + const long pageSize = 1L << 10; + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new SimpleLongSimpleFunctions()); + var bc1 = s1.BasicContext; + + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (int iter = 0; iter < 5; iter++) + { + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + if (readCacheMode == ReadCacheMode.UseRC) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + keyLong = key; + long output = default; + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + + var task = store1.TakeHybridLogCheckpointAsync(checkpointType); + + if (completionSyncMode == CompletionSyncMode.Async) + { + var (status, token) = await task; + _ = await store2.RecoverAsync(default, token); + } + else + { + var (status, token) = task.AsTask().GetAwaiter().GetResult(); + _ = store2.Recover(default, token); + } + + ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress, $"iter {iter}"); + ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress, $"iter {iter}"); + AssertEquivalentTailAddress(store1.Log.TailAddress, store2.Log.TailAddress, pageSize, iteration: iter); + + using var s2 = store2.NewSession(new SimpleLongSimpleFunctions()); + var bc2 = s2.BasicContext; + for (long key = 0; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + } + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + public void RecoveryCheck2Repeated([Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType) + { + Guid token = default; + const long pageSize = 1L << 10; + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (int iter = 0; iter < 6; iter++) + { + using var store = new TsavoriteKV(new() + { + IndexSize = 1L << 13, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + if (iter > 0) + _ = store.Recover(default, token); + + using var s1 = store.NewSession(new SimpleLongSimpleFunctions()); + var bc1 = s1.BasicContext; + + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + var task = store.TakeHybridLogCheckpointAsync(checkpointType); + bool success; + (success, token) = task.AsTask().GetAwaiter().GetResult(); + ClassicAssert.IsTrue(success); + + using var s2 = store.NewSession(new SimpleLongSimpleFunctions()); + var bc2 = s2.BasicContext; + + for (long key = 0; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + } + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + public void RecoveryRollback([Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType) + { + const long pageSize = 1L << 10; + using var store = new TsavoriteKV(new() + { + IndexSize = 1L << 13, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 11, + SegmentSize = 1L << 11, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store.NewSession(new SimpleLongSimpleFunctions()); + var bc1 = s1.BasicContext; + + for (long key = 0; key < 1000; key++) + _ = bc1.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); + + var task = store.TakeHybridLogCheckpointAsync(checkpointType); + (bool success, Guid token) = task.AsTask().GetAwaiter().GetResult(); + ClassicAssert.IsTrue(success); + + for (long key = 0; key < 1000; key++) + { + long output = default; + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + + for (long key = 1000; key < 2000; key++) + _ = bc1.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); + + // Reset store to empty state + store.Reset(); + + for (long key = 0; key < 2000; key++) + { + long output = default; + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.NotFound, $"status = {status}, key = {key}, wasPending = {wasPending}"); + } + + // Rollback to previous checkpoint + _ = store.Recover(default, token); + + for (long key = 0; key < 1000; key++) + { + long output = default; + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + + for (long key = 1000; key < 2000; key++) + { + long output = default; + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.NotFound, $"status = {status}, key = {key}, wasPending = {wasPending}"); + } + + for (long key = 1000; key < 2000; key++) + _ = bc1.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); + + for (long key = 0; key < 2000; key++) + { + long output = default; + var status = bc1.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + } + } + [TestFixture] + public class RecoveryCheck3Tests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + public async ValueTask RecoveryCheck3( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values] CompletionSyncMode completionSyncMode, [Values] ReadCacheMode readCacheMode, [Values(1L << 13, 1L << 16)] long indexSize) + { + const long pageSize = 1L << 10; + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new SimpleLongSimpleFunctions()); + var bc1 = s1.BasicContext; + + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (int iter = 0; iter < 5; iter++) + { + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + if (readCacheMode == ReadCacheMode.UseRC) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + keyLong = key; + long output = default; + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + + var task = store1.TakeFullCheckpointAsync(checkpointType); + + if (completionSyncMode == CompletionSyncMode.Async) + { + var (status, token) = await task; + _ = await store2.RecoverAsync(default, token); + } + else + { + var (status, token) = task.AsTask().GetAwaiter().GetResult(); + _ = store2.Recover(default, token); + } + + ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress, $"iter {iter}"); + ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress, $"iter {iter}"); + AssertEquivalentTailAddress(store1.Log.TailAddress, store2.Log.TailAddress, pageSize, iteration: iter); + + using var s2 = store2.NewSession(new SimpleLongSimpleFunctions()); + var bc2 = s2.BasicContext; + for (long key = 0; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + // Local variables in an async function can be moved, so we must copy the key + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + } + } + [TestFixture] + public class RecoveryCheck4Tests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + [Test] + [Category("TsavoriteKV"), Category("CheckpointRestore")] + public async ValueTask RecoveryCheck4( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values] CompletionSyncMode completionSyncMode, [Values] ReadCacheMode readCacheMode, [Values(1L << 13, 1L << 16)] long indexSize) + { + const long pageSize = 1L << 10; + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new SimpleLongSimpleFunctions()); + var bc1 = s1.BasicContext; + + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (int iter = 0; iter < 5; iter++) + { + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + if (readCacheMode == ReadCacheMode.UseRC) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 1000 * iter; key < 1000 * iter + 1000; key++) + { + keyLong = key; + long output = default; + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + + if (iter == 0) + _ = store1.TakeIndexCheckpointAsync().AsTask().GetAwaiter().GetResult(); + var task = store1.TakeHybridLogCheckpointAsync(checkpointType); + + if (completionSyncMode == CompletionSyncMode.Async) + { + var (status, token) = await task; + _ = await store2.RecoverAsync(default, token); + } + else + { + var (status, token) = task.AsTask().GetAwaiter().GetResult(); + _ = store2.Recover(default, token); + } + + ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress, $"iter {iter}"); + ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress, $"iter {iter}"); + AssertEquivalentTailAddress(store1.Log.TailAddress, store2.Log.TailAddress, pageSize, iteration: iter); + + using var s2 = store2.NewSession(new SimpleLongSimpleFunctions()); + var bc2 = s2.BasicContext; + for (long key = 0; key < 1000 * iter + 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, key = {key}, wasPending = {wasPending}, iter = {iter}"); + } + } + } + } + [TestFixture] + public class RecoveryCheck5Tests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + [Test] + [Category("TsavoriteKV")] + [Category("CheckpointRestore")] + public async ValueTask RecoveryCheck5( + [Values(CheckpointType.Snapshot, CheckpointType.FoldOver)] CheckpointType checkpointType, + [Values] bool isAsync, [Values] bool useReadCache, [Values(1L << 13, 1L << 16)] long indexSize) + { + const long pageSize = 1L << 10; + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = useReadCache, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new MyFunctions()); + var bc1 = s1.BasicContext; + + // Local variables in an async function can be moved, so we must use an array for the key + var keyArray = new byte[sizeof(long)]; + + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + + if (useReadCache) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 0; key < 1000; key++) + { + keyLong = key; + + long output = default; + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, key = {key}, wasPending = {wasPending}"); + } + } + + var result = await store1.GrowIndexAsync(); + ClassicAssert.IsTrue(result); + + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, key = {key}, wasPending = {wasPending}"); + } + + var task = store1.TakeFullCheckpointAsync(checkpointType); + + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = pageSize, + LogMemorySize = 1L << 20, + ReadCacheEnabled = useReadCache, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + if (isAsync) + { + var (status, token) = await task; + _ = await store2.RecoverAsync(default, token); + } + else + { + var (status, token) = task.AsTask().GetAwaiter().GetResult(); + _ = store2.Recover(default, token); + } + + ClassicAssert.AreEqual(store1.Log.HeadAddress, store2.Log.HeadAddress); + ClassicAssert.AreEqual(store1.Log.ReadOnlyAddress, store2.Log.ReadOnlyAddress); + AssertEquivalentTailAddress(store1.Log.TailAddress, store2.Log.TailAddress, pageSize, iteration: 0); + + using var s2 = store2.NewSession(new MyFunctions()); + var bc2 = s2.BasicContext; + + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, key = {key}, wasPending = {wasPending}"); + } + } + } + [TestFixture] + public class RecoveryCheckSnapshotTests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + } + [TestFixture] + public class RecoveryCheckStreamingSnapshotTests : RecoveryCheckBase + { + [SetUp] + public void Setup() => BaseSetup(); + + [TearDown] + public void TearDown() => BaseTearDown(); + + public class SnapshotIterator : IStreamingSnapshotIteratorFunctions + { + readonly TsavoriteKV store2; + readonly long expectedCount; + + ClientSession session2; + BasicContext bc2; + + public SnapshotIterator(TsavoriteKV store2, long expectedCount) + { + this.store2 = store2; + this.expectedCount = expectedCount; + } + + public bool OnStart(Guid checkpointToken, long currentVersion, long nextVersion) + { + store2.SetVersion(nextVersion); + session2 = store2.NewSession(new MyFunctions()); + bc2 = session2.BasicContext; + return true; + } + + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords) + where TSourceLogRecord : ISourceLogRecord + { + _ = bc2.Upsert(TestSpanByteKey.FromPinnedSpan(logRecord.Key), logRecord.ValueSpan); + return true; + } + + public void OnException(Exception exception, long numberOfRecords) + => Assert.Fail(exception.Message); + + public void OnStop(bool completed, long numberOfRecords) + { + Assert.That(numberOfRecords, Is.EqualTo(expectedCount)); + session2.Dispose(); + } + } + + [Test] + [Category("TsavoriteKV")] + [Category("CheckpointRestore")] + [Category("Smoke")] + + public async ValueTask StreamingSnapshotBasicTest([Values] CompletionSyncMode completionSyncMode, [Values] ReadCacheMode readCacheMode, + [Values] bool reInsert, [Values(1L << 13, 1L << 16)] long indexSize) + { + using var store1 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = 1L << 10, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + using var s1 = store1.NewSession(new MyFunctions()); + var bc1 = s1.BasicContext; + + // Local variables in an async function can be moved, so we must use an array for the key and value + var keyArray = new byte[sizeof(long)]; + var valueArray = new byte[sizeof(long)]; + + for (long key = 0; key < (reInsert ? 800 : 1000); key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + var valueSpan = new Span(valueArray); + ref var valueLong = ref valueSpan.AsRef(); + keyLong = key; + // If reInsert, we insert the wrong value during the first pass for the first 500 keys + valueLong = reInsert && key < 500 ? key + 1 : key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), valueSpan); + } + + if (reInsert) + { + store1.Log.FlushAndEvict(true); + for (long key = 0; key < 500; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + for (long key = 800; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + _ = bc1.Upsert(TestSpanByteKey.FromArray(keyArray), keySpan); + } + } + + if (readCacheMode == ReadCacheMode.UseRC) + { + store1.Log.FlushAndEvict(true); + + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + + for (long key = 0; key < 1000; key++) + { + keyLong = key; + long output = default; + // Local variables in an async function can be moved, so we must copy the key + var status = bc1.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc1.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + } + + // First create the new store, we will insert into this store as part of the iterator functions on the old store + using var store2 = new TsavoriteKV(new() + { + IndexSize = indexSize, + LogDevice = log, + MutableFraction = 1, + PageSize = 1L << 10, + LogMemorySize = 1L << 20, + ReadCacheEnabled = readCacheMode == ReadCacheMode.UseRC, + CheckpointDir = MethodTestDir + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + // Take a streaming snapshot checkpoint of the old store + var iterator = new SnapshotIterator(store2, 1000); + var task = store1.TakeFullCheckpointAsync(CheckpointType.StreamingSnapshot, streamingSnapshotIteratorFunctions: iterator); + if (completionSyncMode == CompletionSyncMode.Async) + _ = await task; + else + _ = task.AsTask().GetAwaiter().GetResult(); + + // Verify that the new store has all the records + using var s2 = store2.NewSession(new MyFunctions()); + var bc2 = s2.BasicContext; + for (long key = 0; key < 1000; key++) + { + var keySpan = new Span(keyArray); + ref var keyLong = ref keySpan.AsRef(); + keyLong = key; + + long output = default; + var status = bc2.Read(TestSpanByteKey.FromArray(keyArray), ref output); + var wasPending = status.IsPending; + if (wasPending) + { + Assert.That(bc2.CompletePendingWithOutputs(out var completedOutputs, wait: true), Is.True); + (status, output) = GetSinglePendingResult(completedOutputs); + } + ClassicAssert.IsTrue(status.Found, $"status = {status}, key = {key}, wasPending = {wasPending}"); + ClassicAssert.AreEqual(key, output, $"output = {output}, key = {key}, wasPending = {wasPending}"); + } + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/RecoveryTests.cs b/libs/storage/Tsavorite/cs/test/test.recovery/RecoveryTests.cs similarity index 57% rename from libs/storage/Tsavorite/cs/test/RecoveryTests.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/RecoveryTests.cs index f8b76b4c91e..deefea80307 100644 --- a/libs/storage/Tsavorite/cs/test/RecoveryTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/RecoveryTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,19 +13,15 @@ namespace Tsavorite.test.recovery.sumstore { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - using MyValueAllocator = GenericAllocator>>; - using MyValueStoreFunctions = StoreFunctions>; + using ClassAllocator = ObjectAllocator>; + using ClassStoreFunctions = StoreFunctions; - using SpanByteStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class DeviceTypeRecoveryTests : AllureTestBase + internal class DeviceTypeRecoveryTests : TestBase { internal const long NumUniqueKeys = 1L << 12; internal const long KeySpace = 1L << 20; @@ -34,7 +29,7 @@ internal class DeviceTypeRecoveryTests : AllureTestBase internal const long CompletePendingInterval = 1L << 10; internal const long CheckpointInterval = 1L << 14; - private TsavoriteKV store; + private TsavoriteKV store; private readonly List logTokens = []; private readonly List indexTokens = []; private IDevice log; @@ -55,9 +50,9 @@ private void Setup(TestDeviceType deviceType) { IndexSize = KeySpace, LogDevice = log, - SegmentSize = 1L << 25, //MemorySize = 1L << 14, PageSize = 1L << 9, // locks ups at session.RMW line in Populate() for Local Memory + SegmentSize = 1L << 25, //LogMemorySize = 1L << 14, PageSize = 1L << 9, // locks ups at session.RMW line in Populate() for Local Memory CheckpointDir = MethodTestDir - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -86,7 +81,7 @@ private void PrepareToRecover(TestDeviceType deviceType) [Test] [Category("TsavoriteKV")] [Category("CheckpointRestore")] - public async ValueTask RecoveryTestSeparateCheckpoint([Values] bool isAsync, [Values] TestDeviceType deviceType) + public async ValueTask RecoveryTestSeparateCheckpoint([Values] CompletionSyncMode syncMode, [Values] TestDeviceType deviceType) { Setup(deviceType); Populate(SeparateCheckpointAction); @@ -95,7 +90,7 @@ public async ValueTask RecoveryTestSeparateCheckpoint([Values] bool isAsync, [Va { if (i >= indexTokens.Count) break; PrepareToRecover(deviceType); - await RecoverAndTestAsync(i, isAsync).ConfigureAwait(false); + await RecoverAndTest(i, syncMode == CompletionSyncMode.Async).ConfigureAwait(false); } } @@ -103,7 +98,7 @@ public async ValueTask RecoveryTestSeparateCheckpoint([Values] bool isAsync, [Va [Category("TsavoriteKV")] [Category("CheckpointRestore")] [Category("Smoke")] - public async ValueTask RecoveryTestFullCheckpoint([Values] bool isAsync, [Values] TestDeviceType deviceType) + public async ValueTask RecoveryTestFullCheckpoint([Values] CompletionSyncMode syncMode, [Values] TestDeviceType deviceType) { Setup(deviceType); Populate(FullCheckpointAction); @@ -111,7 +106,7 @@ public async ValueTask RecoveryTestFullCheckpoint([Values] bool isAsync, [Values for (var i = 0; i < logTokens.Count; i++) { PrepareToRecover(deviceType); - await RecoverAndTestAsync(i, isAsync).ConfigureAwait(false); + await RecoverAndTest(i, syncMode == CompletionSyncMode.Async).ConfigureAwait(false); } } @@ -150,7 +145,7 @@ private void SeparateCheckpointAction(int opNum) private void Populate(Action checkpointAction) { // Prepare the dataset - var inputArray = new AdInput[NumOps]; + var inputArray = GC.AllocateArray((int)NumOps, pinned: true); for (int i = 0; i < NumOps; i++) { inputArray[i].adId.adId = i % NumUniqueKeys; @@ -158,13 +153,13 @@ private void Populate(Action checkpointAction) } // Register thread with Tsavorite - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; // Process the batch of input data for (int i = 0; i < NumOps; i++) { - _ = bContext.RMW(ref inputArray[i].adId, ref inputArray[i], Empty.Default); + _ = bContext.RMW(inputArray[i].adId, ref inputArray[i], Empty.Default); checkpointAction(i); @@ -176,7 +171,7 @@ private void Populate(Action checkpointAction) _ = bContext.CompletePending(true); } - private async ValueTask RecoverAndTestAsync(int tokenIndex, bool isAsync) + private async ValueTask RecoverAndTest(int tokenIndex, bool isAsync) { var logToken = logTokens[tokenIndex]; var indexToken = indexTokens[tokenIndex]; @@ -188,7 +183,7 @@ private async ValueTask RecoverAndTestAsync(int tokenIndex, bool isAsync) _ = store.Recover(indexToken, logToken); // Create array for reading - var inputArray = new AdInput[NumUniqueKeys]; + var inputArray = GC.AllocateArray((int)NumUniqueKeys, pinned: true); for (int i = 0; i < NumUniqueKeys; i++) { inputArray[i].adId.adId = i; @@ -196,7 +191,7 @@ private async ValueTask RecoverAndTestAsync(int tokenIndex, bool isAsync) } // Register with thread - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; AdInput input = default; @@ -205,7 +200,7 @@ private async ValueTask RecoverAndTestAsync(int tokenIndex, bool isAsync) // Issue read requests for (var i = 0; i < NumUniqueKeys; i++) { - var status = bContext.Read(ref inputArray[i].adId, ref input, ref output, Empty.Default); + var status = bContext.Read(inputArray[i].adId, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found, $"At tokenIndex {tokenIndex}, keyIndex {i}, AdId {inputArray[i].adId.adId}"); inputArray[i].numClicks = output.value; } @@ -214,10 +209,8 @@ private async ValueTask RecoverAndTestAsync(int tokenIndex, bool isAsync) _ = bContext.CompletePending(true); } } - - [AllureNUnit] [TestFixture] - public class AllocatorTypeRecoveryTests : AllureTestBase + internal class AllocatorTypeRecoveryTests : TestBase { const int StackAllocMax = 12; const int RandSeed = 101; @@ -242,25 +235,29 @@ public void Setup() DeleteDirectory(MethodTestDir, true); } - private TsavoriteKV Setup(AllocatorType allocatorType, Func storeFunctionsCreator, Func allocatorCreator) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + private TsavoriteKV Setup(AllocatorType allocatorType, Func storeFunctionsCreator, Func allocatorCreator) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - log = new LocalMemoryDevice(1L << 26, 1L << 22, 2, sector_size: smallSector ? 64 : (uint)512, fileName: Path.Join(MethodTestDir, $"{typeof(TData).Name}.log")); - objlog = allocatorType == AllocatorType.Generic - ? new LocalMemoryDevice(1L << 26, 1L << 22, 2, fileName: Path.Join(MethodTestDir, $"{typeof(TData).Name}.obj.log")) - : null; - - var result = new TsavoriteKV(new() + var kvSettings = new KVSettings() { IndexSize = DeviceTypeRecoveryTests.KeySpace, LogDevice = log, ObjectLogDevice = objlog, SegmentSize = 1L << 25, + ObjectLogSegmentSize = 1L << 27, CheckpointDir = MethodTestDir - }, storeFunctionsCreator() - , allocatorCreator - ); + }; + + log = new LocalMemoryDevice(kvSettings.SegmentSize * 4, 1L << 22, 2, sector_size: smallSector ? 64 : (uint)512, fileName: Path.Join(MethodTestDir, $"{allocatorType}.log")); + objlog = allocatorType == AllocatorType.Object + ? new LocalMemoryDevice(capacity: kvSettings.ObjectLogSegmentSize * 4, 1L << 22, 2, fileName: Path.Join(MethodTestDir, $"{allocatorType}.obj.log")) + : null; + + kvSettings.LogDevice = log; + kvSettings.ObjectLogDevice = objlog; + + var result = new TsavoriteKV(kvSettings, storeFunctionsCreator(), allocatorCreator); storeDisp = result; return result; @@ -283,46 +280,42 @@ private void TearDown(bool deleteDir) DeleteDirectory(MethodTestDir); } - private TsavoriteKV PrepareToRecover(AllocatorType allocatorType, + private TsavoriteKV PrepareToRecover(AllocatorType allocatorType, Func storeFunctionsCreator, Func allocatorCreator) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { TearDown(deleteDir: false); - return Setup(allocatorType, storeFunctionsCreator, allocatorCreator); + return Setup(allocatorType, storeFunctionsCreator, allocatorCreator); } [Test] [Category("TsavoriteKV")] [Category("CheckpointRestore")] - public async ValueTask RecoveryTestByAllocatorType([Values] AllocatorType allocatorType, [Values] bool isAsync) + public async ValueTask RecoveryTestByAllocatorType([Values] AllocatorType allocatorType, [Values] CompletionSyncMode syncMode) { - await TestDriver(allocatorType, isAsync).ConfigureAwait(false); + await TestDriver(allocatorType, syncMode == CompletionSyncMode.Async).ConfigureAwait(false); } [Test] [Category("TsavoriteKV")] [Category("CheckpointRestore")] - public async ValueTask RecoveryTestFailOnSectorSize([Values] AllocatorType allocatorType, [Values] bool isAsync) + public async ValueTask RecoveryTestFailOnSectorSize([Values] AllocatorType allocatorType, [Values] CompletionSyncMode syncMode) { smallSector = true; - await TestDriver(allocatorType, isAsync).ConfigureAwait(false); + await TestDriver(allocatorType, syncMode == CompletionSyncMode.Async).ConfigureAwait(false); } private async ValueTask TestDriver(AllocatorType allocatorType, [Values] bool isAsync) { var task = allocatorType switch { - AllocatorType.FixedBlittable => RunTest(allocatorType, - () => StoreFunctions.Create(LongKeyComparer.Instance), + AllocatorType.SpanByte => RunTest(allocatorType, + () => StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance), (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions), Populate, Read, Recover, isAsync), - AllocatorType.SpanByte => RunTest>(allocatorType, - StoreFunctions.Create, - (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions), - Populate, Read, Recover, isAsync), - AllocatorType.Generic => RunTest(allocatorType, - () => StoreFunctions.Create(new MyValue.Comparer(), () => new MyValueSerializer(), () => new MyValueSerializer(), DefaultRecordDisposer.Instance), + AllocatorType.Object => RunTest(allocatorType, + () => StoreFunctions.Create(new TestObjectKey.Comparer(), () => new TestObjectValue.Serializer(), DefaultRecordTriggers.Instance), (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions), Populate, Read, Recover, isAsync), _ => throw new ApplicationException("Unknown allocator type"), @@ -331,21 +324,21 @@ private async ValueTask TestDriver(AllocatorType allocatorType, [Values] bool is await task.ConfigureAwait(false); } - private async ValueTask RunTest(AllocatorType allocatorType, + private async ValueTask RunTest(AllocatorType allocatorType, Func storeFunctionsCreator, Func allocatorCreator, - Action> populateAction, - Action> readAction, - Func, bool, ValueTask> recoverFunc, + Action> populateAction, + Action> readAction, + Func, bool, ValueTask> recoverFunc, bool isAsync) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - var store = Setup(allocatorType, storeFunctionsCreator, allocatorCreator); + var store = Setup(allocatorType, storeFunctionsCreator, allocatorCreator); populateAction(store); readAction(store); if (smallSector) { - Assert.ThrowsAsync(async () => await Checkpoint(store, isAsync).ConfigureAwait(false)); + _ = Assert.ThrowsAsync(async () => await Checkpoint(store, isAsync).ConfigureAwait(false)); Assert.Pass("Verified expected exception; the test cannot continue, so exiting early with success"); } else @@ -355,32 +348,22 @@ private async ValueTask RunTest(AllocatorTyp ClassicAssert.AreNotEqual(Guid.Empty, indexToken); readAction(store); - store = PrepareToRecover(allocatorType, storeFunctionsCreator, allocatorCreator); + store = PrepareToRecover(allocatorType, storeFunctionsCreator, allocatorCreator); await recoverFunc(store, isAsync).ConfigureAwait(false); readAction(store); } - private void Populate(TsavoriteKV store) - { - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var bContext = session.BasicContext; - - for (int i = 0; i < DeviceTypeRecoveryTests.NumOps; i++) - _ = bContext.Upsert(i % DeviceTypeRecoveryTests.NumUniqueKeys, i); - _ = bContext.CompletePending(true); - } - static int GetRandomLength(Random r) => r.Next(StackAllocMax) + 1; // +1 to remain in range 1..StackAllocMax - private unsafe void Populate(TsavoriteKV> store) + private unsafe void Populate(TsavoriteKV store) { - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; Random rng = new(RandSeed); // Single alloc outside the loop, to the max length we'll need. - Span keySpan = stackalloc int[1]; + AdId key = new(); Span valueSpan = stackalloc int[StackAllocMax]; for (int i = 0; i < DeviceTypeRecoveryTests.NumOps; i++) @@ -390,36 +373,34 @@ private unsafe void Populate(TsavoriteKV store) + private unsafe void Populate(TsavoriteKV store) { - using var session = store.NewSession(new MyFunctions2()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; for (int i = 0; i < DeviceTypeRecoveryTests.NumOps; i++) { - var key = new MyValue { value = i % (int)DeviceTypeRecoveryTests.NumUniqueKeys }; - var value = new MyValue { value = i }; + var key = new TestObjectKey { key = i % (int)DeviceTypeRecoveryTests.NumUniqueKeys }; + var value = new TestObjectValue { value = i }; _ = bContext.Upsert(key, value); } _ = bContext.CompletePending(true); } - private async ValueTask Checkpoint(TsavoriteKV store, bool isAsync) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + private async ValueTask Checkpoint(TsavoriteKV store, bool isAsync) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { if (isAsync) { @@ -435,79 +416,59 @@ private async ValueTask Checkpoint(Tsavorite indexToken = logToken; } - private async ValueTask RecoverAndReadTest(TsavoriteKV store, bool isAsync) + private async ValueTask RecoverAndReadTest(TsavoriteKV store, bool isAsync) { await Recover(store, isAsync).ConfigureAwait(false); Read(store); } - private static void Read(TsavoriteKV store) + private static void Read(TsavoriteKV store) { - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var bContext = session.BasicContext; - - for (var i = 0; i < DeviceTypeRecoveryTests.NumUniqueKeys; i++) - { - var status = bContext.Read(i % DeviceTypeRecoveryTests.NumUniqueKeys, default, out long output); - ClassicAssert.IsTrue(status.Found, $"keyIndex {i}"); - ClassicAssert.AreEqual(ExpectedValue(i), output); - } - } - - private async ValueTask RecoverAndReadTest(TsavoriteKV> store, bool isAsync) - { - await Recover(store, isAsync).ConfigureAwait(false); - Read(store); - } - - private static void Read(TsavoriteKV> store) - { - using var session = store.NewSession(new VLVectorFunctions()); + using var session = store.NewSession(new VLVectorFunctions()); var bContext = session.BasicContext; Random rng = new(RandSeed); - Span keySpan = stackalloc int[1]; - var keySpanByte = keySpan.AsSpanByte(); + AdId key = new(); for (var i = 0; i < DeviceTypeRecoveryTests.NumUniqueKeys; i++) { - keySpan[0] = i; - - var len = GetRandomLength(rng); + key.adId = i; int[] output = null; - var status = bContext.Read(ref keySpanByte, ref output, Empty.Default); - + var status = bContext.Read(key, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); + + var len = GetRandomLength(rng); for (int j = 0; j < len; j++) ClassicAssert.AreEqual(ExpectedValue(i), output[j], $"mismatched data at position {j}, len {len}"); } } - private async ValueTask RecoverAndReadTest(TsavoriteKV store, bool isAsync) + private async ValueTask RecoverAndReadTest(TsavoriteKV store, bool isAsync) { await Recover(store, isAsync).ConfigureAwait(false); Read(store); } - private static void Read(TsavoriteKV store) + private static void Read(TsavoriteKV store) { - using var session = store.NewSession(new MyFunctions2()); + using var session = store.NewSession(new TestObjectFunctions()); var bContext = session.BasicContext; for (var i = 0; i < DeviceTypeRecoveryTests.NumUniqueKeys; i++) { - var key = new MyValue { value = i }; - var status = bContext.Read(key, default, out MyOutput output); + var key = new TestObjectKey { key = i }; + var output = new TestObjectOutput(); + var status = bContext.Read(key, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found, $"keyIndex {i}"); ClassicAssert.AreEqual(ExpectedValue(i), output.value.value); } } - private async ValueTask Recover(TsavoriteKV store, bool isAsync = false) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + private async ValueTask Recover(TsavoriteKV store, bool isAsync = false) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { if (isAsync) _ = await store.RecoverAsync(indexToken, logToken).ConfigureAwait(false); diff --git a/libs/storage/Tsavorite/cs/test/SimpleRecoveryTest.cs b/libs/storage/Tsavorite/cs/test/test.recovery/SimpleRecoveryTest.cs similarity index 74% rename from libs/storage/Tsavorite/cs/test/SimpleRecoveryTest.cs rename to libs/storage/Tsavorite/cs/test/test.recovery/SimpleRecoveryTest.cs index e0df3807f9e..487e0e0b5a3 100644 --- a/libs/storage/Tsavorite/cs/test/SimpleRecoveryTest.cs +++ b/libs/storage/Tsavorite/cs/test/test.recovery/SimpleRecoveryTest.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using Microsoft.Extensions.Logging; using NUnit.Framework; @@ -17,8 +16,8 @@ namespace Tsavorite.test.recovery.sumstore { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; public class CheckpointManagerWithCookie : DeviceLogCommitCheckpointManager { @@ -39,10 +38,8 @@ public CheckpointManagerWithCookie(bool testCommitCookie, INamedDeviceFactoryCre public override byte[] GetCookie() => Cookie; } - - [AllureNUnit] [TestFixture] - class RecoveryTests : AllureTestBase + class RecoveryTests : TestBase { const int NumOps = 5000; AdId[] inputArray; @@ -50,8 +47,8 @@ class RecoveryTests : AllureTestBase string checkpointDir; CheckpointManagerWithCookie checkpointManager; - private TsavoriteKV store1; - private TsavoriteKV store2; + private TsavoriteKV store1; + private TsavoriteKV store2; private IDevice log; @@ -122,10 +119,10 @@ private async ValueTask SimpleRecoveryTest1_Worker(CheckpointType checkpointType IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -134,10 +131,10 @@ private async ValueTask SimpleRecoveryTest1_Worker(CheckpointType checkpointType IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -145,13 +142,13 @@ private async ValueTask SimpleRecoveryTest1_Worker(CheckpointType checkpointType AdInput inputArg = default; Output output = default; - var session1 = store1.NewSession(new AdSimpleFunctions()); + var session1 = store1.NewSession(new AdSimpleFunctions()); var bContext1 = session1.BasicContext; for (int key = 0; key < NumOps; key++) { value.numClicks = key; - _ = bContext1.Upsert(ref inputArray[key], ref value, Empty.Default); + _ = bContext1.Upsert(inputArray[key], SpanByte.FromPinnedVariable(ref value), Empty.Default); } _ = store1.TryInitiateFullCheckpoint(out Guid token, checkpointType); @@ -171,13 +168,13 @@ private async ValueTask SimpleRecoveryTest1_Worker(CheckpointType checkpointType else ClassicAssert.Null(store2.RecoveredCommitCookie); - var session2 = store2.NewSession(new AdSimpleFunctions()); + var session2 = store2.NewSession(new AdSimpleFunctions()); var bContext2 = session2.BasicContext; ClassicAssert.AreEqual(1, session2.ID); // This is the first session on the recovered store for (int key = 0; key < NumOps; key++) { - var status = bContext2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default); + var status = bContext2.Read(inputArray[key], ref inputArg, ref output, Empty.Default); if (status.IsPending) { @@ -208,9 +205,9 @@ public async ValueTask SimpleRecoveryTest2( IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -219,9 +216,9 @@ public async ValueTask SimpleRecoveryTest2( IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -229,13 +226,13 @@ public async ValueTask SimpleRecoveryTest2( AdInput inputArg = default; Output output = default; - var session1 = store1.NewSession(new AdSimpleFunctions()); + var session1 = store1.NewSession(new AdSimpleFunctions()); var bContext1 = session1.BasicContext; for (int key = 0; key < NumOps; key++) { value.numClicks = key; - _ = bContext1.Upsert(ref inputArray[key], ref value, Empty.Default); + _ = bContext1.Upsert(inputArray[key], SpanByte.FromPinnedVariable(ref value), Empty.Default); } _ = store1.TryInitiateFullCheckpoint(out Guid token, checkpointType); store1.CompleteCheckpointAsync().AsTask().GetAwaiter().GetResult(); @@ -246,12 +243,12 @@ public async ValueTask SimpleRecoveryTest2( else _ = await store2.RecoverAsync(token).ConfigureAwait(false); - var session2 = store2.NewSession(new AdSimpleFunctions()); + var session2 = store2.NewSession(new AdSimpleFunctions()); var bContext2 = session1.BasicContext; for (int key = 0; key < NumOps; key++) { - var status = bContext2.Read(ref inputArray[key], ref inputArg, ref output, Empty.Default); + var status = bContext2.Read(inputArray[key], ref inputArg, ref output, Empty.Default); if (status.IsPending) _ = bContext2.CompletePending(true); @@ -273,9 +270,9 @@ public async ValueTask ShouldRecoverBeginAddress([Values] CompletionSyncMode com IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -284,22 +281,22 @@ public async ValueTask ShouldRecoverBeginAddress([Values] CompletionSyncMode com IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointDir = checkpointDir - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); NumClicks value; - var session1 = store1.NewSession(new AdSimpleFunctions()); + var session1 = store1.NewSession(new AdSimpleFunctions()); var bContext1 = session1.BasicContext; var address = 0L; for (int key = 0; key < NumOps; key++) { value.numClicks = key; - _ = bContext1.Upsert(ref inputArray[key], ref value, Empty.Default); + _ = bContext1.Upsert(inputArray[key], SpanByte.FromPinnedVariable(ref value), Empty.Default); if (key == 2999) address = store1.Log.TailAddress; @@ -334,9 +331,9 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -345,9 +342,9 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c IndexSize = 1L << 13, LogDevice = log, MutableFraction = 0.1, - MemorySize = 1L << 29, + LogMemorySize = 1L << 29, CheckpointManager = checkpointManager - }, StoreFunctions.Create(new AdId.Comparer()) + }, StoreFunctions.Create(new AdId.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); @@ -357,18 +354,18 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c AdSimpleFunctions functions1 = new(1); AdSimpleFunctions functions2 = new(2); - var session1 = store1.NewSession(functions1); + var session1 = store1.NewSession(functions1); var bContext1 = session1.BasicContext; for (int key = 0; key < NumOps; key++) { value.numClicks = key; if ((key & 1) > 0) - _ = bContext1.Upsert(ref inputArray[key], ref value, Empty.Default); + _ = bContext1.Upsert(inputArray[key], SpanByte.FromPinnedVariable(ref value), Empty.Default); else { AdInput input = new() { adId = inputArray[key], numClicks = value }; - _ = bContext1.RMW(ref inputArray[key], ref input); + _ = bContext1.RMW(inputArray[key], ref input); } } _ = store1.TryInitiateFullCheckpoint(out Guid token, CheckpointType.FoldOver); @@ -383,20 +380,20 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c else _ = await store2.RecoverAsync(token).ConfigureAwait(false); - var session2 = store2.NewSession(functions2); + var session2 = store2.NewSession(functions2); var bContext2 = session2.BasicContext; // Just need one operation here to verify readInfo/upsertInfo in the functions var lastKey = inputArray.Length - 1; - var status = bContext2.Read(ref inputArray[lastKey], ref inputArg, ref output, Empty.Default); + var status = bContext2.Read(inputArray[lastKey], ref inputArg, ref output, Empty.Default); ClassicAssert.IsFalse(status.IsPending, status.ToString()); value.numClicks = lastKey; - status = bContext2.Upsert(ref inputArray[lastKey], ref value, Empty.Default); + status = bContext2.Upsert(inputArray[lastKey], SpanByte.FromPinnedVariable(ref value), Empty.Default); ClassicAssert.IsFalse(status.IsPending, status.ToString()); inputArg = new() { adId = inputArray[lastKey], numClicks = new NumClicks { numClicks = 0 } }; // CopyUpdater adds, so make this 0 - status = bContext2.RMW(ref inputArray[lastKey], ref inputArg); + status = bContext2.RMW(inputArray[lastKey], ref inputArg); ClassicAssert.IsFalse(status.IsPending, status.ToString()); // Now verify Pending @@ -404,7 +401,7 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c output.value = new() { numClicks = lastKey }; inputArg.numClicks = new() { numClicks = lastKey }; - status = bContext2.Read(ref inputArray[lastKey], ref inputArg, ref output, Empty.Default); + status = bContext2.Read(inputArray[lastKey], ref inputArg, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext2.CompletePending(wait: true); @@ -413,7 +410,7 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c --lastKey; output.value = new() { numClicks = lastKey }; inputArg.numClicks = new() { numClicks = lastKey }; - status = bContext2.RMW(ref inputArray[lastKey], ref inputArg); + status = bContext2.RMW(inputArray[lastKey], ref inputArg); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext2.CompletePending(wait: true); @@ -421,65 +418,63 @@ public async ValueTask SimpleReadAndUpdateInfoTest([Values] CompletionSyncMode c } } - public class AdSimpleFunctions : SessionFunctionsBase + public class AdSimpleFunctions : SessionFunctionsBase { long expectedVersion; internal AdSimpleFunctions(long ver = -1) => expectedVersion = ver; - public override void ReadCompletionCallback(ref AdId key, ref AdInput input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref AdInput input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(key.adId, output.value.numClicks); + ClassicAssert.AreEqual(diskLogRecord.Key.AsRef().adId, output.value.numClicks); } // Read functions - public override bool SingleReader(ref AdId key, ref AdInput input, ref NumClicks value, ref Output dst, ref ReadInfo readInfo) - { - if (expectedVersion >= 0) - ClassicAssert.AreEqual(expectedVersion, readInfo.Version); - dst.value = value; - return true; - } - - public override bool ConcurrentReader(ref AdId key, ref AdInput input, ref NumClicks value, ref Output dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref AdInput input, ref Output output, ref ReadInfo readInfo) { if (expectedVersion >= 0) ClassicAssert.AreEqual(expectedVersion, readInfo.Version); - dst.value = value; + output.value = srcLogRecord.ValueSpan.AsRef(); return true; } // RMW functions - public override bool InitialUpdater(ref AdId key, ref AdInput input, ref NumClicks value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { if (expectedVersion >= 0) ClassicAssert.AreEqual(expectedVersion, rmwInfo.Version); - value = input.numClicks; + dstLogRecord.ValueSpan.AsRef() = input.numClicks; return true; } - public override bool InPlaceUpdater(ref AdId key, ref AdInput input, ref NumClicks value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { if (expectedVersion >= 0) ClassicAssert.AreEqual(expectedVersion, rmwInfo.Version); - _ = Interlocked.Add(ref value.numClicks, input.numClicks.numClicks); + _ = Interlocked.Add(ref logRecord.ValueSpan.AsRef().numClicks, input.numClicks.numClicks); return true; } - public override bool NeedCopyUpdate(ref AdId key, ref AdInput input, ref NumClicks oldValue, ref Output output, ref RMWInfo rmwInfo) + public override bool NeedCopyUpdate(in TSourceLogRecord srcLogRecord, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { if (expectedVersion >= 0) ClassicAssert.AreEqual(expectedVersion, rmwInfo.Version); return true; } - public override bool CopyUpdater(ref AdId key, ref AdInput input, ref NumClicks oldValue, ref NumClicks newValue, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref AdInput input, ref Output output, ref RMWInfo rmwInfo) { if (expectedVersion >= 0) ClassicAssert.AreEqual(expectedVersion, rmwInfo.Version); - newValue.numClicks += oldValue.numClicks + input.numClicks.numClicks; + dstLogRecord.ValueSpan.AsRef().numClicks += srcLogRecord.ValueSpan.AsRef().numClicks + input.numClicks.numClicks; return true; } + + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref AdInput input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = NumClicks.Size, ValueIsObject = false }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref AdInput input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = NumClicks.Size, ValueIsObject = false }; } } \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.recovery/Tsavorite.test.recovery.csproj b/libs/storage/Tsavorite/cs/test/test.recovery/Tsavorite.test.recovery.csproj new file mode 100644 index 00000000000..493794ca137 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.recovery/Tsavorite.test.recovery.csproj @@ -0,0 +1,32 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + false + + + diff --git a/libs/storage/Tsavorite/cs/test/test.session.context/FunctionPerSessionTests.cs b/libs/storage/Tsavorite/cs/test/test.session.context/FunctionPerSessionTests.cs new file mode 100644 index 00000000000..50257277c1d --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.session.context/FunctionPerSessionTests.cs @@ -0,0 +1,208 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using Garnet.test; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; + +namespace Tsavorite.test +{ + // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. + [StructLayout(LayoutKind.Sequential, Pack = 1)] + public struct RefCountedValueStruct + { + public static unsafe int Size => sizeof(RefCountedValueStruct); + public int ReferenceCount; + public long Value; + } +} + +namespace Tsavorite.test +{ + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; + + public class RefCountedAdder : SessionFunctionsBase + { + public int InitialCount; + public int InPlaceCount; + public int CopyCount; + + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref long input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref InitialCount); + + ref var value = ref logRecord.ValueSpan.AsRef(); + value.Value = input; + value.ReferenceCount = 1; + return true; + } + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref long input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref InPlaceCount); + + ref var value = ref logRecord.ValueSpan.AsRef(); + value.Value = input; + value.ReferenceCount++; + return true; + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref long input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref CopyCount); + + ref var oldValue = ref srcLogRecord.ValueSpan.AsRef(); + ref var newValue = ref dstLogRecord.ValueSpan.AsRef(); + newValue.Value = input; + newValue.ReferenceCount = oldValue.ReferenceCount + 1; + return true; + } + + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref long input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = RefCountedValueStruct.Size, ValueIsObject = false }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref long input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = RefCountedValueStruct.Size, ValueIsObject = false }; + } + + public class RefCountedRemover : SessionFunctionsBase + { + public int InitialCount; + public int InPlaceCount; + public int CopyCount; + + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref Empty input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref InitialCount); + + ref var value = ref logRecord.ValueSpan.AsRef(); + value.Value = 0; + value.ReferenceCount = 0; + return true; + } + + public override bool InPlaceUpdater(ref LogRecord logRecord, ref Empty input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref InPlaceCount); + + ref var value = ref logRecord.ValueSpan.AsRef(); + if (value.ReferenceCount > 0) + value.ReferenceCount--; + + return true; + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref Empty input, ref Empty output, ref RMWInfo rmwInfo) + { + _ = Interlocked.Increment(ref CopyCount); + + ref var oldValue = ref srcLogRecord.ValueSpan.AsRef(); + ref var newValue = ref dstLogRecord.ValueSpan.AsRef(); + + newValue.ReferenceCount = oldValue.ReferenceCount; + if (newValue.ReferenceCount > 0) + newValue.ReferenceCount--; + newValue.Value = oldValue.Value; + return true; + } + + /// + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref Empty input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = RefCountedValueStruct.Size, ValueIsObject = false }; + /// + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref Empty input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = RefCountedValueStruct.Size, ValueIsObject = false }; + } + + public class RefCountedReader : SessionFunctionsBase + { + public override bool Reader(in TSourceLogRecord srcLogRecord, ref Empty input, ref RefCountedValueStruct output, ref ReadInfo readInfo) + { + output = srcLogRecord.ValueSpan.AsRef(); + return true; + } + } + [TestFixture] + public class FunctionPerSessionTests : TestBase + { + private IDevice log; + private TsavoriteKV store; + private RefCountedAdder adder; + private RefCountedRemover remover; + private RefCountedReader reader; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + log = Devices.CreateLogDevice(Path.Join(TestUtils.MethodTestDir, "FunctionPerSessionTests1.log"), deleteOnClose: true); + + store = new(new() + { + IndexSize = 1L << 13, + LogDevice = log, + }, StoreFunctions.Create(IntKeyComparer.Instance, SpanByteRecordTriggers.Instance) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) + ); + + adder = new RefCountedAdder(); + remover = new RefCountedRemover(); + reader = new RefCountedReader(); + } + + [TearDown] + public void TearDown() + { + store?.Dispose(); + store = null; + log?.Dispose(); + log = null; + TestUtils.OnTearDown(); + } + + [Test] + [Category("TsavoriteKV")] + public void Should_create_multiple_sessions_with_different_callbacks() + { + using var adderSession = store.NewSession(adder); + using var removerSession = store.NewSession(remover); + using var readerSession = store.NewSession(reader); + var key = 101; + var input = 1000L; + + _ = adderSession.BasicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input); + _ = adderSession.BasicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input); + _ = adderSession.BasicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref input); + + ClassicAssert.AreEqual(1, adder.InitialCount); + ClassicAssert.AreEqual(2, adder.InPlaceCount); + + var empty = default(Empty); + _ = removerSession.BasicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref empty); + + ClassicAssert.AreEqual(1, remover.InPlaceCount); + + RefCountedValueStruct output = new(); + _ = readerSession.BasicContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref output); + + ClassicAssert.AreEqual(2, output.ReferenceCount); + ClassicAssert.AreEqual(1000L, output.Value); + + store.Log.FlushAndEvict(true); + + _ = removerSession.BasicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref empty); + _ = removerSession.BasicContext.CompletePending(wait: true); + _ = readerSession.BasicContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref empty, ref output); + + ClassicAssert.AreEqual(1, output.ReferenceCount); + ClassicAssert.AreEqual(1000L, output.Value); + ClassicAssert.AreEqual(1, remover.CopyCount); + } + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/SessionTests.cs b/libs/storage/Tsavorite/cs/test/test.session.context/SessionTests.cs similarity index 73% rename from libs/storage/Tsavorite/cs/test/SessionTests.cs rename to libs/storage/Tsavorite/cs/test/test.session.context/SessionTests.cs index 63b31d257b7..916b28cb4cd 100644 --- a/libs/storage/Tsavorite/cs/test/SessionTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session.context/SessionTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,14 +11,12 @@ namespace Tsavorite.test.Session { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class SessionTests : AllureTestBase + internal class SessionTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; [SetUp] @@ -31,8 +28,8 @@ public void Setup() { IndexSize = 1L << 13, LogDevice = log, - MemorySize = 1L << 29, - }, StoreFunctions.Create(new KeyStruct.Comparer()) + LogMemorySize = 1L << 29, + }, StoreFunctions.Create(KeyStruct.Comparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -52,7 +49,7 @@ public void TearDown() [Category("Smoke")] public void SessionTest1() { - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; InputStruct input = default; @@ -61,8 +58,8 @@ public void SessionTest1() var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -79,9 +76,9 @@ public void SessionTest1() [Category("TsavoriteKV")] public void SessionTest2() { - using var session1 = store.NewSession(new Functions()); + using var session1 = store.NewSession(new Functions()); var bContext1 = session1.BasicContext; - using var session2 = store.NewSession(new Functions()); + using var session2 = store.NewSession(new Functions()); var bContext2 = session2.BasicContext; InputStruct input = default; OutputStruct output = default; @@ -91,10 +88,10 @@ public void SessionTest2() var key2 = new KeyStruct { kfield1 = 15, kfield2 = 16 }; var value2 = new ValueStruct { vfield1 = 25, vfield2 = 26 }; - _ = bContext1.Upsert(ref key1, ref value1, Empty.Default); - _ = bContext2.Upsert(ref key2, ref value2, Empty.Default); + _ = bContext1.Upsert(key1, SpanByte.FromPinnedVariable(ref value1), Empty.Default); + _ = bContext2.Upsert(key2, SpanByte.FromPinnedVariable(ref value2), Empty.Default); - var status = bContext1.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext1.Read(key1, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -106,7 +103,7 @@ public void SessionTest2() ClassicAssert.AreEqual(value1.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value1.vfield2, output.value.vfield2); - status = bContext2.Read(ref key2, ref input, ref output, Empty.Default); + status = bContext2.Read(key2, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -123,7 +120,7 @@ public void SessionTest2() [Category("TsavoriteKV")] public void SessionTest3() { - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; Task.CompletedTask.ContinueWith((t) => @@ -134,8 +131,8 @@ public void SessionTest3() var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = bContext.Upsert(ref key1, ref value, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -153,9 +150,9 @@ public void SessionTest3() [Category("TsavoriteKV")] public void SessionTest4() { - using var session1 = store.NewSession(new Functions()); + using var session1 = store.NewSession(new Functions()); var bContext1 = session1.BasicContext; - using var session2 = store.NewSession(new Functions()); + using var session2 = store.NewSession(new Functions()); var bContext2 = session2.BasicContext; var t1 = Task.CompletedTask.ContinueWith((t) => { @@ -165,8 +162,8 @@ public void SessionTest4() var key1 = new KeyStruct { kfield1 = 14, kfield2 = 15 }; var value1 = new ValueStruct { vfield1 = 24, vfield2 = 25 }; - _ = bContext1.Upsert(ref key1, ref value1, Empty.Default); - var status = bContext1.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext1.Upsert(key1, SpanByte.FromPinnedVariable(ref value1), Empty.Default); + var status = bContext1.Read(key1, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -187,9 +184,9 @@ public void SessionTest4() var key2 = new KeyStruct { kfield1 = 15, kfield2 = 16 }; var value2 = new ValueStruct { vfield1 = 25, vfield2 = 26 }; - _ = bContext2.Upsert(ref key2, ref value2, Empty.Default); + _ = bContext2.Upsert(key2, SpanByte.FromPinnedVariable(ref value2), Empty.Default); - var status = bContext2.Read(ref key2, ref input, ref output, Empty.Default); + var status = bContext2.Read(key2, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -211,7 +208,7 @@ public void SessionTest4() public void SessionTest5() { // Not 'using' as we Dispose and recreate - var session = store.NewSession(new Functions()); + var session = store.NewSession(new Functions()); var bContext = session.BasicContext; InputStruct input = default; @@ -220,8 +217,8 @@ public void SessionTest5() var key1 = new KeyStruct { kfield1 = 16, kfield2 = 17 }; var value1 = new ValueStruct { vfield1 = 26, vfield2 = 27 }; - _ = bContext.Upsert(ref key1, ref value1, Empty.Default); - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value1), Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -235,15 +232,15 @@ public void SessionTest5() session.Dispose(); - session = store.NewSession(new Functions()); + session = store.NewSession(new Functions()); bContext = session.BasicContext; var key2 = new KeyStruct { kfield1 = 17, kfield2 = 18 }; var value2 = new ValueStruct { vfield1 = 27, vfield2 = 28 }; - _ = bContext.Upsert(ref key2, ref value2, Empty.Default); + _ = bContext.Upsert(key2, SpanByte.FromPinnedVariable(ref value2), Empty.Default); - status = bContext.Read(ref key2, ref input, ref output, Empty.Default); + status = bContext.Read(key2, ref input, ref output, Empty.Default); if (status.IsPending) { @@ -252,7 +249,7 @@ public void SessionTest5() } ClassicAssert.IsTrue(status.Found); - status = bContext.Read(ref key2, ref input, ref output, Empty.Default); + status = bContext.Read(key2, ref input, ref output, Empty.Default); if (status.IsPending) { diff --git a/libs/storage/Tsavorite/cs/test/LockableUnsafeContextTests.cs b/libs/storage/Tsavorite/cs/test/test.session.context/TransactionalUnsafeContextTests.cs similarity index 59% rename from libs/storage/Tsavorite/cs/test/LockableUnsafeContextTests.cs rename to libs/storage/Tsavorite/cs/test/test.session.context/TransactionalUnsafeContextTests.cs index 8321dea6f37..79a284d79ab 100644 --- a/libs/storage/Tsavorite/cs/test/LockableUnsafeContextTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session.context/TransactionalUnsafeContextTests.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,42 +16,55 @@ using Tsavorite.test.ReadCacheTests; using static Tsavorite.test.TestUtils; -namespace Tsavorite.test.LockableUnsafeContext +namespace Tsavorite.test.TransactionalUnsafeContext { // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - internal class LockableUnsafeComparer : IKeyComparer + internal class TransactionalUnsafeComparer : IKeyComparer { internal int maxSleepMs; readonly Random rng = new(101); - public bool Equals(ref long k1, ref long k2) => k1 == k2; - - public long GetHashCode64(ref long k) + public bool Equals(TKeyFirst k1, TKeySecond k2) + where TKeyFirst : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + where TKeySecond : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif + => k1.KeyBytes.AsRef() == k2.KeyBytes.AsRef(); + + public long GetHashCode64(TKey k) + where TKey : IKey +#if NET9_0_OR_GREATER + , allows ref struct +#endif { if (maxSleepMs > 0) Thread.Sleep(rng.Next(maxSleepMs)); - return Utility.GetHashCode(k); + return Utility.GetHashCode(k.KeyBytes.AsRef()); } } } -namespace Tsavorite.test.LockableUnsafeContext +namespace Tsavorite.test.TransactionalUnsafeContext { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; // Functions for the "Simple lock transaction" case, e.g.: // - Lock key1, key2, key3, keyResult // - Do some operation on value1, value2, value3 and write the result to valueResult - internal class LockableUnsafeFunctions : SimpleSimpleFunctions + internal class TransactionalUnsafeFunctions : SimpleLongSimpleFunctions { internal long recordAddress; - public override void PostSingleDeleter(ref long key, ref DeleteInfo deleteInfo) + public override void PostInitialDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { recordAddress = deleteInfo.Address; } - public override bool ConcurrentDeleter(ref long key, ref long value, ref DeleteInfo deleteInfo, ref RecordInfo recordInfo) + public override bool InPlaceDeleter(ref LogRecord logRecord, ref DeleteInfo deleteInfo) { recordAddress = deleteInfo.Address; return true; @@ -70,16 +82,16 @@ public BucketLockTracker() buckets = []; } - internal readonly void Increment(FixedLengthLockableKeyStruct key) => Increment(ref key); // easier with 'foreach' because iteration vars can't be passed by 'ref' - internal readonly void Increment(ref FixedLengthLockableKeyStruct key) + internal readonly void Increment(FixedLengthTransactionalKeyStruct key) => Increment(ref key); // easier with 'foreach' because iteration vars can't be passed by 'ref' + internal readonly void Increment(ref FixedLengthTransactionalKeyStruct key) { if (key.LockType == LockType.Exclusive) IncrementX(ref key); else IncrementS(ref key); } - internal readonly void Decrement(FixedLengthLockableKeyStruct key) => Decrement(ref key); - internal readonly void Decrement(ref FixedLengthLockableKeyStruct key) + internal readonly void Decrement(FixedLengthTransactionalKeyStruct key) => Decrement(ref key); + internal readonly void Decrement(ref FixedLengthTransactionalKeyStruct key) { if (key.LockType == LockType.Exclusive) DecrementX(ref key); @@ -87,12 +99,12 @@ internal readonly void Decrement(ref FixedLengthLockableKeyStruct key) DecrementS(ref key); } - internal readonly void IncrementX(ref FixedLengthLockableKeyStruct key) => AddX(ref key, 1); - internal readonly void DecrementX(ref FixedLengthLockableKeyStruct key) => AddX(ref key, -1); - internal readonly void IncrementS(ref FixedLengthLockableKeyStruct key) => AddS(ref key, 1); - internal readonly void DecrementS(ref FixedLengthLockableKeyStruct key) => AddS(ref key, -1); + internal readonly void IncrementX(ref FixedLengthTransactionalKeyStruct key) => AddX(ref key, 1); + internal readonly void DecrementX(ref FixedLengthTransactionalKeyStruct key) => AddX(ref key, -1); + internal readonly void IncrementS(ref FixedLengthTransactionalKeyStruct key) => AddS(ref key, 1); + internal readonly void DecrementS(ref FixedLengthTransactionalKeyStruct key) => AddS(ref key, -1); - private readonly void AddX(ref FixedLengthLockableKeyStruct key, int addend) + private readonly void AddX(ref FixedLengthTransactionalKeyStruct key, int addend) { if (!buckets.TryGetValue(key.KeyHash, out var counts)) counts = default; @@ -101,7 +113,7 @@ private readonly void AddX(ref FixedLengthLockableKeyStruct key, int adden buckets[key.KeyHash] = counts; } - private readonly void AddS(ref FixedLengthLockableKeyStruct key, int addend) + private readonly void AddS(ref FixedLengthTransactionalKeyStruct key, int addend) { if (!buckets.TryGetValue(key.KeyHash, out var counts)) counts = default; @@ -110,7 +122,7 @@ private readonly void AddS(ref FixedLengthLockableKeyStruct key, int adden buckets[key.KeyHash] = counts; } - internal readonly bool GetLockCounts(ref FixedLengthLockableKeyStruct key, out (int x, int s) counts) + internal readonly bool GetLockCounts(ref FixedLengthTransactionalKeyStruct key, out (int x, int s) counts) { if (!buckets.TryGetValue(key.KeyHash, out counts)) { @@ -141,10 +153,8 @@ internal readonly void AssertNoLocks() } } } - - [AllureNUnit] [TestFixture] - class LockableUnsafeContextTests : AllureTestBase + class TransactionalUnsafeContextTests : TestBase { const int NumRecords = 1000; const int UseNewKey = 1010; @@ -152,12 +162,12 @@ class LockableUnsafeContextTests : AllureTestBase const int ValueMult = 1_000_000; - LockableUnsafeFunctions functions; - LockableUnsafeComparer comparer; + TransactionalUnsafeFunctions functions; + TransactionalUnsafeComparer comparer; - private TsavoriteKV store; - private ClientSession session; - private BasicContext bContext; + private TsavoriteKV store; + private ClientSession session; + private BasicContext bContext; private IDevice log; [SetUp] @@ -171,12 +181,12 @@ public void Setup(bool forRecovery) } log = Devices.CreateLogDevice(Path.Combine(MethodTestDir, "test.log"), deleteOnClose: false, recoverDevice: forRecovery); - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = 1L << 26, LogDevice = log, PageSize = 1L << 12, - MemorySize = 1L << 22 + LogMemorySize = 1L << 22 }; foreach (var arg in TestContext.CurrentContext.Test.Arguments) @@ -198,15 +208,15 @@ public void Setup(bool forRecovery) } } - comparer = new LockableUnsafeComparer(); - functions = new LockableUnsafeFunctions(); + comparer = new TransactionalUnsafeComparer(); + functions = new TransactionalUnsafeFunctions(); store = new(kvSettings - , StoreFunctions.Create(comparer) + , StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - session = store.NewSession(functions); + session = store.NewSession(functions); bContext = session.BasicContext; } @@ -230,18 +240,21 @@ public void TearDown(bool forRecovery) void Populate() { - for (int key = 0; key < NumRecords; key++) - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + for (long key = 0; key < NumRecords; key++) + { + var value = key * ValueMult; + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)).IsPending); + } } - void AssertIsLocked(FixedLengthLockableKeyStruct key, bool xlock, bool slock) + void AssertIsLocked(FixedLengthTransactionalKeyStruct key, bool xlock, bool slock) => OverflowBucketLockTableTests.AssertLockCounts(store, ref key, xlock, slock); - void AssertIsLocked(ref FixedLengthLockableKeyStruct key, bool xlock, bool slock) + void AssertIsLocked(ref FixedLengthTransactionalKeyStruct key, bool xlock, bool slock) => OverflowBucketLockTableTests.AssertLockCounts(store, ref key, xlock, slock); void PrepareRecordLocation(FlushMode recordLocation) => PrepareRecordLocation(store, recordLocation); - static void PrepareRecordLocation(TsavoriteKV store, FlushMode recordLocation) + static void PrepareRecordLocation(TsavoriteKV store, FlushMode recordLocation) { if (recordLocation == FlushMode.ReadOnly) store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); @@ -249,15 +262,15 @@ static void PrepareRecordLocation(TsavoriteKV luContext) + static void ClearCountsOnError(ClientSession luContext) { // If we already have an exception, clear these counts so "Run" will not report them spuriously. luContext.sharedLockCount = 0; luContext.exclusiveLockCount = 0; } - static void ClearCountsOnError(ClientSession luContext) - where TFunctions : ISessionFunctions + static void ClearCountsOnError(ClientSession luContext) + where TFunctions : ISessionFunctions { // If we already have an exception, clear these counts so "Run" will not report them spuriously. luContext.sharedLockCount = 0; @@ -287,14 +300,14 @@ void AssertNoLocks(ref BucketLockTracker blt) AssertTotalLockCounts(0, 0); } - internal void AssertBucketLockCount(ref FixedLengthLockableKeyStruct key, long expectedX, long expectedS) => OverflowBucketLockTableTests.AssertBucketLockCount(store, ref key, expectedX, expectedS); + internal void AssertBucketLockCount(ref FixedLengthTransactionalKeyStruct key, long expectedX, long expectedS) => OverflowBucketLockTableTests.AssertBucketLockCount(store, ref key, expectedX, expectedS); internal enum LockOperationType { Lock, Unlock } - internal static IEnumerable EnumActionKeyIndices(FixedLengthLockableKeyStruct[] keys, LockOperationType lockOpType) + internal static IEnumerable EnumActionKeyIndices(FixedLengthTransactionalKeyStruct[] keys, LockOperationType lockOpType) { // "Action" means the keys that will actually be locked or unlocked. - // See comments in LockableContext.DoInternalLockOp. Apps shouldn't need to do this; key sorting and enumeration + // See comments in TransactionalContext.DoInternalLockOp. Apps shouldn't need to do this; key sorting and enumeration // should be a black-box to them, so this code is just for test. if (lockOpType == LockOperationType.Lock) { @@ -322,23 +335,24 @@ public void ManualLockCollidingHashCodes([Values] UseSingleBucketComparer /* jus uint bucketIndex = 42; long genHashCode(uint uniquifier) => ((long)uniquifier << 30) | bucketIndex; - var lContext = session.LockableContext; - lContext.BeginLockable(); + var lContext = session.TransactionalContext; + lContext.BeginTransaction(); + long key1 = 101L, key2 = 102L, key3 = 103L; var keys = new[] { - new FixedLengthLockableKeyStruct(101L, genHashCode(1), LockType.Exclusive, lContext), - new FixedLengthLockableKeyStruct(102L, genHashCode(2), LockType.Exclusive, lContext), - new FixedLengthLockableKeyStruct(103L, genHashCode(3), LockType.Exclusive, lContext), + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key1), genHashCode(1), LockType.Exclusive, lContext), + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key2), genHashCode(2), LockType.Exclusive, lContext), + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key3), genHashCode(3), LockType.Exclusive, lContext), }; for (var ii = 0; ii < keys.Length; ++ii) ClassicAssert.AreEqual(bucketIndex, store.LockTable.GetBucketIndex(keys[ii].KeyHash), $"BucketIndex mismatch on key {ii}"); - lContext.Lock>(keys); - lContext.Unlock>(keys); + lContext.Lock(keys); + lContext.Unlock(keys); - lContext.EndLockable(); + lContext.EndTransaction(); } [Test] @@ -346,50 +360,52 @@ public void ManualLockCollidingHashCodes([Values] UseSingleBucketComparer /* jus [Category("Smoke")] public async Task TestShiftHeadAddressLUC([Values] CompletionSyncMode syncMode) { - long input = default; + long input = 0; const int RandSeed = 10; const int RandRange = NumRecords; const int NumRecs = 200; - Random r = new(RandSeed); + Random rng = new(RandSeed); var sw = Stopwatch.StartNew(); // Copied from UnsafeContextTests. - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new FixedLengthLockableKeyStruct[1]; + var keyVec = new FixedLengthTransactionalKeyStruct[1]; try { for (int c = 0; c < NumRecs; c++) { - keyVec[0] = new(r.Next(RandRange), LockType.Exclusive, luContext); - luContext.Lock>(keyVec); + long rand = rng.Next(RandRange); + keyVec[0] = new(SpanByte.FromPinnedVariable(ref rand), LockType.Exclusive, luContext); + luContext.Lock(keyVec); AssertBucketLockCount(ref keyVec[0], 1, 0); - var value = keyVec[0].Key + NumRecords; - _ = luContext.Upsert(ref keyVec[0].Key, ref value, Empty.Default); - luContext.Unlock>(keyVec); + var value = keyVec[0].Key.KeyBytes.AsRef() + NumRecords; + _ = luContext.Upsert(keyVec[0].Key, SpanByte.FromPinnedVariable(ref value), Empty.Default); + luContext.Unlock(keyVec); AssertBucketLockCount(ref keyVec[0], 0, 0); } AssertTotalLockCounts(0, 0); - r = new Random(RandSeed); + rng = new Random(RandSeed); sw.Restart(); for (int c = 0; c < NumRecs; c++) { - keyVec[0] = new(r.Next(RandRange), LockType.Shared, luContext); - var value = keyVec[0].Key + NumRecords; + long rand = rng.Next(RandRange); + keyVec[0] = new(SpanByte.FromPinnedVariable(ref rand), LockType.Shared, luContext); + var value = keyVec[0].Key.KeyBytes.AsRef() + NumRecords; long output = 0; - luContext.Lock>(keyVec); + luContext.Lock(keyVec); AssertBucketLockCount(ref keyVec[0], 0, 1); - Status status = luContext.Read(ref keyVec[0].Key, ref input, ref output, Empty.Default); - luContext.Unlock>(keyVec); + Status status = luContext.Read(keyVec[0].Key, ref input, ref output, Empty.Default); + luContext.Unlock(keyVec); AssertBucketLockCount(ref keyVec[0], 0, 0); ClassicAssert.IsFalse(status.IsPending); } @@ -410,15 +426,24 @@ public async Task TestShiftHeadAddressLUC([Values] CompletionSyncMode syncMode) // Shift head and retry - should not find in main memory now store.Log.FlushAndEvict(true); - r = new Random(RandSeed); + rng = new Random(RandSeed); sw.Restart(); // Since we do random selection with replacement, we may not lock all keys--so need to track which we do // Similarly, we need to track bucket counts. BucketLockTracker blt = new(); - var lockKeys = Enumerable.Range(0, NumRecs).Select(ii => new FixedLengthLockableKeyStruct(r.Next(RandRange), LockType.Shared, luContext)).ToArray(); - luContext.SortKeyHashes>(lockKeys); - luContext.Lock>(lockKeys); + + // Must have a pinned array to keep the key values present. + var lockLongs = GC.AllocateArray(NumRecs, pinned: true); + var lockKeys = Enumerable.Range(0, NumRecs) + .Select(ii => + { + lockLongs[ii] = rng.Next(RandRange); + return new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref lockLongs[ii]), LockType.Shared, luContext); + }).ToArray(); + + luContext.SortKeyHashes(lockKeys); + luContext.Lock(lockKeys); var expectedS = 0; foreach (var idx in EnumActionKeyIndices(lockKeys, LockOperationType.Lock)) @@ -426,14 +451,14 @@ public async Task TestShiftHeadAddressLUC([Values] CompletionSyncMode syncMode) ++expectedS; long output = 0; blt.IncrementS(ref lockKeys[idx]); - Status foundStatus = luContext.Read(ref lockKeys[idx].Key, ref input, ref output, Empty.Default); + Status foundStatus = luContext.Read(lockKeys[idx].Key, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(foundStatus.IsPending); } // We did not lock all keys, only the "Action" ones - one lock per bucket, all shared in this test AssertTotalLockCounts(0, expectedS); - CompletedOutputIterator outputs; + CompletedOutputIterator outputs; if (syncMode == CompletionSyncMode.Sync) { _ = luContext.CompletePendingWithOutputs(out outputs, wait: true); @@ -447,7 +472,7 @@ public async Task TestShiftHeadAddressLUC([Values] CompletionSyncMode syncMode) foreach (var idx in EnumActionKeyIndices(lockKeys, LockOperationType.Unlock)) { - luContext.Unlock>(lockKeys.AsSpan().Slice(idx, 1)); + luContext.Unlock(lockKeys.AsSpan().Slice(idx, 1)); blt.DecrementS(ref lockKeys[idx]); } @@ -458,20 +483,20 @@ public async Task TestShiftHeadAddressLUC([Values] CompletionSyncMode syncMode) while (outputs.Next()) { count++; - ClassicAssert.AreEqual(outputs.Current.Key + NumRecords, outputs.Current.Output); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef() + NumRecords, outputs.Current.Output); } outputs.Dispose(); ClassicAssert.AreEqual(expectedS, count); } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget, [Values] FlushMode flushMode, [Values(Phase.REST, Phase.PREPARE)] Phase phase, @@ -482,34 +507,33 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget // SetUp also reads this to determine whether to supply ReadCache settings. If ReadCache is specified it wins over CopyToTail. var useRMW = updateOp == UpdateOp.RMW; - const int readKey24 = 24, readKey51 = 51; + long readKey24 = 24, readKey51 = 51, resultValue = -1; long resultKey = resultLockTarget == ResultLockTarget.LockTable ? NumRecords + 1 : readKey24 + readKey51; - long resultValue; long expectedResult = (readKey24 + readKey51) * ValueMult; Status status; BucketLockTracker blt = new(); - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); var keys = new[] { - new FixedLengthLockableKeyStruct(readKey24, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(readKey51, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(resultKey, LockType.Exclusive, luContext), // Destination, exclusive + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref readKey24), LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref readKey51), LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref resultKey), LockType.Exclusive, luContext), // Destination, exclusive }; - luContext.SortKeyHashes>(keys); + luContext.SortKeyHashes(keys); try { - luContext.Lock>(keys); + luContext.Lock(keys); // Verify locks. Note that while we do not increment lock counts for multiple keys (each bucket gets a single lock per thread, // shared or exclusive), each key mapping to that bucket will report 'locked'. foreach (var key in keys) { - if (key.Key == resultKey) + if (key.Key.KeyBytes.AsRef() == resultKey) AssertIsLocked(key, xlock: true, slock: false); else AssertIsLocked(key, xlock: false, slock: true); @@ -522,15 +546,16 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget // Re-get source values, to verify (e.g. they may be in readcache now). // We just locked this above, but for FlushMode.OnDisk it will be in the LockTable and will still be PENDING. - status = luContext.Read(readKey24, out var readValue24); + long output = -1; + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref readKey24)), ref output); if (flushMode == FlushMode.OnDisk) { if (status.IsPending) { _ = luContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); ClassicAssert.True(completedOutputs.Next()); - readValue24 = completedOutputs.Current.Output; - ClassicAssert.AreEqual(24 * ValueMult, readValue24); + output = completedOutputs.Current.Output; + ClassicAssert.AreEqual(24 * ValueMult, output); ClassicAssert.False(completedOutputs.Next()); completedOutputs.Dispose(); } @@ -540,15 +565,15 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget ClassicAssert.IsFalse(status.IsPending, status.ToString()); } - status = luContext.Read(readKey51, out var readValue51); + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref readKey51)), ref output); if (flushMode == FlushMode.OnDisk) { if (status.IsPending) { _ = luContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); ClassicAssert.True(completedOutputs.Next()); - readValue51 = completedOutputs.Current.Output; - ClassicAssert.AreEqual(51 * ValueMult, readValue51); + output = completedOutputs.Current.Output; + ClassicAssert.AreEqual(51 * ValueMult, output); ClassicAssert.False(completedOutputs.Next()); completedOutputs.Dispose(); } @@ -562,8 +587,8 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); long dummyInOut = 0; status = useRMW - ? luContext.RMW(ref resultKey, ref expectedResult, ref dummyInOut, out RecordMetadata recordMetadata) - : luContext.Upsert(ref resultKey, ref dummyInOut, ref expectedResult, ref dummyInOut, out recordMetadata); + ? luContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref expectedResult, ref dummyInOut) + : luContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref dummyInOut, SpanByte.FromPinnedVariable(ref expectedResult), ref dummyInOut); if (flushMode == FlushMode.OnDisk) { if (status.IsPending) @@ -582,11 +607,11 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget } // Reread the destination to verify - status = luContext.Read(resultKey, out resultValue); + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref resultValue); ClassicAssert.IsFalse(status.IsPending, status.ToString()); ClassicAssert.AreEqual(expectedResult, resultValue); - luContext.Unlock>(keys); + luContext.Unlock(keys); foreach (var idx in EnumActionKeyIndices(keys, LockOperationType.Lock)) blt.Decrement(ref keys[idx]); @@ -599,19 +624,19 @@ public void InMemorySimpleLockTxnTest([Values] ResultLockTarget resultLockTarget } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } // Verify reading the destination from the BasicContext. - status = bContext.Read(resultKey, out resultValue); + status = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref resultValue); ClassicAssert.IsFalse(status.IsPending, status.ToString()); ClassicAssert.AreEqual(expectedResult, resultValue); AssertTotalLockCounts(0, 0); } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void InMemoryLongLockTest([Values] ResultLockTarget resultLockTarget, [Values] FlushMode flushMode, [Values(Phase.REST, Phase.PREPARE)] Phase phase, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) @@ -620,38 +645,37 @@ public void InMemoryLongLockTest([Values] ResultLockTarget resultLockTarget, [Va PrepareRecordLocation(flushMode); bool initialDestWillBeLockTable = resultLockTarget == ResultLockTarget.LockTable || flushMode == FlushMode.OnDisk; - const int readKey24 = 24, readKey51 = 51, valueMult2 = 10; + long readKey24 = 24, readKey51 = 51, valueMult2 = 10, resultValue = -1; long resultKey = initialDestWillBeLockTable ? NumRecords + 1 : readKey24 + readKey51; - long resultValue; - int expectedResult = (readKey24 + readKey51) * ValueMult * valueMult2; + long expectedResult = (readKey24 + readKey51) * ValueMult * valueMult2; var useRMW = updateOp == UpdateOp.RMW; Status status; BucketLockTracker blt = new(); - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); var keys = new[] { - new FixedLengthLockableKeyStruct(readKey24, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(readKey51, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(resultKey, LockType.Exclusive, luContext), // Destination, exclusive + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref readKey24), LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref readKey51), LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref resultKey), LockType.Exclusive, luContext), // Destination, exclusive }; - luContext.SortKeyHashes>(keys); + luContext.SortKeyHashes(keys); var buckets = keys.Select(key => store.LockTable.GetBucketIndex(key.KeyHash)).ToArray(); try { - luContext.Lock>(keys); + luContext.Lock(keys); // Verify locks. Note that while we do not increment lock counts for multiple keys (each bucket gets a single lock per thread, // shared or exclusive), each key mapping to that bucket will report 'locked'. foreach (var key in keys) { - if (key.Key == resultKey) + if (key.Key.KeyBytes.AsRef() == resultKey) AssertIsLocked(key, xlock: true, slock: false); else AssertIsLocked(key, xlock: false, slock: true); @@ -662,36 +686,38 @@ public void InMemoryLongLockTest([Values] ResultLockTarget resultLockTarget, [Va blt.Increment(ref keys[idx]); AssertTotalLockCounts(ref blt); - status = luContext.Read(readKey24, out var readValue24); + long read24Output = 0, read51Output = 0; + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref readKey24)), ref read24Output); if (flushMode == FlushMode.OnDisk) { ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = luContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); - (status, readValue24) = GetSinglePendingResult(completedOutputs, out var recordMetadata); + (status, read24Output) = GetSinglePendingResult(completedOutputs, out var recordMetadata); ClassicAssert.IsTrue(status.Found, status.ToString()); } else ClassicAssert.IsFalse(status.IsPending, status.ToString()); - ClassicAssert.AreEqual(readKey24 * ValueMult, readValue24); + ClassicAssert.AreEqual(readKey24 * ValueMult, read24Output); // We just locked this above, but for FlushMode.OnDisk it will still be PENDING. - status = luContext.Read(readKey51, out var readValue51); + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref readKey51)), ref read51Output); if (flushMode == FlushMode.OnDisk) { ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = luContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); ClassicAssert.True(completedOutputs.Next()); - readValue51 = completedOutputs.Current.Output; + read51Output = completedOutputs.Current.Output; ClassicAssert.False(completedOutputs.Next()); completedOutputs.Dispose(); } else ClassicAssert.IsFalse(status.IsPending, status.ToString()); - ClassicAssert.AreEqual(readKey51 * ValueMult, readValue51); + ClassicAssert.AreEqual(readKey51 * ValueMult, read51Output); if (!initialDestWillBeLockTable) { - status = luContext.Read(resultKey, out var initialResultValue); + long initialResultValue = 0; + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref initialResultValue); if (flushMode == FlushMode.OnDisk) { ClassicAssert.IsTrue(status.IsPending, status.ToString()); @@ -706,16 +732,17 @@ public void InMemoryLongLockTest([Values] ResultLockTarget resultLockTarget, [Va // Set the phase to Phase.INTERMEDIATE to test the non-Phase.REST blocks session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); + resultValue = (read24Output + read51Output) * valueMult2; status = useRMW - ? luContext.RMW(resultKey, (readValue24 + readValue51) * valueMult2) - : luContext.Upsert(resultKey, (readValue24 + readValue51) * valueMult2); + ? luContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref resultValue) // value is 'input' for RMW + : luContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), SpanByte.FromPinnedVariable(ref resultValue)); ClassicAssert.IsFalse(status.IsPending, status.ToString()); - status = luContext.Read(resultKey, out resultValue); + status = luContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref resultValue); ClassicAssert.IsFalse(status.IsPending, status.ToString()); ClassicAssert.AreEqual(expectedResult, resultValue); - luContext.Unlock>(keys); + luContext.Unlock(keys); foreach (var idx in EnumActionKeyIndices(keys, LockOperationType.Lock)) blt.Decrement(ref keys[idx]); @@ -728,19 +755,20 @@ public void InMemoryLongLockTest([Values] ResultLockTarget resultLockTarget, [Va } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } // Verify from the full Basic Context - status = bContext.Read(resultKey, out resultValue); + var value = 0L; + status = bContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKey)), ref value); ClassicAssert.IsFalse(status.IsPending, status.ToString()); ClassicAssert.AreEqual(expectedResult, resultValue); AssertTotalLockCounts(0, 0); } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] #pragma warning disable IDE0060 // Remove unused parameter: readCopyDestination is used by Setup public void InMemoryDeleteTest([Values] ResultLockTarget resultLockTarget, [Values] ReadCopyDestination readCopyDestination, @@ -754,19 +782,20 @@ public void InMemoryDeleteTest([Values] ResultLockTarget resultLockTarget, [Valu BucketLockTracker blt = new(); // SetUp also reads this to determine whether to supply ReadCache settings. If ReadCache is specified it wins over CopyToTail. - long resultKey = resultLockTarget == ResultLockTarget.LockTable ? NumRecords + 1 : 75; + long resultKeyVal = resultLockTarget == ResultLockTarget.LockTable ? NumRecords + 1 : 75, output = 0, resultValue = -1; + var resultKey = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref resultKeyVal)); Status status; - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new[] { new FixedLengthLockableKeyStruct(resultKey, LockType.Exclusive, luContext) }; + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(resultKey.KeyBytes, LockType.Exclusive, luContext) }; try { // Lock destination value. - luContext.Lock>(keyVec); + luContext.Lock(keyVec); AssertIsLocked(ref keyVec[0], xlock: true, slock: false); blt.Increment(ref keyVec[0]); @@ -774,14 +803,14 @@ public void InMemoryDeleteTest([Values] ResultLockTarget resultLockTarget, [Valu // Set the phase to Phase.INTERMEDIATE to test the non-Phase.REST blocks session.ctx.SessionState = SystemState.Make(phase, session.ctx.version); - status = luContext.Delete(ref resultKey); + status = luContext.Delete(resultKey); ClassicAssert.IsFalse(status.IsPending, status.ToString()); // Reread the destination to verify - status = luContext.Read(resultKey, out var _); + status = luContext.Read(resultKey, ref output); ClassicAssert.IsFalse(status.Found, status.ToString()); - luContext.Unlock>(keyVec); + luContext.Unlock(keyVec); blt.Decrement(ref keyVec[0]); AssertNoLocks(ref blt); @@ -793,18 +822,18 @@ public void InMemoryDeleteTest([Values] ResultLockTarget resultLockTarget, [Valu } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } // Verify reading the destination from the full Basic Context - status = bContext.Read(resultKey, out var _); + status = bContext.Read(resultKey, ref resultValue); ClassicAssert.IsFalse(status.Found, status.ToString()); AssertTotalLockCounts(0, 0); } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void StressManualLocks([Values(1, 8)] int numLockThreads, [Values(0, 1, 8)] int numOpThreads) { @@ -816,7 +845,7 @@ public void StressManualLocks([Values(1, 8)] int numLockThreads, [Values(0, 1, 8 const int numIncrement = 5; const int numIterations = 1000; - IEnumerable enumKeys(Random rng) + IEnumerable enumKeys(Random rng) { for (var key = baseKey + rng.Next(numIncrement); key < baseKey + numKeys; key += rng.Next(1, numIncrement)) yield return key; @@ -828,29 +857,30 @@ void runManualLockThread(int tid) Random rng = new(tid + 101); - using var localSession = store.NewSession(new LockableUnsafeFunctions()); - var luContext = localSession.LockableUnsafeContext; + using var localSession = store.NewSession(new TransactionalUnsafeFunctions()); + var luContext = localSession.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - IEnumerable> enumKeysToLock() + IEnumerable enumKeysToLock() { foreach (var key in enumKeys(rng)) { var lockType = rng.Next(100) < 60 ? LockType.Shared : LockType.Exclusive; - yield return new(key, lockType, luContext); + var keyNum = key; + yield return new(SpanByte.FromPinnedVariable(ref keyNum), lockType, luContext); } } for (var iteration = 0; iteration < numIterations; ++iteration) { var keys = enumKeysToLock().ToArray(); - FixedLengthLockableKeyStruct.Sort(keys, luContext); - luContext.Lock>(keys); - luContext.Unlock>(keys); + FixedLengthTransactionalKeyStruct.Sort(keys, luContext); + luContext.Lock(keys); + luContext.Unlock(keys); } - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } @@ -858,18 +888,21 @@ void runLTransientLockOpThread(int tid) { Random rng = new(tid + 101); - using var localSession = store.NewSession(new LockableUnsafeFunctions()); + using var localSession = store.NewSession(new TransactionalUnsafeFunctions()); var basicContext = localSession.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) { - foreach (var key in enumKeys(rng)) + foreach (var key0 in enumKeys(rng)) + { + long key = key0, value = key * ValueMult; _ = rng.Next(100) switch { - int rand when rand < 33 => basicContext.Read(key).status, - int rand when rand < 66 => basicContext.Upsert(key, key * ValueMult), - _ => basicContext.RMW(key, key * ValueMult) + int rand when rand < 33 => basicContext.Read(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key))).status, + int rand when rand < 66 => basicContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref value)), + _ => basicContext.RMW(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), ref value) }; + } } } @@ -889,13 +922,13 @@ void runLTransientLockOpThread(int tid) AssertTotalLockCounts(0, 0); } - FixedLengthLockableKeyStruct AddLockTableEntry(LockableUnsafeContext luContext, long key) - where TFunctions : ISessionFunctions + FixedLengthTransactionalKeyStruct AddLockTableEntry(TransactionalUnsafeContext luContext, TestSpanByteKey key) + where TFunctions : ISessionFunctions { - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Exclusive, luContext) }; - luContext.Lock>(keyVec); + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(key.KeyBytes, LockType.Exclusive, luContext) }; + luContext.Lock(keyVec); - HashEntryInfo hei = new(comparer.GetHashCode64(ref key)); + HashEntryInfo hei = new(comparer.GetHashCode64(key)); PopulateHei(ref hei); var lockState = store.LockTable.GetLockState(ref hei); @@ -905,41 +938,42 @@ FixedLengthLockableKeyStruct AddLockTableEntry(LockableUnsafeC return keyVec[0]; } - void VerifyAndUnlockSplicedInKey(LockableUnsafeContext luContext, long expectedKey) - where TFunctions : ISessionFunctions + void VerifyAndUnlockSplicedInKey(TransactionalUnsafeContext luContext, TestSpanByteKey expectedKey) + where TFunctions : ISessionFunctions { // Scan to the end of the readcache chain and verify we inserted the value. - var (_, pa) = ChainTests.SkipReadCacheChain(store, expectedKey); - var storedKey = store.hlog.GetKey(pa); - ClassicAssert.AreEqual(expectedKey, storedKey); + var (_, pa) = ReadCacheChainTestUtils.SkipReadCacheChain(store, expectedKey); + var storedKey = LogRecord.GetInlineKey(pa); + ClassicAssert.AreEqual(expectedKey.AsRef(), storedKey.AsRef()); - var keyVec = new[] { new FixedLengthLockableKeyStruct(expectedKey, LockType.Exclusive, luContext) }; - luContext.Unlock>(keyVec); + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(expectedKey.KeyBytes, LockType.Exclusive, luContext) }; + luContext.Unlock(keyVec); } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void VerifyLocksAfterReadAndCTTTest() { Populate(); store.Log.FlushAndEvict(wait: true); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; - long input = 0, output = 0, key = 24; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; + long input = 0, output = 0, keyVal = 24; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); ReadOptions readOptions = new() { CopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog) }; BucketLockTracker blt = new(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); try { var keyStruct = AddLockTableEntry(luContext, key); blt.Increment(ref keyStruct); AssertTotalLockCounts(ref blt); - var status = luContext.Read(ref key, ref input, ref output, ref readOptions, out _); + var status = luContext.Read(key, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = luContext.CompletePending(wait: true); @@ -954,36 +988,36 @@ public void VerifyLocksAfterReadAndCTTTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void VerifyCountsAfterFlushAndEvict() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; BucketLockTracker blt = new(); long key = 24; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); try { - var keyVec = new[] { new FixedLengthLockableKeyStruct(key, LockType.Exclusive, luContext) }; - luContext.Lock>(keyVec); + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref key), LockType.Exclusive, luContext) }; + luContext.Lock(keyVec); blt.Increment(ref keyVec[0]); AssertTotalLockCounts(ref blt); store.Log.FlushAndEvict(wait: true); AssertTotalLockCounts(1, 0); - luContext.Unlock>(keyVec); + luContext.Unlock(keyVec); blt.Decrement(ref keyVec[0]); blt.AssertNoLocks(); @@ -996,7 +1030,7 @@ public void VerifyCountsAfterFlushAndEvict() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } @@ -1012,27 +1046,30 @@ void PopulateAndEvict(bool immutable = false) } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] - public void VerifyCountAfterUpsertToTailTest([Values] ChainTests.RecordRegion recordRegion) + public void VerifyCountAfterUpsertToTailTest([Values] RecordRegion recordRegion) { - PopulateAndEvict(recordRegion == ChainTests.RecordRegion.Immutable); + PopulateAndEvict(recordRegion == RecordRegion.Immutable); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; BucketLockTracker blt = new(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - FixedLengthLockableKeyStruct keyStruct = default; + FixedLengthTransactionalKeyStruct keyStruct = default; try { - if (recordRegion is ChainTests.RecordRegion.Immutable or ChainTests.RecordRegion.OnDisk) - keyStruct = AddLockTableEntry(luContext, UseExistingKey); + long keyNum = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseExistingKey)); else - keyStruct = AddLockTableEntry(luContext, UseNewKey); + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseNewKey)); + blt.Increment(ref keyStruct); - var status = luContext.Upsert(keyStruct.Key, keyStruct.Key * ValueMult); + var status = luContext.Upsert(key, key.KeyBytes); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); VerifyAndUnlockSplicedInKey(luContext, keyStruct.Key); @@ -1046,38 +1083,42 @@ public void VerifyCountAfterUpsertToTailTest([Values] ChainTests.RecordRegion re } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] - public void VerifyCountAfterRMWToTailTest([Values] ChainTests.RecordRegion recordRegion) + public void VerifyCountAfterRMWToTailTest([Values] RecordRegion recordRegion) { - PopulateAndEvict(recordRegion == ChainTests.RecordRegion.Immutable); + PopulateAndEvict(recordRegion == RecordRegion.Immutable); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; BucketLockTracker blt = new(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - FixedLengthLockableKeyStruct keyStruct = default; + FixedLengthTransactionalKeyStruct keyStruct = default; try { - if (recordRegion is ChainTests.RecordRegion.Immutable or ChainTests.RecordRegion.OnDisk) + long keyVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) { - keyStruct = AddLockTableEntry(luContext, UseExistingKey); - var status = luContext.RMW(keyStruct.Key, keyStruct.Key * ValueMult); - ClassicAssert.IsTrue(recordRegion == ChainTests.RecordRegion.OnDisk ? status.IsPending : status.Found); + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseExistingKey)); + var input = keyStruct.Key.KeyBytes.AsRef() * ValueMult; + var status = luContext.RMW(keyStruct.Key, ref input); + ClassicAssert.IsTrue(recordRegion == RecordRegion.OnDisk ? status.IsPending : status.Found); _ = luContext.CompletePending(wait: true); } else { - keyStruct = AddLockTableEntry(luContext, UseNewKey); - var status = luContext.RMW(keyStruct.Key, keyStruct.Key * ValueMult); + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseNewKey)); + var input = keyStruct.Key.KeyBytes.AsRef() * ValueMult; + var status = luContext.RMW(keyStruct.Key, ref input); ClassicAssert.IsFalse(status.Found, status.ToString()); } blt.Increment(ref keyStruct); @@ -1093,30 +1134,32 @@ public void VerifyCountAfterRMWToTailTest([Values] ChainTests.RecordRegion recor } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] - public void VerifyCountAfterDeleteToTailTest([Values] ChainTests.RecordRegion recordRegion) + public void VerifyCountAfterDeleteToTailTest([Values] RecordRegion recordRegion) { - PopulateAndEvict(recordRegion == ChainTests.RecordRegion.Immutable); + PopulateAndEvict(recordRegion == RecordRegion.Immutable); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; BucketLockTracker blt = new(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - FixedLengthLockableKeyStruct keyStruct = default; + FixedLengthTransactionalKeyStruct keyStruct = default; try { - if (recordRegion is ChainTests.RecordRegion.Immutable or ChainTests.RecordRegion.OnDisk) + long keyVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) { - keyStruct = AddLockTableEntry(luContext, UseExistingKey); + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseExistingKey)); blt.Increment(ref keyStruct); var status = luContext.Delete(keyStruct.Key); @@ -1125,7 +1168,7 @@ public void VerifyCountAfterDeleteToTailTest([Values] ChainTests.RecordRegion re } else { - keyStruct = AddLockTableEntry(luContext, UseNewKey); + keyStruct = AddLockTableEntry(luContext, key.Set((long)UseNewKey)); blt.Increment(ref keyStruct); var status = luContext.Delete(keyStruct.Key); ClassicAssert.IsFalse(status.Found, status.ToString()); @@ -1142,32 +1185,35 @@ public void VerifyCountAfterDeleteToTailTest([Values] ChainTests.RecordRegion re } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void LockAndUnlockInLockTableOnlyTest() { // For this, just don't load anything, and it will happen in lock table. - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; BucketLockTracker blt = new(); - FixedLengthLockableKeyStruct createKey(long key) => new(key, (key & 1) == 0 ? LockType.Exclusive : LockType.Shared, luContext); + FixedLengthTransactionalKeyStruct createKey(ReadOnlySpan key) => new(key, (key.AsRef() & 1) == 0 ? LockType.Exclusive : LockType.Shared, luContext); + + // Need a pinned array for SpanByteFrom + var keys = GC.AllocateArray(NumRecords, pinned: true); var rng = new Random(101); - var keyVec = Enumerable.Range(0, NumRecords).Select(ii => createKey(rng.Next(NumRecords))).ToArray(); + var keyVec = Enumerable.Range(0, NumRecords).Select(ii => { keys[ii] = rng.Next(NumRecords); return createKey(SpanByte.FromPinnedVariable(ref keys[ii])); }).ToArray(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); try { - store.LockTable.SortKeyHashes>(keyVec); - luContext.Lock>(keyVec); + store.LockTable.SortKeyHashes(keyVec); + luContext.Lock(keyVec); foreach (var idx in EnumActionKeyIndices(keyVec, LockOperationType.Lock)) blt.Increment(ref keyVec[idx]); AssertTotalLockCounts(ref blt); @@ -1183,7 +1229,7 @@ public void LockAndUnlockInLockTableOnlyTest() if (key.LockType == LockType.Shared) ClassicAssert.IsTrue(lockState.IsLocked); // Could be either shared or exclusive; we only lock the bucket once per Lock() call - luContext.Unlock>(keyVec.AsSpan().Slice(idx, 1)); + luContext.Unlock(keyVec.AsSpan().Slice(idx, 1)); blt.Decrement(ref key); } @@ -1197,13 +1243,13 @@ public void LockAndUnlockInLockTableOnlyTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void VerifyCountAfterReadOnlyToUpdateRecordTest([Values] UpdateOp updateOp) { @@ -1212,21 +1258,26 @@ public void VerifyCountAfterReadOnlyToUpdateRecordTest([Values] UpdateOp updateO static long getValue(long key) => key + ValueMult; - var luContext = session.LockableUnsafeContext; + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new[] { new FixedLengthLockableKeyStruct(42, LockType.Exclusive, luContext) }; + long key42Val = 42L; + var key42 = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key42Val)); + var keyVec = new[] { new FixedLengthTransactionalKeyStruct(key42.KeyBytes, LockType.Exclusive, luContext) }; try { - luContext.Lock>(keyVec); + luContext.Lock(keyVec); + + long valueVal = getValue(key42Val); + var value = SpanByte.FromPinnedVariable(ref valueVal); var status = updateOp switch { - UpdateOp.Upsert => luContext.Upsert(keyVec[0].Key, getValue(keyVec[0].Key)), - UpdateOp.RMW => luContext.RMW(keyVec[0].Key, getValue(keyVec[0].Key)), - UpdateOp.Delete => luContext.Delete(keyVec[0].Key), + UpdateOp.Upsert => luContext.Upsert(key42, value), + UpdateOp.RMW => luContext.RMW(keyVec[0].Key, ref valueVal), + UpdateOp.Delete => luContext.Delete(key42), _ => new(StatusCode.Error) }; ClassicAssert.IsFalse(status.IsFaulted, $"Unexpected UpdateOp {updateOp}, status {status}"); @@ -1235,10 +1286,10 @@ public void VerifyCountAfterReadOnlyToUpdateRecordTest([Values] UpdateOp updateO else ClassicAssert.IsTrue(status.Record.Created, status.ToString()); - OverflowBucketLockTableTests.AssertLockCounts(store, keyVec[0].Key, true, 0); + OverflowBucketLockTableTests.AssertLockCounts(store, keyVec[0].Key.KeyBytes, true, 0); - luContext.Unlock>(keyVec); - OverflowBucketLockTableTests.AssertLockCounts(store, keyVec[0].Key, false, 0); + luContext.Unlock(keyVec); + OverflowBucketLockTableTests.AssertLockCounts(store, keyVec[0].Key.KeyBytes, false, 0); } catch (Exception) { @@ -1247,27 +1298,31 @@ public void VerifyCountAfterReadOnlyToUpdateRecordTest([Values] UpdateOp updateO } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] public void LockNewRecordThenUpdateAndUnlockTest([Values] UpdateOp updateOp) { const int numNewRecords = 100; - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; + + long getValue(long kk) => kk + ValueMult; - int getValue(int key) => key + ValueMult; + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + Span value = SpanByte.FromPinnedVariable(ref valueVal); // If we are testing Delete, then we need to have the records ON-DISK first; Delete is a no-op for unfound records. if (updateOp == UpdateOp.Delete) { - for (var key = NumRecords; key < NumRecords + numNewRecords; ++key) - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + for (long keyNum = NumRecords; keyNum < NumRecords + numNewRecords; ++keyNum) + ClassicAssert.IsFalse(bContext.Upsert(key.Set(keyNum), value.Set(keyVal * ValueMult)).IsPending); store.Log.FlushAndEvict(wait: true); } @@ -1276,26 +1331,26 @@ public void LockNewRecordThenUpdateAndUnlockTest([Values] UpdateOp updateOp) BucketLockTracker blt = new(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new FixedLengthLockableKeyStruct[1]; + var keyVec = new FixedLengthTransactionalKeyStruct[1]; try { // We don't sleep in this test comparer.maxSleepMs = 0; - for (var key = NumRecords; key < NumRecords + numNewRecords; ++key) + for (long keyNum = NumRecords; keyNum < NumRecords + numNewRecords; ++keyNum) { - keyVec[0] = new(key, LockType.Exclusive, luContext); - luContext.Lock>(keyVec); + keyVec[0] = new(key.Set(keyNum).KeyBytes, LockType.Exclusive, luContext); + luContext.Lock(keyVec); for (var iter = 0; iter < 2; ++iter) { - OverflowBucketLockTableTests.AssertLockCounts(store, key, true, 0); + OverflowBucketLockTableTests.AssertLockCounts(store, key.KeyBytes, true, 0); updater(key, iter); } - luContext.Unlock>(keyVec); - OverflowBucketLockTableTests.AssertLockCounts(store, key, false, 0); + luContext.Unlock(keyVec); + OverflowBucketLockTableTests.AssertLockCounts(store, key.KeyBytes, false, 0); } } catch (Exception) @@ -1305,26 +1360,27 @@ public void LockNewRecordThenUpdateAndUnlockTest([Values] UpdateOp updateOp) } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } - void updater(int key, int iter) + void updater(TestSpanByteKey key, int iter) { + var localValueNum = getValue(key.AsRef()); try { Status status; switch (updateOp) { case UpdateOp.Upsert: - status = luContext.Upsert(key, getValue(key)); + status = luContext.Upsert(key, SpanByte.FromPinnedVariable(ref localValueNum)); if (iter == 0) ClassicAssert.IsTrue(status.NotFound && status.Record.Created, status.ToString()); else ClassicAssert.IsTrue(status.Found && status.Record.InPlaceUpdated, status.ToString()); break; case UpdateOp.RMW: - status = luContext.RMW(key, getValue(key)); + status = luContext.RMW(key, ref localValueNum); if (iter == 0) ClassicAssert.IsTrue(status.NotFound && status.Record.Created, status.ToString()); else @@ -1340,6 +1396,7 @@ void updater(int key, int iter) Assert.Fail($"Unexpected updateOp {updateOp}"); return; } + ; ClassicAssert.IsFalse(status.IsFaulted, $"Unexpected UpdateOp {updateOp}, status {status}"); } catch (Exception) @@ -1351,28 +1408,26 @@ void updater(int key, int iter) } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] //[Repeat(100)] public void LockNewRecordThenUnlockThenUpdateTest([Values] UpdateOp updateOp) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); - const int numNewRecords = 50; - using var lockSession = store.NewSession>(new SimpleSimpleFunctions()); - var lockLuContext = lockSession.LockableUnsafeContext; + using var lockSession = store.NewSession(new SimpleLongSimpleFunctions()); + var lockLuContext = lockSession.TransactionalUnsafeContext; - using var updateSession = store.NewSession>(new SimpleSimpleFunctions()); + using var updateSession = store.NewSession(new SimpleLongSimpleFunctions()); var basicContext = updateSession.BasicContext; - int getValue(int key) => key + ValueMult; + long getValue(long kk) => kk + ValueMult; // If we are testing Delete, then we need to have the records ON-DISK first; Delete is a no-op for unfound records. + // The actual value here is not important as we don't test it later. if (updateOp == UpdateOp.Delete) { - for (var key = NumRecords; key < NumRecords + numNewRecords; ++key) - ClassicAssert.IsFalse(bContext.Upsert(key, key * ValueMult).IsPending); + for (long keyNum = NumRecords; keyNum < NumRecords + numNewRecords; ++keyNum) + ClassicAssert.IsFalse(bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), SpanByte.FromPinnedVariable(ref keyNum)).IsPending); store.Log.FlushAndEvict(wait: true); } @@ -1380,26 +1435,26 @@ public void LockNewRecordThenUnlockThenUpdateTest([Values] UpdateOp updateOp) Populate(); lockLuContext.BeginUnsafe(); - lockLuContext.BeginLockable(); + lockLuContext.BeginTransaction(); // These are for debugging - int[] lastLockerKeys = new int[6], lastUpdaterKeys = new int[3]; + long[] lastLockerKeys = new long[6], lastUpdaterKeys = new long[3]; // Randomize the start and lock-hold wait times int maxSleepMs = 10; Random lockRng = new(101), updateRng = new(107); - var lockKeyVec = new FixedLengthLockableKeyStruct[1]; + var lockKeyVec = new FixedLengthTransactionalKeyStruct[1]; try { - for (var key = NumRecords; key < NumRecords + numNewRecords; ++key) + for (long keyNum = NumRecords; keyNum < NumRecords + numNewRecords; ++keyNum) { for (var iter = 0; iter < 2; ++iter) { // Use Task instead of Thread because this propagates exceptions (such as Assert.* failures) back to this thread. // BasicContext's transient lock will wait for the lock/unlock combo to complete, or the lock/unlock will wait for basicContext to finish if it wins. - Task.WaitAll(Task.Run(() => locker(key)), Task.Run(() => updater(key, iter))); + Task.WaitAll(Task.Run(() => locker(keyNum)), Task.Run(() => updater(keyNum, iter))); } AssertBucketLockCount(ref lockKeyVec[0], 0, 0); @@ -1412,29 +1467,29 @@ public void LockNewRecordThenUnlockThenUpdateTest([Values] UpdateOp updateOp) } finally { - lockLuContext.EndLockable(); + lockLuContext.EndTransaction(); lockLuContext.EndUnsafe(); } - void locker(int key) + void locker(long keyNum) { - lockKeyVec[0] = new(key, LockType.Exclusive, lockLuContext); + lockKeyVec[0] = new(SpanByte.FromPinnedVariable(ref keyNum), LockType.Exclusive, lockLuContext); try { - // Begin/EndLockable are called outside this function; we could not EndLockable in here as the lock lifetime is beyond that. - // (BeginLockable's scope is the session; BeginUnsafe's scope is the thread. The session is still "mono-threaded" here because + // Begin/EndTransaction are called outside this function; we could not EndTransaction in here as the lock lifetime is beyond that. + // (BeginTransaction's scope is the session; BeginUnsafe's scope is the thread. The session is still "mono-threaded" here because // only one thread at a time is making calls on it.) - lastLockerKeys[0] = key; + lastLockerKeys[0] = keyNum; lockLuContext.BeginUnsafe(); - lastLockerKeys[1] = key; + lastLockerKeys[1] = keyNum; Thread.Sleep(lockRng.Next(maxSleepMs)); - lastLockerKeys[2] = key; - lockLuContext.Lock>(lockKeyVec); - lastLockerKeys[3] = key; + lastLockerKeys[2] = keyNum; + lockLuContext.Lock(lockKeyVec); + lastLockerKeys[3] = keyNum; Thread.Sleep(lockRng.Next(maxSleepMs)); - lastLockerKeys[4] = key; - lockLuContext.Unlock>(lockKeyVec); - lastLockerKeys[5] = key; + lastLockerKeys[4] = keyNum; + lockLuContext.Unlock(lockKeyVec); + lastLockerKeys[5] = keyNum; } catch (Exception) { @@ -1447,25 +1502,28 @@ void locker(int key) } } - void updater(int key, int iter) + void updater(long keyNum, int iter) { try { - lastUpdaterKeys[0] = key; + lastUpdaterKeys[0] = keyNum; Thread.Sleep(updateRng.Next(maxSleepMs)); - lastUpdaterKeys[1] = key; + lastUpdaterKeys[1] = keyNum; Status status; + var localValueNum = getValue(keyNum); + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref localValueNum); switch (updateOp) { case UpdateOp.Upsert: - status = basicContext.Upsert(key, getValue(key)); + status = basicContext.Upsert(key, value); if (iter == 0) ClassicAssert.IsTrue(status.NotFound && status.Record.Created, status.ToString()); else ClassicAssert.IsTrue(status.Found && status.Record.InPlaceUpdated, status.ToString()); break; case UpdateOp.RMW: - status = basicContext.RMW(key, getValue(key)); + status = basicContext.RMW(key, ref localValueNum); if (iter == 0) ClassicAssert.IsTrue(status.NotFound && status.Record.Created, status.ToString()); else @@ -1483,7 +1541,7 @@ void updater(int key, int iter) } ; ClassicAssert.IsFalse(status.IsFaulted, $"Unexpected UpdateOp {updateOp}, status {status}"); - lastUpdaterKeys[2] = key; + lastUpdaterKeys[2] = keyNum; } catch (Exception) { @@ -1494,36 +1552,37 @@ void updater(int key, int iter) } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void MultiSharedLockTest() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; - const int key = 42; + long keyNum = 42; + var key = SpanByte.FromPinnedVariable(ref keyNum); var maxLocks = 63; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new FixedLengthLockableKeyStruct[1]; + var keyVec = new FixedLengthTransactionalKeyStruct[1]; try { for (var ii = 0; ii < maxLocks; ++ii) { keyVec[0] = new(key, LockType.Shared, luContext); - luContext.Lock>(keyVec); + luContext.Lock(keyVec); OverflowBucketLockTableTests.AssertLockCounts(store, key, false, ii + 1); } for (var ii = 0; ii < maxLocks; ++ii) { keyVec[0] = new(key, LockType.Shared, luContext); - luContext.Unlock>(keyVec); + luContext.Unlock(keyVec); OverflowBucketLockTableTests.AssertLockCounts(store, key, false, maxLocks - ii - 1); } OverflowBucketLockTableTests.AssertLockCounts(store, key, false, 0); @@ -1535,36 +1594,41 @@ public void MultiSharedLockTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void TryLockTimeSpanLimitTest() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var keyVec = new FixedLengthLockableKeyStruct[] + var keys = GC.AllocateArray(3, pinned: true); + keys[0] = 42; + keys[1] = 43; + keys[2] = 44; + + var keyVec = new FixedLengthTransactionalKeyStruct[] { - new(42, LockType.Exclusive, luContext), - new(43, LockType.Exclusive, luContext), - new(44, LockType.Exclusive, luContext) + new(SpanByte.FromPinnedVariable(ref keys[0]), LockType.Exclusive, luContext), + new(SpanByte.FromPinnedVariable(ref keys[1]), LockType.Exclusive, luContext), + new(SpanByte.FromPinnedVariable(ref keys[2]), LockType.Exclusive, luContext) }; // First ensure things work with no blocking locks. - ClassicAssert.IsTrue(luContext.TryLock>(keyVec)); - luContext.Unlock>(keyVec); + ClassicAssert.IsTrue(luContext.TryLock(keyVec)); + luContext.Unlock(keyVec); - var blockingVec = new FixedLengthLockableKeyStruct[1]; + var blockingVec = new FixedLengthTransactionalKeyStruct[1]; try { @@ -1572,17 +1636,17 @@ public void TryLockTimeSpanLimitTest() { // This key blocks the lock. Test all positions in keyVec to ensure rollback of locks on failure. blockingVec[0] = keyVec[blockingIdx]; - luContext.Lock>(blockingVec); + luContext.Lock(blockingVec); // Now try the lock, and verify there are no locks left after (any taken must be rolled back on failure). - ClassicAssert.IsFalse(luContext.TryLock>(keyVec, TimeSpan.FromMilliseconds(20))); + ClassicAssert.IsFalse(luContext.TryLock(keyVec, TimeSpan.FromMilliseconds(20))); foreach (var k in keyVec) { - if (k.Key != blockingVec[0].Key) - OverflowBucketLockTableTests.AssertLockCounts(store, k.Key, false, 0); + if (k.Key.KeyBytes.AsRef() != blockingVec[0].Key.KeyBytes.AsRef()) + OverflowBucketLockTableTests.AssertLockCounts(store, k.Key.KeyBytes, false, 0); } - luContext.Unlock>(blockingVec); + luContext.Unlock(blockingVec); } } catch (Exception) @@ -1592,36 +1656,41 @@ public void TryLockTimeSpanLimitTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void TryLockCancellationTest() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); + + var keys = GC.AllocateArray(3, pinned: true); + keys[0] = 42; + keys[1] = 43; + keys[2] = 44; - var keyVec = new FixedLengthLockableKeyStruct[] + var keyVec = new FixedLengthTransactionalKeyStruct[] { - new(42, LockType.Exclusive, luContext), - new(43, LockType.Exclusive, luContext), - new(44, LockType.Exclusive, luContext) + new(SpanByte.FromPinnedVariable(ref keys[0]), LockType.Exclusive, luContext), + new(SpanByte.FromPinnedVariable(ref keys[1]), LockType.Exclusive, luContext), + new(SpanByte.FromPinnedVariable(ref keys[2]), LockType.Exclusive, luContext) }; // First ensure things work with no blocking locks. - ClassicAssert.IsTrue(luContext.TryLock>(keyVec)); - luContext.Unlock>(keyVec); + ClassicAssert.IsTrue(luContext.TryLock(keyVec)); + luContext.Unlock(keyVec); - var blockingVec = new FixedLengthLockableKeyStruct[1]; + var blockingVec = new FixedLengthTransactionalKeyStruct[1]; try { @@ -1629,19 +1698,19 @@ public void TryLockCancellationTest() { // This key blocks the lock. Test all positions in keyVec to ensure rollback of locks on failure. blockingVec[0] = keyVec[blockingIdx]; - luContext.Lock>(blockingVec); + luContext.Lock(blockingVec); using var cts = new CancellationTokenSource(20); // Now try the lock, and verify there are no locks left after (any taken must be rolled back on failure). - ClassicAssert.IsFalse(luContext.TryLock>(keyVec, cts.Token)); + ClassicAssert.IsFalse(luContext.TryLock(keyVec, cts.Token)); foreach (var k in keyVec) { - if (k.Key != blockingVec[0].Key) - OverflowBucketLockTableTests.AssertLockCounts(store, k.Key, false, 0); + if (k.Key.KeyBytes.AsRef() != blockingVec[0].Key.KeyBytes.AsRef()) + OverflowBucketLockTableTests.AssertLockCounts(store, k.Key.KeyBytes, false, 0); } - luContext.Unlock>(blockingVec); + luContext.Unlock(blockingVec); } } catch (Exception) @@ -1651,41 +1720,42 @@ public void TryLockCancellationTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void TryPromoteLockTimeSpanLimitTest() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var key = 42; + long keyNum = 42; + var key = SpanByte.FromPinnedVariable(ref keyNum); - var exclusiveVec = new FixedLengthLockableKeyStruct[] { new(key, LockType.Exclusive, luContext) }; - var sharedVec = new FixedLengthLockableKeyStruct[] { new(key, LockType.Shared, luContext) }; + var exclusiveVec = new FixedLengthTransactionalKeyStruct[] { new(key, LockType.Exclusive, luContext) }; + var sharedVec = new FixedLengthTransactionalKeyStruct[] { new(key, LockType.Shared, luContext) }; try { // Lock twice so it is blocked by the second reader - ClassicAssert.IsTrue(luContext.TryLock>(sharedVec)); - ClassicAssert.IsTrue(luContext.TryLock>(sharedVec)); + ClassicAssert.IsTrue(luContext.TryLock(sharedVec)); + ClassicAssert.IsTrue(luContext.TryLock(sharedVec)); ClassicAssert.IsFalse(luContext.TryPromoteLock(exclusiveVec[0], TimeSpan.FromMilliseconds(20))); // Unlock one of the readers and verify successful promotion - luContext.Unlock>(sharedVec); + luContext.Unlock(sharedVec); ClassicAssert.IsTrue(luContext.TryPromoteLock(exclusiveVec[0])); - luContext.Unlock>(exclusiveVec); + luContext.Unlock(exclusiveVec); } catch (Exception) { @@ -1694,42 +1764,43 @@ public void TryPromoteLockTimeSpanLimitTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } [Test] - [Category(LockableUnsafeContextTestCategory)] + [Category(TransactionalUnsafeContextTestCategory)] [Category(SmokeTestCategory)] public void TryPromoteLockCancellationTest() { Populate(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); - var key = 42; + long keyNum = 42; + var key = SpanByte.FromPinnedVariable(ref keyNum); - var exclusiveVec = new FixedLengthLockableKeyStruct[] { new(key, LockType.Exclusive, luContext) }; - var sharedVec = new FixedLengthLockableKeyStruct[] { new(key, LockType.Shared, luContext) }; + var exclusiveVec = new FixedLengthTransactionalKeyStruct[] { new(key, LockType.Exclusive, luContext) }; + var sharedVec = new FixedLengthTransactionalKeyStruct[] { new(key, LockType.Shared, luContext) }; try { // Lock twice so it is blocked by the second reader - ClassicAssert.IsTrue(luContext.TryLock>(sharedVec)); - ClassicAssert.IsTrue(luContext.TryLock>(sharedVec)); + ClassicAssert.IsTrue(luContext.TryLock(sharedVec)); + ClassicAssert.IsTrue(luContext.TryLock(sharedVec)); using var cts = new CancellationTokenSource(20); ClassicAssert.IsFalse(luContext.TryPromoteLock(exclusiveVec[0], cts.Token)); // Unlock one of the readers and verify successful promotion - luContext.Unlock>(sharedVec); + luContext.Unlock(sharedVec); ClassicAssert.IsTrue(luContext.TryPromoteLock(exclusiveVec[0])); - luContext.Unlock>(exclusiveVec); + luContext.Unlock(exclusiveVec); } catch (Exception) { @@ -1738,7 +1809,7 @@ public void TryPromoteLockCancellationTest() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } } diff --git a/libs/storage/Tsavorite/cs/test/test.session.context/Tsavorite.test.session.context.csproj b/libs/storage/Tsavorite/cs/test/test.session.context/Tsavorite.test.session.context.csproj new file mode 100644 index 00000000000..493794ca137 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.session.context/Tsavorite.test.session.context.csproj @@ -0,0 +1,32 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + false + + + diff --git a/libs/storage/Tsavorite/cs/test/UnsafeContextTests.cs b/libs/storage/Tsavorite/cs/test/test.session.context/UnsafeContextTests.cs similarity index 76% rename from libs/storage/Tsavorite/cs/test/UnsafeContextTests.cs rename to libs/storage/Tsavorite/cs/test/test.session.context/UnsafeContextTests.cs index 34ff438d3d5..d6846cca0ac 100644 --- a/libs/storage/Tsavorite/cs/test/UnsafeContextTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session.context/UnsafeContextTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,18 +14,16 @@ namespace Tsavorite.test.UnsafeContext { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; //** These tests ensure the basics are fully covered - taken from BasicTests - - [AllureNUnit] [TestFixture] - internal class BasicUnsafeContextTests : AllureTestBase + internal class BasicUnsafeContextTests : TestBase { - private TsavoriteKV store; - private ClientSession fullSession; - private UnsafeContext uContext; + private TsavoriteKV store; + private ClientSession fullSession; + private UnsafeContext uContext; private IDevice log; TestDeviceType deviceType; @@ -37,7 +34,7 @@ public void Setup() DeleteDirectory(MethodTestDir, wait: true); } - private void Setup(KVSettings kvSettings, TestDeviceType deviceType) + private void Setup(KVSettings kvSettings, TestDeviceType deviceType) { string filename = Path.Join(MethodTestDir, TestContext.CurrentContext.Test.Name + deviceType.ToString() + ".log"); log = CreateTestDevice(deviceType, filename); @@ -45,10 +42,10 @@ private void Setup(KVSettings kvSettings, TestDeviceType kvSettings.IndexSize = 1L << 13; store = new(kvSettings - , StoreFunctions.Create(KeyStruct.Comparer.Instance) + , StoreFunctions.Create(KeyStruct.Comparer.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); - fullSession = store.NewSession(new Functions()); + fullSession = store.NewSession(new Functions()); uContext = fullSession.UnsafeContext; } @@ -83,7 +80,7 @@ private void AssertCompleted(Status expected, Status actual) [Category("Smoke")] public void NativeInMemWriteRead([Values] TestDeviceType deviceType) { - Setup(new() { PageSize = 1L << 10, MemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); + Setup(new() { PageSize = 1L << 10, LogMemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); uContext.BeginUnsafe(); try @@ -94,8 +91,8 @@ public void NativeInMemWriteRead([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); - var status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); @@ -112,7 +109,7 @@ public void NativeInMemWriteRead([Values] TestDeviceType deviceType) [Category("Smoke")] public void NativeInMemWriteReadDelete([Values] TestDeviceType deviceType) { - Setup(new() { PageSize = 1L << 10, MemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); + Setup(new() { PageSize = 1L << 10, LogMemorySize = 1L << 12, SegmentSize = 1L << 22 }, deviceType); uContext.BeginUnsafe(); try @@ -123,20 +120,20 @@ public void NativeInMemWriteReadDelete([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); - var status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); - _ = uContext.Delete(ref key1, Empty.Default); + _ = uContext.Delete(key1, Empty.Default); - status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); var key2 = new KeyStruct { kfield1 = 14, kfield2 = 15 }; var value2 = new ValueStruct { vfield1 = 24, vfield2 = 25 }; - _ = uContext.Upsert(ref key2, ref value2, Empty.Default); - status = uContext.Read(ref key2, ref input, ref output, Empty.Default); + _ = uContext.Upsert(key2, SpanByte.FromPinnedVariable(ref value2), Empty.Default); + status = uContext.Read(key2, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(value2.vfield1, output.value.vfield1); @@ -159,8 +156,8 @@ public void NativeInMemWriteReadDelete2() const int count = 10; - // Setup(new () { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); - Setup(new() { MemorySize = 1L << 29 }, deviceType); + // Setup(new () { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 29 }, deviceType); uContext.BeginUnsafe(); try @@ -173,13 +170,13 @@ public void NativeInMemWriteReadDelete2() var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; - _ = uContext.Delete(ref key1, Empty.Default); + _ = uContext.Delete(key1, Empty.Default); } for (int i = 0; i < 10 * count; i++) @@ -187,16 +184,16 @@ public void NativeInMemWriteReadDelete2() var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; var value = new ValueStruct { vfield1 = i, vfield2 = 24 }; - var status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); - _ = uContext.Upsert(ref key1, ref value, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } for (int i = 0; i < 10 * count; i++) { var key1 = new KeyStruct { kfield1 = i, kfield2 = 14 }; - var status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); } } @@ -216,8 +213,8 @@ public unsafe void NativeInMemWriteRead2() int count = 200; - // Setup(128, new () { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); - Setup(new() { MemorySize = 1L << 29 }, deviceType); + // Setup(128, new () { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 29 }, deviceType); uContext.BeginUnsafe(); try @@ -230,7 +227,7 @@ public unsafe void NativeInMemWriteRead2() var i = r.Next(10000); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } r = new Random(10); @@ -242,7 +239,7 @@ public unsafe void NativeInMemWriteRead2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - if (uContext.Read(ref key1, ref input, ref output, Empty.Default).IsPending) + if (uContext.Read(key1, ref input, ref output, Empty.Default).IsPending) _ = uContext.CompletePending(true); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); @@ -258,7 +255,7 @@ public unsafe void NativeInMemWriteRead2() var i = r.Next(10000); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - ClassicAssert.IsFalse(uContext.Read(ref key1, ref input, ref output, Empty.Default).Found); + ClassicAssert.IsFalse(uContext.Read(key1, ref input, ref output, Empty.Default).Found); } } finally @@ -280,7 +277,7 @@ public async Task TestShiftHeadAddressUC([Values] TestDeviceType deviceType, [Va Random r = new(RandSeed); var sw = Stopwatch.StartNew(); - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try @@ -290,7 +287,7 @@ public async Task TestShiftHeadAddressUC([Values] TestDeviceType deviceType, [Va var i = r.Next(RandRange); var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } r = new Random(RandSeed); @@ -303,7 +300,7 @@ public async Task TestShiftHeadAddressUC([Values] TestDeviceType deviceType, [Va var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - Status status = uContext.Read(ref key1, ref input, ref output, Empty.Default); + Status status = uContext.Read(key1, ref input, ref output, Empty.Default); if (!status.IsPending) { ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); @@ -332,11 +329,11 @@ public async Task TestShiftHeadAddressUC([Values] TestDeviceType deviceType, [Va var i = r.Next(RandRange); OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; - Status foundStatus = uContext.Read(ref key1, ref input, ref output, Empty.Default); + Status foundStatus = uContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(foundStatus.IsPending); } - CompletedOutputIterator outputs; + CompletedOutputIterator outputs; if (syncMode == CompletionSyncMode.Sync) { _ = uContext.CompletePendingWithOutputs(out outputs, wait: true); @@ -352,8 +349,8 @@ public async Task TestShiftHeadAddressUC([Values] TestDeviceType deviceType, [Va while (outputs.Next()) { count++; - ClassicAssert.AreEqual(outputs.Current.Key.kfield1, outputs.Current.Output.value.vfield1); - ClassicAssert.AreEqual(outputs.Current.Key.kfield2, outputs.Current.Output.value.vfield2); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield1, outputs.Current.Output.value.vfield1); + ClassicAssert.AreEqual(outputs.Current.Key.KeyBytes.AsRef().kfield2, outputs.Current.Output.value.vfield2); } outputs.Dispose(); ClassicAssert.AreEqual(NumRecs, count); @@ -372,7 +369,7 @@ public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) InputStruct input = default; OutputStruct output = default; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try @@ -382,9 +379,7 @@ public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) for (int i = 0; i < nums.Length; ++i) { int randomIndex = rnd.Next(nums.Length); - int temp = nums[randomIndex]; - nums[randomIndex] = nums[i]; - nums[i] = temp; + (nums[i], nums[randomIndex]) = (nums[randomIndex], nums[i]); } for (int j = 0; j < nums.Length; ++j) @@ -392,14 +387,14 @@ public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = uContext.RMW(ref key1, ref input, Empty.Default); + _ = uContext.RMW(key1, ref input, Empty.Default); } for (int j = 0; j < nums.Length; ++j) { var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - if (uContext.RMW(ref key1, ref input, ref output, Empty.Default).IsPending) + if (uContext.RMW(key1, ref input, ref output, Empty.Default).IsPending) { _ = uContext.CompletePending(true); } @@ -420,7 +415,7 @@ public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; ValueStruct value = new() { vfield1 = i, vfield2 = i + 1 }; - status = uContext.Read(ref key, ref input, ref output, Empty.Default); + status = uContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(2 * value.vfield1, output.value.vfield1); @@ -428,7 +423,7 @@ public unsafe void NativeInMemRMWRefKeys([Values] TestDeviceType deviceType) } key = new KeyStruct { kfield1 = nums.Length, kfield2 = nums.Length + 1 }; - status = uContext.Read(ref key, ref input, ref output, Empty.Default); + status = uContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); } finally @@ -444,7 +439,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) { InputStruct input = default; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try @@ -454,9 +449,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) for (int i = 0; i < nums.Length; ++i) { int randomIndex = rnd.Next(nums.Length); - int temp = nums[randomIndex]; - nums[randomIndex] = nums[i]; - nums[i] = temp; + (nums[i], nums[randomIndex]) = (nums[randomIndex], nums[i]); } for (int j = 0; j < nums.Length; ++j) @@ -464,14 +457,14 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = uContext.RMW(ref key1, ref input, Empty.Default); + _ = uContext.RMW(key1, ref input, Empty.Default); } for (int j = 0; j < nums.Length; ++j) { var i = nums[j]; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = i, ifield2 = i + 1 }; - _ = uContext.RMW(key1, input); // no ref and do not set any other params + _ = uContext.RMW(key1, ref input, Empty.Default); } OutputStruct output = default; @@ -485,7 +478,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) key = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; ValueStruct value = new() { vfield1 = i, vfield2 = i + 1 }; - status = uContext.Read(ref key, ref input, ref output, Empty.Default); + status = uContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); ClassicAssert.AreEqual(2 * value.vfield1, output.value.vfield1); @@ -493,7 +486,7 @@ public unsafe void NativeInMemRMWNoRefKeys([Values] TestDeviceType deviceType) } key = new KeyStruct { kfield1 = nums.Length, kfield2 = nums.Length + 1 }; - status = uContext.Read(ref key, ref input, ref output, Empty.Default); + status = uContext.Read(key, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.NotFound), status); } finally @@ -510,16 +503,17 @@ public void ReadNoRefKeyInputOutput([Values] TestDeviceType deviceType) { InputStruct input = default; - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try { var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; + OutputStruct output = new(); - _ = uContext.Upsert(ref key1, ref value, Empty.Default); - var status = uContext.Read(key1, input, out OutputStruct output, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = uContext.Read(key1, ref input, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -539,16 +533,17 @@ public void ReadNoRefKeyInputOutput([Values] TestDeviceType deviceType) [Category("TsavoriteKV")] public void ReadNoRefKey([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try { var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; + OutputStruct output = new(); - _ = uContext.Upsert(ref key1, ref value, Empty.Default); - var status = uContext.Read(key1, out OutputStruct output, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = uContext.Read(key1, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -570,7 +565,7 @@ public void ReadNoRefKey([Values] TestDeviceType deviceType) [Category("Smoke")] public void ReadWithoutInput([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try @@ -580,8 +575,8 @@ public void ReadWithoutInput([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); - var status = uContext.Read(ref key1, ref output, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); + var status = uContext.Read(key1, ref output, Empty.Default); AssertCompleted(new(StatusCode.Found), status); // Verify the read data @@ -602,7 +597,7 @@ public void ReadWithoutInput([Values] TestDeviceType deviceType) [Category("Smoke")] public void ReadBareMinParams([Values] TestDeviceType deviceType) { - Setup(new() { MemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); + Setup(new() { LogMemorySize = 1L << 22, SegmentSize = 1L << 22, PageSize = 1L << 10 }, deviceType); uContext.BeginUnsafe(); try @@ -610,7 +605,7 @@ public void ReadBareMinParams([Values] TestDeviceType deviceType) var key1 = new KeyStruct { kfield1 = 13, kfield2 = 14 }; var value = new ValueStruct { vfield1 = 23, vfield2 = 24 }; - _ = uContext.Upsert(ref key1, ref value, Empty.Default); + _ = uContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); var (status, output) = uContext.Read(key1); AssertCompleted(new(StatusCode.Found), status); diff --git a/libs/storage/Tsavorite/cs/test/NativeReadCacheTests.cs b/libs/storage/Tsavorite/cs/test/test.session/NativeReadCacheTests.cs similarity index 75% rename from libs/storage/Tsavorite/cs/test/NativeReadCacheTests.cs rename to libs/storage/Tsavorite/cs/test/test.session/NativeReadCacheTests.cs index 798cae74c5d..a8560d3cbe0 100644 --- a/libs/storage/Tsavorite/cs/test/NativeReadCacheTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session/NativeReadCacheTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,14 +9,12 @@ namespace Tsavorite.test.ReadCacheTests { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - public class NativeReadCacheTests : AllureTestBase + public class NativeReadCacheTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; [SetUp] @@ -29,12 +26,12 @@ public void Setup() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, ReadCacheMemorySize = 1L << 15, ReadCachePageSize = 1L << 10, ReadCacheEnabled = true - }, StoreFunctions.Create(new KeyStruct.Comparer()) + }, StoreFunctions.Create(new KeyStruct.Comparer(), SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -54,7 +51,7 @@ public void TearDown() [Category("Smoke")] public void NativeDiskWriteReadCache() { - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; InputStruct input = default; @@ -63,9 +60,9 @@ public void NativeDiskWriteReadCache() { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } - bContext.CompletePending(true); + _ = bContext.CompletePending(true); // Evict all records from main memory of hybrid log store.Log.FlushAndEvict(true); @@ -77,9 +74,9 @@ public void NativeDiskWriteReadCache() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read last 100 keys - all should be served from cache @@ -89,7 +86,7 @@ public void NativeDiskWriteReadCache() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); @@ -105,9 +102,9 @@ public void NativeDiskWriteReadCache() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read 100 keys - all should be served from cache @@ -117,7 +114,7 @@ public void NativeDiskWriteReadCache() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); @@ -128,7 +125,7 @@ public void NativeDiskWriteReadCache() { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i + 1, vfield2 = i + 2 }; - bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } // RMW to overwrite the read cache @@ -137,10 +134,10 @@ public void NativeDiskWriteReadCache() OutputStruct output = default; var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; input = new InputStruct { ifield1 = 1, ifield2 = 1 }; - var status = bContext.RMW(ref key1, ref input, ref output, Empty.Default); + var status = bContext.RMW(key1, ref input, ref output, Empty.Default); if (status.IsPending) { - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } else { @@ -156,7 +153,7 @@ public void NativeDiskWriteReadCache() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i + 1, vfield2 = i + 2 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); @@ -167,7 +164,7 @@ public void NativeDiskWriteReadCache() [Category("TsavoriteKV")] public void NativeDiskWriteReadCache2() { - using var session = store.NewSession(new Functions()); + using var session = store.NewSession(new Functions()); var bContext = session.BasicContext; InputStruct input = default; @@ -176,12 +173,11 @@ public void NativeDiskWriteReadCache2() { var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - bContext.Upsert(ref key1, ref value, Empty.Default); + _ = bContext.Upsert(key1, SpanByte.FromPinnedVariable(ref value), Empty.Default); } - bContext.CompletePending(true); + _ = bContext.CompletePending(true); - // Dispose the hybrid log from memory entirely - store.Log.DisposeFromMemory(); + store.Log.FlushAndEvict(wait: true); // Read 2000 keys - all should be served from disk, populating and evicting the read cache FIFO for (int i = 0; i < 2000; i++) @@ -190,9 +186,9 @@ public void NativeDiskWriteReadCache2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read last 100 keys - all should be served from cache @@ -202,7 +198,7 @@ public void NativeDiskWriteReadCache2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); @@ -218,9 +214,9 @@ public void NativeDiskWriteReadCache2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.IsPending); - bContext.CompletePending(true); + _ = bContext.CompletePending(true); } // Read 100 keys - all should be served from cache @@ -230,7 +226,7 @@ public void NativeDiskWriteReadCache2() var key1 = new KeyStruct { kfield1 = i, kfield2 = i + 1 }; var value = new ValueStruct { vfield1 = i, vfield2 = i + 1 }; - var status = bContext.Read(ref key1, ref input, ref output, Empty.Default); + var status = bContext.Read(key1, ref input, ref output, Empty.Default); ClassicAssert.IsTrue(status.Found); ClassicAssert.AreEqual(value.vfield1, output.value.vfield1); ClassicAssert.AreEqual(value.vfield2, output.value.vfield2); diff --git a/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs b/libs/storage/Tsavorite/cs/test/test.session/RandomReadCacheTests.cs similarity index 62% rename from libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs rename to libs/storage/Tsavorite/cs/test/test.session/RandomReadCacheTests.cs index 216b5c94a29..e53b12f1ddf 100644 --- a/libs/storage/Tsavorite/cs/test/ReproReadCacheTest.cs +++ b/libs/storage/Tsavorite/cs/test/test.session/RandomReadCacheTests.cs @@ -1,14 +1,12 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Buffers; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,36 +15,31 @@ namespace Tsavorite.test.ReadCacheTests { - using SpanByteStoreFunctions = StoreFunctions; - - [AllureNUnit] + using SpanByteStoreFunctions = StoreFunctions; [TestFixture] - internal class RandomReadCacheTests : AllureTestBase + internal class RandomReadCacheTests : TestBase { class Functions : SpanByteFunctions { - public override bool ConcurrentReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo, ref RecordInfo recordInfo) - => SingleReader(ref key, ref input, ref value, ref dst, ref readInfo); - - public override bool SingleReader(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory dst, ref ReadInfo readInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory dst, ref ReadInfo readInfo) { - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); - var inputString = new string(MemoryMarshal.Cast(input.AsReadOnlySpan())); - var valueString = new string(MemoryMarshal.Cast(value.AsReadOnlySpan())); + var keyString = new string(MemoryMarshal.Cast(srcLogRecord.Key)); + var inputString = new string(MemoryMarshal.Cast(input.ReadOnlySpan)); + var valueString = new string(MemoryMarshal.Cast(srcLogRecord.ValueSpan)); var actualValue = long.Parse(valueString); ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue); ClassicAssert.AreEqual(long.Parse(inputString), actualValue); - value.CopyTo(ref dst, MemoryPool.Shared); + srcLogRecord.ValueSpan.CopyTo(ref dst, MemoryPool.Shared); return true; } - public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, Empty context, Status status, RecordMetadata recordMetadata) + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, Empty context, Status status, RecordMetadata recordMetadata) { ClassicAssert.IsTrue(status.Found); - var keyString = new string(MemoryMarshal.Cast(key.AsReadOnlySpan())); - var inputString = new string(MemoryMarshal.Cast(input.AsReadOnlySpan())); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); + var keyString = new string(MemoryMarshal.Cast(diskLogRecord.Key)); + var inputString = new string(MemoryMarshal.Cast(input.ReadOnlySpan)); + var outputString = new string(MemoryMarshal.Cast(output.ReadOnlySpan)); var actualValue = long.Parse(outputString); ClassicAssert.AreEqual(long.Parse(keyString) * 2, actualValue); ClassicAssert.AreEqual(long.Parse(inputString), actualValue); @@ -55,7 +48,7 @@ public override void ReadCompletionCallback(ref SpanByte key, ref SpanByte input } IDevice log = default; - TsavoriteKV> store = default; + TsavoriteKV> store = default; [SetUp] public void Setup() @@ -64,10 +57,10 @@ public void Setup() string filename = Path.Join(MethodTestDir, "BasicTests.log"); - var kvSettings = new KVSettings() + var kvSettings = new KVSettings() { IndexSize = 1L << 26, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 12, }; @@ -75,7 +68,7 @@ public void Setup() { if (arg is ReadCacheMode rcm) { - if (rcm == ReadCacheMode.UseReadCache) + if (rcm == ReadCacheMode.UseRC) { kvSettings.ReadCacheMemorySize = 1L << 15; kvSettings.ReadCachePageSize = 1L << 12; @@ -94,7 +87,7 @@ public void Setup() kvSettings.LogDevice = log ??= Devices.CreateLogDevice(filename, deleteOnClose: true); store = new(kvSettings - , StoreFunctions.Create() + , StoreFunctions.Create(SpanByteComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -113,44 +106,41 @@ public void TearDown() [Category(TsavoriteKVTestCategory)] [Category(ReadCacheTestCategory)] [Category(StressTestCategory)] - //[Repeat(300)] + //[Repeat(1000)] public unsafe void RandomReadCacheTest([Values(1, 2, 8)] int numThreads, [Values] KeyContentionMode keyContentionMode, [Values] ReadCacheMode readCacheMode) { - if (numThreads == 1 && keyContentionMode == KeyContentionMode.Contention) + if (numThreads == 1 && keyContentionMode == KeyContentionMode.Cont) Assert.Ignore("Skipped because 1 thread cannot have contention"); if (numThreads > 2 && IsRunningAzureTests) Assert.Ignore("Skipped because > 2 threads when IsRunningAzureTests"); - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); const int PendingMod = 16; - void LocalRead(BasicContext> sessionContext, int i, ref int numPending, bool isLast) + void LocalRead(BasicContext> sessionContext, int i, ref int numPending, bool isLast) { + // These are OK to be local to this LocalRead call; if it goes pending, they will be copied into IHeapContainers. var keyString = $"{i}"; - var inputString = $"{i * 2}"; var key = MemoryMarshal.Cast(keyString.AsSpan()); - var input = MemoryMarshal.Cast(inputString.AsSpan()); - fixed (byte* kptr = key, iptr = input) + SpanByteAndMemory output = default; + Status status; + var inputString = $"{i * 2}"; + fixed (char* _ = inputString) { - var sbKey = SpanByte.FromPinnedSpan(key); - var sbInput = SpanByte.FromPinnedSpan(input); - SpanByteAndMemory output = default; - - var status = sessionContext.Read(ref sbKey, ref sbInput, ref output); + var input = PinnedSpanByte.FromPinnedSpan(MemoryMarshal.Cast(inputString.AsSpan())); + status = sessionContext.Read(TestSpanByteKey.CopySpan(key), ref input, ref output); + } - if (status.Found) - { - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); - ClassicAssert.AreEqual(i * 2, long.Parse(outputString)); - output.Memory.Dispose(); - } - else - { - ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {keyString}; status {status}"); - ++numPending; - } + if (status.Found) + { + var outputString = new string(MemoryMarshal.Cast(output.ReadOnlySpan)); + ClassicAssert.AreEqual(i * 2, long.Parse(outputString)); + output.Memory.Dispose(); + } + else + { + ClassicAssert.IsTrue(status.IsPending, $"was not Pending: {keyString}; status {status}"); + ++numPending; } if (numPending > 0 && ((numPending % PendingMod) == 0 || isLast)) @@ -160,14 +150,14 @@ void LocalRead(BasicContext(completedOutputs.Current.Key.AsReadOnlySpan()))); + long keyLong = long.Parse(new string(MemoryMarshal.Cast(completedOutputs.Current.Key.KeyBytes))); ClassicAssert.IsTrue(status.Found, $"key {keyLong}, {status}, wasPending {true}, pt 1"); ClassicAssert.IsNotNull(output.Memory, $"key {keyLong}, wasPending {true}, pt 2"); - var outputString = new string(MemoryMarshal.Cast(output.AsReadOnlySpan())); + var outputString = new string(MemoryMarshal.Cast(output.ReadOnlySpan)); ClassicAssert.AreEqual(keyLong * 2, long.Parse(outputString), $"key {keyLong}, wasPending {true}, pt 3"); output.Memory.Dispose(); } @@ -177,25 +167,25 @@ void LocalRead(BasicContext(new Functions()); + var session = store.NewSession(new Functions()); var sessionContext = session.BasicContext; int numPending = 0; // read through the keys in order (works) - for (int i = startKey; i < endKey; i++) - LocalRead(sessionContext, i, ref numPending, i == endKey - 1); + for (int keyNum = startKey; keyNum < endKey; keyNum++) + LocalRead(sessionContext, keyNum, ref numPending, keyNum == endKey - 1); // pick random keys to read var r = new Random(2115); - for (int i = startKey; i < endKey; i++) - LocalRead(sessionContext, r.Next(startKey, endKey), ref numPending, i == endKey - 1); + for (int keyNum = startKey; keyNum < endKey; keyNum++) + LocalRead(sessionContext, r.Next(startKey, endKey), ref numPending, keyNum == endKey - 1); } const int MaxKeys = 8000; { // Write the values first (single-threaded, all keys) - var session = store.NewSession(new Functions()); + var session = store.NewSession(new Functions()); var bContext = session.BasicContext; for (int i = 0; i < MaxKeys; i++) { @@ -203,11 +193,16 @@ void LocalRun(int startKey, int endKey) var valueString = $"{i * 2}"; var key = MemoryMarshal.Cast(keyString.AsSpan()); var value = MemoryMarshal.Cast(valueString.AsSpan()); - fixed (byte* k = key, v = value) + + fixed (byte* keyPtr = key) { - var sbKey = SpanByte.FromPinnedSpan(key); - var sbValue = SpanByte.FromPinnedSpan(value); - var status = bContext.Upsert(sbKey, sbValue); + var status = bContext.Upsert(TestSpanByteKey.FromPointer(keyPtr, key.Length), value); + + if (status.IsPending) + { + _ = bContext.CompletePending(wait: true); + } + ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } } @@ -225,7 +220,7 @@ void LocalRun(int startKey, int endKey) for (int t = 0; t < numThreads; t++) { var tid = t; - if (keyContentionMode == KeyContentionMode.Contention) + if (keyContentionMode == KeyContentionMode.Cont) tasks.Add(Task.Factory.StartNew(() => LocalRun(0, MaxKeys))); else tasks.Add(Task.Factory.StartNew(() => LocalRun(numKeysPerThread * tid, numKeysPerThread * (tid + 1)))); diff --git a/libs/storage/Tsavorite/cs/test/ReadAddressTests.cs b/libs/storage/Tsavorite/cs/test/test.session/ReadAddressTests.cs similarity index 65% rename from libs/storage/Tsavorite/cs/test/ReadAddressTests.cs rename to libs/storage/Tsavorite/cs/test/test.session/ReadAddressTests.cs index 40ae638f319..d8fefd7977a 100644 --- a/libs/storage/Tsavorite/cs/test/ReadAddressTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session/ReadAddressTests.cs @@ -2,9 +2,10 @@ // Licensed under the MIT license. using System; +using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Runtime.InteropServices; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,18 +15,23 @@ namespace Tsavorite.test.readaddress { // Must be in a separate block so the "using StructStoreFunctions" is the first line in its namespace declaration. - public struct KeyStruct(long first) + public struct KeyStruct(long first) : IKey { public long key = first; - public override readonly string ToString() => key.ToString(); + /// Not always pinned, so don't assume it is + public readonly bool IsPinned => false; - internal struct Comparer : IKeyComparer - { - public readonly long GetHashCode64(ref KeyStruct key) => Utility.GetHashCode(key.key); + [UnscopedRef] + public readonly ReadOnlySpan KeyBytes => MemoryMarshal.Cast(new(in key)); - public readonly bool Equals(ref KeyStruct k1, ref KeyStruct k2) => k1.key == k2.key; - } + /// + public bool HasNamespace => false; + + /// + public ReadOnlySpan NamespaceBytes => []; + + public override readonly string ToString() => key.ToString(); } public struct ValueStruct(long value) @@ -38,12 +44,10 @@ public struct ValueStruct(long value) namespace Tsavorite.test.readaddress { - using StructAllocator = BlittableAllocator>>; - using StructStoreFunctions = StoreFunctions>; - - [AllureNUnit] + using StructAllocator = SpanByteAllocator>; + using StructStoreFunctions = StoreFunctions; [TestFixture] - internal class ReadAddressTests : AllureTestBase + internal class ReadAddressTests : TestBase { const int NumKeys = 1000; const int KeyMod = 100; @@ -57,92 +61,100 @@ public struct Output { public long value; public long address; + public RecordInfo recordInfo; - public override readonly string ToString() => $"val {value}; addr {address}"; + public override readonly string ToString() => $"val {value}; addr {address}; RecordInfo {recordInfo}"; } private static long SetReadOutput(long key, long value) => (key << 32) | value; - public enum UseReadCache { NoReadCache, ReadCache } - - internal class Functions : SessionFunctionsBase + internal class Functions : SessionFunctionsBase { - internal long lastWriteAddress = Constants.kInvalidAddress; + internal long lastWriteAddress = LogAddress.kInvalidAddress; readonly bool useReadCache; internal ReadCopyOptions readCopyOptions = ReadCopyOptions.None; - bool preserveCopyUpdaterSource; + readonly bool preserveCopyUpdaterSource; internal Functions(bool preserveCopyUpdaterSource = false) { this.preserveCopyUpdaterSource = preserveCopyUpdaterSource; foreach (var arg in TestContext.CurrentContext.Test.Arguments) { - if (arg is UseReadCache urc) + if (arg is ReadCacheMode urc) { - useReadCache = urc == UseReadCache.ReadCache; + useReadCache = urc == ReadCacheMode.UseRC; continue; } } } - public override bool ConcurrentReader(ref KeyStruct key, ref ValueStruct input, ref ValueStruct value, ref Output output, ref ReadInfo readInfo, ref RecordInfo recordInfo) + public override bool Reader(in TSourceLogRecord srcLogRecord, ref ValueStruct input, ref Output output, ref ReadInfo readInfo) { - output.value = SetReadOutput(key.key, value.value); - output.address = readInfo.Address; - return true; - } - - public override bool SingleReader(ref KeyStruct key, ref ValueStruct input, ref ValueStruct value, ref Output output, ref ReadInfo readInfo) - { - output.value = SetReadOutput(key.key, value.value); + output.value = SetReadOutput(srcLogRecord.Key.AsRef().key, srcLogRecord.ValueSpan.AsRef().value); output.address = readInfo.Address; + output.recordInfo = srcLogRecord.Info; return true; } // Return false to force a chain of values. - public override bool ConcurrentWriter(ref KeyStruct key, ref ValueStruct input, ref ValueStruct src, ref ValueStruct dst, ref Output output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) => false; + public override bool InPlaceWriter(ref LogRecord logRecord, ref ValueStruct input, ReadOnlySpan src, ref Output output, ref UpsertInfo upsertInfo) => false; - public override bool InPlaceUpdater(ref KeyStruct key, ref ValueStruct input, ref ValueStruct value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) => false; + public override bool InPlaceUpdater(ref LogRecord logRecord, ref ValueStruct input, ref Output output, ref RMWInfo rmwInfo) => false; // Record addresses - public override bool SingleWriter(ref KeyStruct key, ref ValueStruct input, ref ValueStruct src, ref ValueStruct dst, ref Output output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ValueStruct input, ReadOnlySpan src, ref Output output, ref UpsertInfo upsertInfo) { - dst = src; + dstLogRecord.ValueSpan.AsRef() = src.AsRef(); output.address = upsertInfo.Address; + output.recordInfo = dstLogRecord.Info; lastWriteAddress = upsertInfo.Address; return true; } - public override bool InitialUpdater(ref KeyStruct key, ref ValueStruct input, ref ValueStruct value, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref ValueStruct input, ref Output output, ref RMWInfo rmwInfo) { lastWriteAddress = rmwInfo.Address; output.address = rmwInfo.Address; - output.value = value.value = input.value; + output.value = logRecord.ValueSpan.AsRef().value = input.value; + output.recordInfo = logRecord.Info; return true; } - public override bool CopyUpdater(ref KeyStruct key, ref ValueStruct input, ref ValueStruct oldValue, ref ValueStruct newValue, ref Output output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref ValueStruct input, ref Output output, ref RMWInfo rmwInfo) { lastWriteAddress = rmwInfo.Address; output.address = rmwInfo.Address; - output.value = newValue.value = input.value; + output.value = dstLogRecord.ValueSpan.AsRef().value = input.value; + output.recordInfo = dstLogRecord.Info; rmwInfo.PreserveCopyUpdaterSourceRecord = preserveCopyUpdaterSource; return true; } - public override void ReadCompletionCallback(ref KeyStruct key, ref ValueStruct input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) + /// + public override unsafe RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref ValueStruct input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = sizeof(ValueStruct), ValueIsObject = false }; + /// + public override unsafe RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref ValueStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = sizeof(ValueStruct), ValueIsObject = false }; + /// + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref ValueStruct input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length, ValueIsObject = false }; + + public override void ReadCompletionCallback(ref DiskLogRecord diskLogRecord, ref ValueStruct input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) { + Assert.That(output.recordInfo.IsNull, Is.Not.True); if (status.Found) { if (useReadCache && readCopyOptions.CopyTo == ReadCopyTo.ReadCache) - ClassicAssert.AreEqual(Constants.kInvalidAddress, recordMetadata.Address, $"key {key}"); + ClassicAssert.AreEqual(LogAddress.kInvalidAddress, recordMetadata.Address, $"key {diskLogRecord.Key.ToShortString()}"); else - ClassicAssert.AreEqual(output.address, recordMetadata.Address, $"key {key}"); // Should agree with what SingleWriter set + ClassicAssert.AreEqual(output.address, recordMetadata.Address, $"key {diskLogRecord.Key.ToShortString()}"); // Should agree with what InitialWriter set } } - public override void RMWCompletionCallback(ref KeyStruct key, ref ValueStruct input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) + public override void RMWCompletionCallback(ref DiskLogRecord diskLogRecord, ref ValueStruct input, ref Output output, Empty ctx, Status status, RecordMetadata recordMetadata) { + Assert.That(output.recordInfo.IsNull, Is.Not.True); if (status.Found) ClassicAssert.AreEqual(output.address, recordMetadata.Address); } @@ -150,7 +162,7 @@ public override void RMWCompletionCallback(ref KeyStruct key, ref ValueStruct in private class TestStore : IDisposable { - internal TsavoriteKV store; + internal TsavoriteKV store; internal IDevice logDevice; private readonly bool flush; @@ -170,10 +182,10 @@ internal TestStore(bool useReadCache, ReadCopyOptions readCopyOptions, bool flus ReadCopyOptions = readCopyOptions, // Use small-footprint values PageSize = 1L << 12, // (4K pages) - MemorySize = 1L << 20, // (1M memory for main log) + LogMemorySize = 1L << 20, // (1M memory for main log) CheckpointDir = Path.Join(MethodTestDir, "chkpt") - }, StoreFunctions.Create(new KeyStruct.Comparer()) + }, StoreFunctions.Create(LongKeyComparer.Instance, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -191,7 +203,7 @@ internal async ValueTask Flush() internal async Task Populate(bool useRMW, bool preserveCopyUpdaterSource = false) { var functions = new Functions(preserveCopyUpdaterSource); - using var session = store.NewSession(functions); + using var session = store.NewSession(functions); var bContext = session.BasicContext; var prevLap = 0; @@ -210,8 +222,8 @@ internal async Task Populate(bool useRMW, bool preserveCopyUpdaterSource = false var value = new ValueStruct(key.key + LapOffset(lap)); var status = useRMW - ? bContext.RMW(ref key, ref value) - : bContext.Upsert(ref key, ref value); + ? bContext.RMW(key, ref value) + : bContext.Upsert(key, SpanByte.FromPinnedVariable(ref value)); if (status.IsPending) await bContext.CompletePendingAsync().ConfigureAwait(false); @@ -221,15 +233,15 @@ internal async Task Populate(bool useRMW, bool preserveCopyUpdaterSource = false // Illustrate that deleted records can be shown as well (unless overwritten by in-place operations, which are not done here) if (lap == DeleteLap) - _ = bContext.Delete(ref key); + _ = bContext.Delete(key); } await Flush().ConfigureAwait(false); } - internal bool ProcessChainRecord(Status status, RecordMetadata recordMetadata, int lap, ref Output actualOutput) + internal bool ProcessChainRecord(int lap, ref Output actualOutput) { - var recordInfo = recordMetadata.RecordInfo; + var recordInfo = actualOutput.recordInfo; ClassicAssert.GreaterOrEqual(lap, 0); long expectedValue = SetReadOutput(DefaultKeyToScan, LapOffset(lap) + DefaultKeyToScan); @@ -241,7 +253,7 @@ internal bool ProcessChainRecord(Status status, RecordMetadata recordMetadata, i return recordInfo.PreviousAddress >= store.Log.BeginAddress; } - internal static void ProcessNoKeyRecord(bool useRMW, Status status, RecordInfo recordInfo, ref Output actualOutput, int keyOrdinal) + internal static void ProcessNoKeyRecord(Status status, RecordInfo recordInfo, ref Output actualOutput, int keyOrdinal) { if (status.Found) { @@ -264,20 +276,20 @@ public void Dispose() } // readCache and copyReadsToTail are mutually exclusive and orthogonal to populating by RMW vs. Upsert. - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] [Category("TsavoriteKV"), Category("Read")] - public void VersionedReadTests(UseReadCache urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) + public void VersionedReadTests(ReadCacheMode urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) { - var useReadCache = urc == UseReadCache.ReadCache; + var useReadCache = urc == ReadCacheMode.UseRC; var readCopyOptions = new ReadCopyOptions(readCopyFrom, readCopyTo); using var testStore = new TestStore(useReadCache, readCopyOptions, flushMode == FlushMode.OnDisk); testStore.Populate(updateOp == UpdateOp.RMW).GetAwaiter().GetResult(); - using var session = testStore.store.NewSession(new Functions()); + using var session = testStore.store.NewSession(new Functions()); var bContext = session.BasicContext; // Two iterations to ensure no issues due to read-caching or copying to tail. @@ -294,8 +306,8 @@ public void VersionedReadTests(UseReadCache urc, ReadCopyFrom readCopyFrom, Read { // We need a non-AtAddress read to start the loop of returning the previous address to read at. var status = readAtAddress == 0 - ? bContext.Read(ref key, ref input, ref output, ref readOptions, out _) - : bContext.ReadAtAddress(readAtAddress, ref key, ref input, ref output, ref readOptions, out _); + ? bContext.Read(key, ref input, ref output, ref readOptions, out _) + : bContext.ReadAtAddress(readAtAddress, key, ref input, ref output, ref readOptions, out _); if (status.IsPending) { @@ -303,14 +315,14 @@ public void VersionedReadTests(UseReadCache urc, ReadCopyFrom readCopyFrom, Read _ = bContext.CompletePendingWithOutputs(out var completedOutputs, wait: true); (status, output) = GetSinglePendingResult(completedOutputs, out recordMetadata); } - if (!testStore.ProcessChainRecord(status, recordMetadata, lap, ref output)) + if (!testStore.ProcessChainRecord(lap, ref output)) break; - readAtAddress = recordMetadata.RecordInfo.PreviousAddress; + readAtAddress = output.recordInfo.PreviousAddress; } } } - struct IterateKeyTestScanIteratorFunctions : IScanIteratorFunctions + struct IterateKeyTestScanIteratorFunctions : IScanIteratorFunctions { readonly TestStore testStore; internal int numRecords; @@ -320,15 +332,13 @@ struct IterateKeyTestScanIteratorFunctions : IScanIteratorFunctions true; - public bool ConcurrentReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) - => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult); - - public bool SingleReader(ref KeyStruct key, ref ValueStruct value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + public bool Reader(in TSourceLogRecord logRecord, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult) + where TSourceLogRecord : ISourceLogRecord { cursorRecordResult = CursorRecordResult.Accept; // default; not used here - Output output = new() { address = recordMetadata.Address, value = SetReadOutput(key.key, value.value) }; + Output output = new() { address = recordMetadata.Address, value = SetReadOutput(logRecord.Key.AsRef().key, logRecord.ValueSpan.AsRef().value), recordInfo = logRecord.Info }; int lap = MaxLap - ++numRecords; - ClassicAssert.AreEqual(lap != 0, testStore.ProcessChainRecord(new(StatusCode.Found), recordMetadata, lap, ref output), $"lap ({lap}) == 0 != ProcessChainRecord(...)"); + ClassicAssert.AreEqual(lap != 0, testStore.ProcessChainRecord(lap, ref output), $"lap ({lap}) == 0 != ProcessChainRecord(...)"); ClassicAssert.AreEqual(numRecords, numberOfRecords, "mismatched record count"); return stopAt != numRecords; } @@ -352,7 +362,7 @@ public void IterateKeyTests([Values(FlushMode.NoFlush, FlushMode.OnDisk)] FlushM { var key = new KeyStruct(DefaultKeyToScan); IterateKeyTestScanIteratorFunctions scanFunctions = new(testStore); - ClassicAssert.IsTrue(testStore.store.Log.IterateKeyVersions(ref scanFunctions, ref key)); + ClassicAssert.IsTrue(testStore.store.Log.IterateKeyVersions(ref scanFunctions, key)); ClassicAssert.AreEqual(MaxLap, scanFunctions.numRecords); } } @@ -371,26 +381,26 @@ public void IterateKeyStopTests([Values(FlushMode.NoFlush, FlushMode.OnDisk)] Fl { var key = new KeyStruct(DefaultKeyToScan); IterateKeyTestScanIteratorFunctions scanFunctions = new(testStore) { stopAt = 4 }; - ClassicAssert.IsFalse(testStore.store.Log.IterateKeyVersions(ref scanFunctions, ref key)); + ClassicAssert.IsFalse(testStore.store.Log.IterateKeyVersions(ref scanFunctions, key)); ClassicAssert.AreEqual(scanFunctions.stopAt, scanFunctions.numRecords); } } // readCache and copyReadsToTail are mutually exclusive and orthogonal to populating by RMW vs. Upsert. - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] [Category("TsavoriteKV"), Category("Read")] - public void ReadAtAddressTests(UseReadCache urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) + public void ReadAtAddressTests(ReadCacheMode urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) { - var useReadCache = urc == UseReadCache.ReadCache; + var useReadCache = urc == ReadCacheMode.UseRC; var readCopyOptions = new ReadCopyOptions(readCopyFrom, readCopyTo); using var testStore = new TestStore(useReadCache, readCopyOptions, flushMode == FlushMode.OnDisk); testStore.Populate(updateOp == UpdateOp.RMW).GetAwaiter().GetResult(); - using var session = testStore.store.NewSession(new Functions()); + using var session = testStore.store.NewSession(new Functions()); var bContext = session.BasicContext; // Two iterations to ensure no issues due to read-caching or copying to tail. @@ -406,7 +416,7 @@ public void ReadAtAddressTests(UseReadCache urc, ReadCopyFrom readCopyFrom, Read for (int lap = MaxLap - 1; /* tested in loop */; --lap) { var status = readAtAddress == 0 - ? bContext.Read(ref key, ref input, ref output, ref readOptions, out recordMetadata) + ? bContext.Read(key, ref input, ref output, ref readOptions, out recordMetadata) : bContext.ReadAtAddress(readAtAddress, ref input, ref output, ref readOptions, out recordMetadata); if (status.IsPending) { @@ -415,28 +425,28 @@ public void ReadAtAddressTests(UseReadCache urc, ReadCopyFrom readCopyFrom, Read (status, output) = GetSinglePendingResult(completedOutputs, out recordMetadata); } - if (!testStore.ProcessChainRecord(status, recordMetadata, lap, ref output)) + if (!testStore.ProcessChainRecord(lap, ref output)) break; - readAtAddress = recordMetadata.RecordInfo.PreviousAddress; + readAtAddress = output.recordInfo.PreviousAddress; } } } // Test is similar to others but tests the Overload where RadCopy*.None is set -- probably don't need all combinations of test but doesn't hurt - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] [Category("TsavoriteKV"), Category("Read")] - public async Task ReadAtAddressCopyOptNoRcTest(UseReadCache urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) + public async Task ReadAtAddressCopyOptNoRcTest(ReadCacheMode urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) { - var useReadCache = urc == UseReadCache.ReadCache; + var useReadCache = urc == ReadCacheMode.UseRC; var readCopyOptions = new ReadCopyOptions(readCopyFrom, readCopyTo); using var testStore = new TestStore(useReadCache, readCopyOptions, flushMode == FlushMode.OnDisk); await testStore.Populate(updateOp == UpdateOp.RMW).ConfigureAwait(false); - using var session = testStore.store.NewSession(new Functions()); + using var session = testStore.store.NewSession(new Functions()); var bContext = session.BasicContext; // Two iterations to ensure no issues due to read-caching or copying to tail. @@ -452,33 +462,34 @@ public async Task ReadAtAddressCopyOptNoRcTest(UseReadCache urc, ReadCopyFrom re { Output output = new(); Status status = readAtAddress == 0 - ? bContext.Read(ref key, ref input, ref output, ref readOptions, out recordMetadata) + ? bContext.Read(key, ref input, ref output, ref readOptions, out recordMetadata) : bContext.ReadAtAddress(readAtAddress, ref input, ref output, ref readOptions, out recordMetadata); if (status.IsPending) (status, output) = bContext.GetSinglePendingResult(out recordMetadata); + Assert.That(output.recordInfo.Tombstone, Is.EqualTo(lap == DeleteLap), $"lap {lap}"); - if (!testStore.ProcessChainRecord(status, recordMetadata, lap, ref output)) + if (!testStore.ProcessChainRecord(lap, ref output)) break; - readAtAddress = recordMetadata.RecordInfo.PreviousAddress; + readAtAddress = output.recordInfo.PreviousAddress; } } } // readCache and copyReadsToTail are mutually exclusive and orthogonal to populating by RMW vs. Upsert. - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.NoReadCache, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] - [TestCase(UseReadCache.ReadCache, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.NoFlush)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.NoRC, ReadCopyFrom.Device, ReadCopyTo.MainLog, UpdateOp.RMW, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.Upsert, FlushMode.OnDisk)] + [TestCase(ReadCacheMode.UseRC, ReadCopyFrom.None, ReadCopyTo.None, UpdateOp.RMW, FlushMode.OnDisk)] [Category("TsavoriteKV"), Category("Read")] - public async ValueTask ReadNoKeyTests(UseReadCache urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) + public async ValueTask ReadNoKeyTests(ReadCacheMode urc, ReadCopyFrom readCopyFrom, ReadCopyTo readCopyTo, UpdateOp updateOp, FlushMode flushMode) { - var useReadCache = urc == UseReadCache.ReadCache; + var useReadCache = urc == ReadCacheMode.UseRC; var readCopyOptions = new ReadCopyOptions(readCopyFrom, readCopyTo); using var testStore = new TestStore(useReadCache, readCopyOptions, flushMode == FlushMode.OnDisk); await testStore.Populate(updateOp == UpdateOp.RMW).ConfigureAwait(false); - using var session = testStore.store.NewSession(new Functions()); + using var session = testStore.store.NewSession(new Functions()); var bContext = session.BasicContext; // Two iterations to ensure no issues due to read-caching or copying to tail. @@ -504,7 +515,7 @@ public async ValueTask ReadNoKeyTests(UseReadCache urc, ReadCopyFrom readCopyFro (status, output) = GetSinglePendingResult(completedOutputs); } - TestStore.ProcessNoKeyRecord(updateOp == UpdateOp.RMW, status, recordMetadata.RecordInfo, ref output, keyOrdinal); + TestStore.ProcessNoKeyRecord(status, output.recordInfo, ref output, keyOrdinal); } await testStore.Flush().AsTask().ConfigureAwait(false); diff --git a/libs/storage/Tsavorite/cs/test/ReadCacheChainTests.cs b/libs/storage/Tsavorite/cs/test/test.session/ReadCacheChainTests.cs similarity index 67% rename from libs/storage/Tsavorite/cs/test/ReadCacheChainTests.cs rename to libs/storage/Tsavorite/cs/test/test.session/ReadCacheChainTests.cs index 288f600a095..cc3b9366e82 100644 --- a/libs/storage/Tsavorite/cs/test/ReadCacheChainTests.cs +++ b/libs/storage/Tsavorite/cs/test/test.session/ReadCacheChainTests.cs @@ -1,41 +1,38 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.test; using NUnit.Framework; using NUnit.Framework.Legacy; using Tsavorite.core; -using Tsavorite.test.LockableUnsafeContext; using Tsavorite.test.LockTable; +using Tsavorite.test.TransactionalUnsafeContext; +using static Tsavorite.core.LogAddress; using static Tsavorite.test.TestUtils; #pragma warning disable // Add parentheses for clarity namespace Tsavorite.test.ReadCacheTests { - using LongAllocator = BlittableAllocator>>; - using LongStoreFunctions = StoreFunctions>; - using SpanByteStoreFunctions = StoreFunctions; + using LongAllocator = SpanByteAllocator>; + using LongStoreFunctions = StoreFunctions; + using SpanByteStoreFunctions = StoreFunctions; internal static class RcTestGlobals { internal const int PendingMod = 16; } - - [AllureNUnit] [TestFixture] - class ChainTests : AllureTestBase + class ChainTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; - private LongComparerModulo comparer; + private LongKeyComparerModulo comparer; const long LowChainKey = 40; const long MidChainKey = LowChainKey + ChainLen * (HashMod / 2); @@ -62,17 +59,17 @@ public void Setup() DeleteDirectory(MethodTestDir, wait: true); log = Devices.CreateLogDevice(Path.Join(MethodTestDir, "NativeReadCacheTests.log"), deleteOnClose: true); - comparer = new LongComparerModulo(HashMod); + comparer = new LongKeyComparerModulo(HashMod); store = new(new() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, ReadCacheMemorySize = 1L << 15, ReadCachePageSize = 1L << 9, ReadCacheEnabled = true - }, StoreFunctions.Create(comparer) + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -91,13 +88,16 @@ public enum RecordRegion { Immutable, OnDisk, Mutable }; void PopulateAndEvict(RecordRegion recordRegion = RecordRegion.OnDisk) { - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; + long keyVal = 0, valueVal = 0; + Span key = SpanByte.FromPinnedVariable(ref keyVal), value = SpanByte.FromPinnedVariable(ref valueVal); + if (recordRegion != RecordRegion.Immutable) { - for (int key = 0; key < NumKeys; key++) - _ = bContext.Upsert(key, key + ValueAdd); + for (int keyNum = 0; keyNum < NumKeys; keyNum++) + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key.SetSlice(keyNum)), value.SetSlice(keyNum + ValueAdd)); _ = bContext.CompletePending(true); if (recordRegion == RecordRegion.OnDisk) store.Log.FlushAndEvict(true); @@ -105,31 +105,36 @@ void PopulateAndEvict(RecordRegion recordRegion = RecordRegion.OnDisk) } // Two parts, so we can have some evicted (and bring them into the readcache), and some in immutable (readonly). - for (int key = 0; key < ImmutableSplitKey; key++) - _ = bContext.Upsert(key, key + ValueAdd); + for (int keyNum = 0; keyNum < ImmutableSplitKey; keyNum++) + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key.SetSlice(keyNum)), value.SetSlice(keyNum + ValueAdd)); _ = bContext.CompletePending(true); store.Log.FlushAndEvict(true); - for (long key = ImmutableSplitKey; key < NumKeys; key++) - _ = bContext.Upsert(key, key + ValueAdd); + for (long keyNum = ImmutableSplitKey; keyNum < NumKeys; keyNum++) + _ = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key.SetSlice(keyNum)), value.SetSlice(keyNum + ValueAdd)); _ = bContext.CompletePending(true); store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); } void CreateChain(RecordRegion recordRegion = RecordRegion.OnDisk) { - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; long output = -1; bool expectPending(long key) => recordRegion == RecordRegion.OnDisk || (recordRegion == RecordRegion.Immutable && key < ImmutableSplitKey); + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + var value = SpanByte.FromPinnedVariable(ref valueVal); + // Pass1: PENDING reads and populate the cache for (long ii = 0; ii < ChainLen; ++ii) { - var key = LowChainKey + ii * HashMod; - var status = bContext.Read(key, out _); - if (expectPending(key)) + var keyNum = LowChainKey + ii * HashMod; + key.Set((long)keyNum); + var status = bContext.Read(key, ref output); + if (expectPending(keyNum)) { ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); @@ -137,24 +142,25 @@ void CreateChain(RecordRegion recordRegion = RecordRegion.OnDisk) ClassicAssert.IsTrue(status.Record.CopiedToReadCache, status.ToString()); } ClassicAssert.IsTrue(status.Found, status.ToString()); - if (key < MidChainKey) + if (keyNum < MidChainKey) readCacheBelowMidChainKeyEvictionAddress = store.ReadCache.TailAddress; } // Pass2: non-PENDING reads from the cache for (var ii = 0; ii < ChainLen; ++ii) { - var status = bContext.Read(LowChainKey + ii * HashMod, out _); + var status = bContext.Read(key.Set((long)LowChainKey + ii * HashMod), ref output); ClassicAssert.IsTrue(!status.IsPending && status.Found, status.ToString()); } // Pass 3: Put in bunch of extra keys into the cache so when we FlushAndEvict we get all the ones of interest. - for (var key = 0; key < NumKeys; ++key) + for (var keyNum = 0; keyNum < NumKeys; ++keyNum) { - if ((key % HashMod) != 0) + if ((keyNum % HashMod) != 0) { - var status = bContext.Read(key, out _); - if (expectPending(key)) + key.Set((long)keyNum); + var status = bContext.Read(key, ref output); + if (expectPending(keyNum)) { ClassicAssert.IsTrue(status.IsPending); _ = bContext.CompletePendingWithOutputs(out var outputs, wait: true); @@ -167,65 +173,64 @@ void CreateChain(RecordRegion recordRegion = RecordRegion.OnDisk) } } - unsafe bool GetRecordInInMemoryHashChain(long key, out bool isReadCache) + unsafe bool GetRecordInInMemoryHashChain(long keyNum, out bool isReadCache) { // returns whether the key was found before we'd go pending - var (la, pa) = GetHashChain(store, key, out long recordKey, out bool invalid, out isReadCache); + var (la, pa) = GetHashChain(store, TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)), out var recordKey, out bool invalid, out isReadCache); while (isReadCache || la >= store.hlogBase.HeadAddress) { - if (recordKey == key && !invalid) + if (recordKey.ReadOnlySpan.AsRef() == keyNum && !invalid) return true; (la, pa) = NextInChain(store, pa, out recordKey, out invalid, ref isReadCache); } return false; } - internal bool FindRecordInReadCache(long key, out bool invalid, out long logicalAddress, out long physicalAddress) + internal bool FindRecordInReadCache(TestSpanByteKey key, out bool invalid, out long logicalAddress, out long physicalAddress) { // returns whether the key was found before we'd go pending - (logicalAddress, physicalAddress) = GetHashChain(store, key, out long recordKey, out invalid, out bool isReadCache); + (logicalAddress, physicalAddress) = GetHashChain(store, key, out var recordKey, out invalid, out bool isReadCache); while (isReadCache) { - if (recordKey == key) + if (recordKey.ReadOnlySpan.AsRef() == key.AsRef()) return true; (logicalAddress, physicalAddress) = NextInChain(store, physicalAddress, out recordKey, out invalid, ref isReadCache); } return false; } - internal static (long logicalAddress, long physicalAddress) GetHashChain(TsavoriteKV store, long key, out long recordKey, out bool invalid, out bool isReadCache) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static (long logicalAddress, long physicalAddress) GetHashChain(TsavoriteKV store, TestSpanByteKey key, out PinnedSpanByte recordKey, out bool invalid, out bool isReadCache) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - var tagExists = store.FindHashBucketEntryForKey(ref key, out var entry); + var tagExists = store.FindHashBucketEntryForKey(key, out var entry); ClassicAssert.IsTrue(tagExists); - isReadCache = entry.ReadCache; - var log = isReadCache ? store.readcache : store.hlog; - var pa = log.GetPhysicalAddress(entry.Address & ~Constants.kReadCacheBitMask); - recordKey = log.GetKey(pa); - invalid = log.GetInfo(pa).Invalid; + isReadCache = entry.IsReadCache; + var log = isReadCache ? store.readcacheBase : store.hlogBase; + var pa = log.GetPhysicalAddress(entry.Address); + recordKey = PinnedSpanByte.FromPinnedSpan(LogRecord.GetInlineKey(pa)); // Must return PinnedSpanByte to avoid scope issues with ReadOnlySpan + invalid = LogRecord.GetInfo(pa).Invalid; return (entry.Address, pa); } - (long logicalAddress, long physicalAddress) NextInChain(long physicalAddress, out long recordKey, out bool invalid, ref bool isReadCache) + (long logicalAddress, long physicalAddress) NextInChain(long physicalAddress, out PinnedSpanByte recordKey, out bool invalid, ref bool isReadCache) => NextInChain(store, physicalAddress, out recordKey, out invalid, ref isReadCache); - internal static (long logicalAddress, long physicalAddress) NextInChain(TsavoriteKV store, long physicalAddress, out long recordKey, out bool invalid, ref bool isReadCache) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static (long logicalAddress, long physicalAddress) NextInChain(TsavoriteKV store, long physicalAddress, out PinnedSpanByte recordKey, out bool invalid, ref bool isReadCache) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { - var log = isReadCache ? store.readcache : store.hlog; - var info = log.GetInfo(physicalAddress); + var log = isReadCache ? store.readcacheBase : store.hlogBase; + var info = LogRecord.GetInfo(physicalAddress); var la = info.PreviousAddress; - isReadCache = new HashBucketEntry { word = la }.ReadCache; - log = isReadCache ? store.readcache : store.hlog; - la &= ~Constants.kReadCacheBitMask; + isReadCache = IsReadCache(la); + log = isReadCache ? store.readcacheBase : store.hlogBase; var pa = log.GetPhysicalAddress(la); - recordKey = log.GetKey(pa); - invalid = log.GetInfo(pa).Invalid; + recordKey = PinnedSpanByte.FromPinnedSpan(LogRecord.GetInlineKey(pa)); // Must return PinnedSpanByte to avoid scope issues with ReadOnlySpan + invalid = LogRecord.GetInfo(pa).Invalid; return (la, pa); } @@ -233,8 +238,12 @@ internal static (long logicalAddress, long physicalAddress) NextInChain= LowChainKey; expectedKey -= HashMod) + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + var value = SpanByte.FromPinnedVariable(ref valueVal); + + var (la, pa) = GetHashChain(store, key.Set((long)LowChainKey), out var actualKey, out bool invalid, out bool isReadCache); + for (long expectedKey = HighChainKey; expectedKey >= LowChainKey; expectedKey -= HashMod) { // We evict from readcache only to just below midChainKey if (!evicted || expectedKey >= MidChainKey) @@ -242,13 +251,13 @@ internal static (long logicalAddress, long physicalAddress) NextInChain()); if (omitted.Contains(expectedKey)) ClassicAssert.IsTrue(invalid); } - else if (omitted.Contains(actualKey)) + else if (omitted.Contains(actualKey.ReadOnlySpan.AsRef())) { - ClassicAssert.AreEqual(deleted, store.hlog.GetInfo(pa).Tombstone); + ClassicAssert.AreEqual(deleted, LogRecord.GetInfo(pa).Tombstone); } (la, pa) = NextInChain(pa, out actualKey, out invalid, ref isReadCache); @@ -259,12 +268,12 @@ internal static (long logicalAddress, long physicalAddress) NextInChain SkipReadCacheChain(store, key); - internal static (long logicalAddress, long physicalAddress) SkipReadCacheChain(TsavoriteKV store, long key) - where TStoreFunctions : IStoreFunctions - where TAllocator : IAllocator + internal static (long logicalAddress, long physicalAddress) SkipReadCacheChain(TsavoriteKV store, TestSpanByteKey key) + where TStoreFunctions : IStoreFunctions + where TAllocator : IAllocator { var (la, pa) = GetHashChain(store, key, out _, out _, out bool isReadCache); while (isReadCache) @@ -272,15 +281,15 @@ internal static (long logicalAddress, long physicalAddress) SkipReadCacheChain(), storedKey.AsRef()); } - static void ClearCountsOnError(ClientSession, LongStoreFunctions, LongAllocator> luContext) + static void ClearCountsOnError(ClientSession luContext) { // If we already have an exception, clear these counts so "Run" will not report them spuriously. luContext.sharedLockCount = 0; @@ -308,15 +317,20 @@ public void DeleteCacheRecordTest() { PopulateAndEvict(); CreateChain(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - void doTest(long key) + void doTest(long keyNum) { + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + var value = SpanByte.FromPinnedVariable(ref valueVal); + + key.Set((long)keyNum); var status = bContext.Delete(key); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); - status = bContext.Read(key, out var value); + status = bContext.Read(key, ref valueVal); ClassicAssert.IsFalse(status.Found, status.ToString()); } @@ -337,15 +351,20 @@ public void DeleteHalfOfAllReadCacheRecordsTest() { PopulateAndEvict(); CreateChain(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - void doTest(long key) + void doTest(long keyNum) { + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + var value = SpanByte.FromPinnedVariable(ref valueVal); + + key.Set((long)keyNum); var status = bContext.Delete(key); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); - status = bContext.Read(key, out var value); + status = bContext.Read(key, ref valueVal); ClassicAssert.IsFalse(status.Found, status.ToString()); } @@ -404,18 +423,24 @@ void DoUpdateTest(bool useRMW) { PopulateAndEvict(); CreateChain(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - void doTest(long key) + void doTest(long keyNum) { - var status = bContext.Read(key, out var value); + long keyVal = 0, valueVal = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyVal)); + var value = SpanByte.FromPinnedVariable(ref valueVal); + + key.Set((long)keyNum); + var status = bContext.Read(key, ref valueVal); ClassicAssert.IsTrue(status.Found, status.ToString()); + long input = valueVal + ValueAdd; if (useRMW) { // RMW will use the readcache entry for its source and then invalidate it. - status = bContext.RMW(key, value + ValueAdd); + status = bContext.RMW(key, ref input); ClassicAssert.IsTrue(status.Found && status.Record.CopyUpdated, status.ToString()); ClassicAssert.IsTrue(FindRecordInReadCache(key, out bool invalid, out _, out _)); @@ -423,13 +448,13 @@ void doTest(long key) } else { - status = bContext.Upsert(key, value + ValueAdd); + status = bContext.Upsert(key, value.Set(input)); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } - status = bContext.Read(key, out value); + status = bContext.Read(key, ref valueVal); ClassicAssert.IsTrue(status.Found, status.ToString()); - ClassicAssert.AreEqual(key + ValueAdd * 2, value); + ClassicAssert.AreEqual(keyNum + ValueAdd * 2, valueVal); } doTest(LowChainKey); @@ -450,13 +475,14 @@ public void SpliceInFromCTTTest() PopulateAndEvict(); CreateChain(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - long input = 0, output = 0, key = LowChainKey - HashMod; // key must be in evicted region for this test + long input = 0, output = 0, keyNum = LowChainKey - HashMod; // key must be in evicted region for this test + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); ReadOptions readOptions = new() { CopyOptions = new(ReadCopyFrom.AllImmutable, ReadCopyTo.MainLog) }; - var status = bContext.Read(ref key, ref input, ref output, ref readOptions, out _); + var status = bContext.Read(key, ref input, ref output, ref readOptions, out _); ClassicAssert.IsTrue(status.IsPending, status.ToString()); _ = bContext.CompletePending(wait: true); @@ -472,21 +498,23 @@ public void SpliceInFromUpsertTest([Values] RecordRegion recordRegion) PopulateAndEvict(recordRegion); CreateChain(recordRegion); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - long key = -1; + long keyNum = -1, valueNum = 0; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) { - key = SpliceInExistingKey; - var status = bContext.Upsert(key, key + ValueAdd); + keyNum = SpliceInExistingKey; + var status = bContext.Upsert(key, value.Set(keyNum + ValueAdd)); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } else { - key = SpliceInNewKey; - var status = bContext.Upsert(key, key + ValueAdd); + keyNum = SpliceInNewKey; + var status = bContext.Upsert(key, value.Set(keyNum + ValueAdd)); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } @@ -502,15 +530,19 @@ public void SpliceInFromRMWTest([Values] RecordRegion recordRegion) PopulateAndEvict(recordRegion); CreateChain(recordRegion); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - long key = -1, output = -1; + long keyNum = -1, valueNum = 0, output = -1; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); + var value = SpanByte.FromPinnedVariable(ref valueNum); + + long input = keyNum + ValueAdd; if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) { // Existing key - key = SpliceInExistingKey; - var status = bContext.RMW(key, key + ValueAdd); + keyNum = SpliceInExistingKey; + var status = bContext.RMW(key, ref input); // If OnDisk, this used the readcache entry for its source and then invalidated it. ClassicAssert.IsTrue(status.Found && status.Record.CopyUpdated, status.ToString()); @@ -521,8 +553,8 @@ public void SpliceInFromRMWTest([Values] RecordRegion recordRegion) } { // New key - key = SpliceInNewKey; - status = bContext.RMW(key, key + ValueAdd); + keyNum = SpliceInNewKey; + status = bContext.RMW(key, ref input); // This NOTFOUND key will return PENDING because we have to trace back through the collisions. ClassicAssert.IsTrue(status.IsPending, status.ToString()); @@ -533,8 +565,8 @@ public void SpliceInFromRMWTest([Values] RecordRegion recordRegion) } else { - key = SpliceInNewKey; - var status = bContext.RMW(key, key + ValueAdd); + keyNum = SpliceInNewKey; + var status = bContext.RMW(key, ref input); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } @@ -550,19 +582,20 @@ public void SpliceInFromDeleteTest([Values] RecordRegion recordRegion) PopulateAndEvict(recordRegion); CreateChain(recordRegion); - using var session = store.NewSession>(new SimpleSimpleFunctions()); + using var session = store.NewSession(new SimpleLongSimpleFunctions()); var bContext = session.BasicContext; - long key = -1; + long keyNum = -1; + var key = TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref keyNum)); if (recordRegion is RecordRegion.Immutable or RecordRegion.OnDisk) { - key = SpliceInExistingKey; + keyNum = SpliceInExistingKey; var status = bContext.Delete(key); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } else { - key = SpliceInNewKey; + keyNum = SpliceInNewKey; var status = bContext.Delete(key); ClassicAssert.IsTrue(!status.Found && status.Record.Created, status.ToString()); } @@ -579,30 +612,35 @@ public void VerifyLockCountsAfterReadCacheEvict() PopulateAndEvict(); CreateChain(); - using var session = store.NewSession>(new SimpleSimpleFunctions()); - var luContext = session.LockableUnsafeContext; + using var session = store.NewSession(new SimpleLongSimpleFunctions()); + var luContext = session.TransactionalUnsafeContext; + + var keyNums = GC.AllocateArray(3, pinned: true); + keyNums[0] = LowChainKey; + keyNums[1] = MidChainKey; + keyNums[2] = HighChainKey; var keys = new[] { - new FixedLengthLockableKeyStruct(LowChainKey, LockType.Exclusive, luContext), - new FixedLengthLockableKeyStruct(MidChainKey, LockType.Shared, luContext), - new FixedLengthLockableKeyStruct(HighChainKey, LockType.Exclusive, luContext) + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref keyNums[0]), LockType.Exclusive, luContext), + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref keyNums[1]), LockType.Shared, luContext), + new FixedLengthTransactionalKeyStruct(SpanByte.FromPinnedVariable(ref keyNums[2]), LockType.Exclusive, luContext) }; luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); try { - luContext.SortKeyHashes>(keys); + luContext.SortKeyHashes(keys); // For this single-threaded test, the locking does not really have to be in order, but for consistency do it. - luContext.Lock>(keys); + luContext.Lock(keys); store.ReadCache.FlushAndEvict(wait: true); int xlocks = 0, slocks = 0; - foreach (var idx in LockableUnsafeContextTests.EnumActionKeyIndices(keys, LockableUnsafeContextTests.LockOperationType.Unlock)) + foreach (var idx in TransactionalUnsafeContextTestUtils.EnumActionKeyIndices(keys, LockOperationType.Unlock)) { if (keys[idx].LockType == LockType.Exclusive) ++xlocks; @@ -611,10 +649,10 @@ public void VerifyLockCountsAfterReadCacheEvict() } AssertTotalLockCounts(xlocks, slocks); - foreach (var idx in LockableUnsafeContextTests.EnumActionKeyIndices(keys, LockableUnsafeContextTests.LockOperationType.Unlock)) + foreach (var idx in TransactionalUnsafeContextTestUtils.EnumActionKeyIndices(keys, LockOperationType.Unlock)) { ref var key = ref keys[idx]; - HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(ref key.Key)); + HashEntryInfo hei = new(store.storeFunctions.GetKeyHashCode64(key.Key)); OverflowBucketLockTableTests.PopulateHei(store, ref hei); var lockState = store.LockTable.GetLockState(ref hei); @@ -622,7 +660,7 @@ public void VerifyLockCountsAfterReadCacheEvict() ClassicAssert.AreEqual(key.LockType == LockType.Exclusive, lockState.IsLockedExclusive); ClassicAssert.AreEqual(key.LockType != LockType.Exclusive, lockState.NumLockedShared > 0); - luContext.Unlock>(keys.AsSpan().Slice(idx, 1)); + luContext.Unlock(keys.AsSpan().Slice(idx, 1)); lockState = store.LockTable.GetLockState(ref hei); ClassicAssert.IsFalse(lockState.IsLockedExclusive); ClassicAssert.AreEqual(0, lockState.NumLockedShared); @@ -636,21 +674,19 @@ public void VerifyLockCountsAfterReadCacheEvict() } finally { - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); } AssertTotalLockCounts(0, 0); } } - - [AllureNUnit] [TestFixture] - class LongStressChainTests : AllureTestBase + public class LongStressChainTests : TestBase { - private TsavoriteKV store; + private TsavoriteKV store; private IDevice log; - private LongComparerModulo comparer; + private LongKeyComparerModulo comparer; const long ValueAdd = 1_000_000_000; const long NumKeys = 2_000; @@ -681,19 +717,19 @@ public void Setup() } } - comparer = new LongComparerModulo((long)modRange); + comparer = new LongKeyComparerModulo((long)modRange); // Make the main log small enough that we force the readcache store = new(new() { IndexSize = 1L << 26, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, ReadCacheMemorySize = 1L << 15, ReadCachePageSize = 1L << 9, ReadCacheEnabled = true - }, StoreFunctions.Create(comparer) + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -708,38 +744,10 @@ public void TearDown() OnTearDown(); } - internal class RmwLongFunctions : SimpleSessionFunctions + internal class RmwLongFunctions : SimpleLongSimpleFunctions { /// - public override bool ConcurrentWriter(ref long key, ref long input, ref long src, ref long dst, ref long output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) - { - dst = output = src; - return true; - } - - /// - public override bool SingleWriter(ref long key, ref long input, ref long src, ref long dst, ref long output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) - { - dst = output = src; - return true; - } - - /// - public override bool CopyUpdater(ref long key, ref long input, ref long oldValue, ref long newValue, ref long output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - newValue = output = input; - return true; - } - - /// - public override bool InPlaceUpdater(ref long key, ref long input, ref long value, ref long output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - value = output = input; - return true; - } - - /// - public override bool InitialUpdater(ref long key, ref long input, ref long value, ref long output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref long input, ref long output, ref RMWInfo rmwInfo) { Assert.Fail("For these tests, InitialUpdater should never be called"); return false; @@ -748,13 +756,13 @@ public override bool InitialUpdater(ref long key, ref long input, ref long value unsafe void PopulateAndEvict() { - using var session = store.NewSession>(new SimpleSessionFunctions()); + using var session = store.NewSession(new RmwLongFunctions()); var bContext = session.BasicContext; for (long ii = 0; ii < NumKeys; ii++) { long key = ii; - var status = bContext.Upsert(ref key, ref key); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(SpanByte.FromPinnedVariable(ref key)), SpanByte.FromPinnedVariable(ref key)); ClassicAssert.IsFalse(status.IsPending); ClassicAssert.IsTrue(status.Record.Created, $"key {key}, status {status}"); } @@ -766,25 +774,25 @@ unsafe void PopulateAndEvict() [Category(TsavoriteKVTestCategory)] [Category(ReadCacheTestCategory)] [Category(StressTestCategory)] - //[Repeat(300)] #pragma warning disable IDE0060 // Remove unused parameter (modRange is used by Setup()) - public void LongRcMultiThreadTest([Values] HashModulo modRange, [Values(0, 1, 2, 8)] int numReadThreads, [Values(0, 1, 2, 8)] int numWriteThreads, + public void LongRcMultiThreadTest([Values] HashModulo modRange, [Values(0, 1, 2)] int numReadThreads, [Values(0, 1, 2)] int numWriteThreads, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) #pragma warning restore IDE0060 // Remove unused parameter + => LongRcMultiThreadWorker(numReadThreads, numWriteThreads, updateOp); + + internal void LongRcMultiThreadWorker(int numReadThreads, int numWriteThreads, UpdateOp updateOp) { if (numReadThreads == 0 && numWriteThreads == 0) Assert.Ignore("Skipped due to 0 threads for both read and update"); if ((numReadThreads > 2 || numWriteThreads > 2) && IsRunningAzureTests) Assert.Ignore("Skipped because > 2 threads when IsRunningAzureTests"); - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); PopulateAndEvict(); const int numIterations = 1; unsafe void runReadThread(int tid) { - using var session = store.NewSession>(new SimpleSessionFunctions()); + using var session = store.NewSession(new RmwLongFunctions()); var bContext = session.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) @@ -793,7 +801,7 @@ unsafe void runReadThread(int tid) for (var ii = 0; ii < NumKeys; ++ii) { long key = ii, output = 0; - var status = bContext.Read(ref key, ref output); + var status = bContext.Read(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key)), ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -816,8 +824,8 @@ unsafe void runReadThread(int tid) status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; - key = completedOutputs.Current.Key; - ClassicAssert.AreEqual(completedOutputs.Current.RecordMetadata.Address == Constants.kInvalidAddress, status.Record.CopiedToReadCache, $"key {key}: {status}"); + key = completedOutputs.Current.Key.KeyBytes.AsRef(); + ClassicAssert.IsTrue(status.Found, $"key {key}, status {status}, wasPending {true}"); ClassicAssert.AreEqual(key, output % ValueAdd); } @@ -830,7 +838,7 @@ unsafe void runReadThread(int tid) unsafe void runUpdateThread(int tid) { - using var session = store.NewSession(new RmwLongFunctions()); + using var session = store.NewSession(new RmwLongFunctions()); var bContext = session.BasicContext; for (var iteration = 0; iteration < numIterations; ++iteration) @@ -840,8 +848,8 @@ unsafe void runUpdateThread(int tid) { long key = ii, input = ii + ValueAdd * tid, output = 0; var status = updateOp == UpdateOp.RMW - ? bContext.RMW(ref key, ref input, ref output) - : bContext.Upsert(ref key, ref input, ref input, ref output); + ? bContext.RMW(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key)), ref input, ref output) + : bContext.Upsert(TestSpanByteKey.CopySpan(SpanByte.FromPinnedVariable(ref key)), ref input, SpanByte.FromPinnedVariable(ref input), ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -866,8 +874,8 @@ unsafe void runUpdateThread(int tid) { ++numCompleted; if (updateOp == UpdateOp.RMW) // Upsert will not try to find records below HeadAddress, but it may find them in-memory - ClassicAssert.IsTrue(completedOutputs.Current.Status.Found, $"key {completedOutputs.Current.Key}, status {completedOutputs.Current.Status}, wasPending {true}"); - ClassicAssert.AreEqual(completedOutputs.Current.Key + ValueAdd * tid, completedOutputs.Current.Output); + ClassicAssert.IsTrue(completedOutputs.Current.Status.Found, $"key {completedOutputs.Current.Key.KeyBytes.ToShortString()}, status {completedOutputs.Current.Status}, wasPending {true}"); + ClassicAssert.AreEqual(completedOutputs.Current.Key.KeyBytes.AsRef() + ValueAdd * tid, completedOutputs.Current.Output); } } } @@ -889,14 +897,12 @@ unsafe void runUpdateThread(int tid) Task.WaitAll([.. tasks]); } } - - [AllureNUnit] [TestFixture] - class SpanByteStressChainTests : AllureTestBase + class SpanByteStressChainTests : TestBase { - private TsavoriteKV> store; + private TsavoriteKV> store; private IDevice log; - SpanByteComparerModulo comparer; + SpanByteKeyComparerModulo comparer; const long ValueAdd = 1_000_000_000; @@ -928,19 +934,19 @@ public void Setup() } } - comparer = new SpanByteComparerModulo(modRange); + comparer = new SpanByteKeyComparerModulo(modRange); // Make the main log small enough that we force the readcache store = new(new() { IndexSize = 1L << 20, LogDevice = log, - MemorySize = 1L << 15, + LogMemorySize = 1L << 15, PageSize = 1L << 10, ReadCacheMemorySize = 1L << 15, ReadCachePageSize = 1L << 9, ReadCacheEnabled = true - }, StoreFunctions.Create(comparer, SpanByteRecordDisposer.Instance) + }, StoreFunctions.Create(comparer, SpanByteRecordTriggers.Instance) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions) ); } @@ -958,40 +964,45 @@ public void TearDown() internal class RmwSpanByteFunctions : SpanByteFunctions { /// - public override bool ConcurrentWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, ref RecordInfo recordInfo) + public override bool InPlaceWriter(ref LogRecord logRecord, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - src.CopyTo(ref dst); - src.CopyTo(ref output, memoryPool); + if (!base.InPlaceWriter(ref logRecord, ref input, srcValue, ref output, ref upsertInfo)) + return false; + srcValue.CopyTo(ref output, memoryPool); return true; } /// - public override bool SingleWriter(ref SpanByte key, ref SpanByte input, ref SpanByte src, ref SpanByte dst, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo, WriteReason reason, ref RecordInfo recordInfo) + public override bool InitialWriter(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ReadOnlySpan srcValue, ref SpanByteAndMemory output, ref UpsertInfo upsertInfo) { - src.CopyTo(ref dst); - src.CopyTo(ref output, memoryPool); + if (!base.InitialWriter(ref dstLogRecord, in sizeInfo, ref input, srcValue, ref output, ref upsertInfo)) + return false; + srcValue.CopyTo(ref output, memoryPool); return true; } /// - public override bool CopyUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte oldValue, ref SpanByte newValue, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - input.CopyTo(ref newValue); + if (!dstLogRecord.TrySetValueSpanAndPrepareOptionals(input, in sizeInfo)) + return false; input.CopyTo(ref output, memoryPool); return true; } /// - public override bool InPlaceUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InPlaceUpdater(ref LogRecord logRecord, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { - // The default implementation of IPU simply writes input to destination, if there is space - base.InPlaceUpdater(ref key, ref input, ref value, ref output, ref rmwInfo, ref recordInfo); + var sizeInfo = new RecordSizeInfo() { FieldInfo = GetRMWModifiedFieldInfo(logRecord, ref input) }; + logRecord.PopulateRecordSizeInfoForIPU(ref sizeInfo); + if (!logRecord.TrySetValueSpanAndPrepareOptionals(input, in sizeInfo)) + return false; input.CopyTo(ref output, memoryPool); return true; } /// - public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref SpanByte value, ref SpanByteAndMemory output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) + public override bool InitialUpdater(ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref PinnedSpanByte input, ref SpanByteAndMemory output, ref RMWInfo rmwInfo) { Assert.Fail("For these tests, InitialUpdater should never be called"); return false; @@ -1000,16 +1011,15 @@ public override bool InitialUpdater(ref SpanByte key, ref SpanByte input, ref Sp unsafe void PopulateAndEvict() { - using var session = store.NewSession>(new SpanByteFunctions()); + using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[sizeof(long)]; for (long ii = 0; ii < NumKeys; ii++) { - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - var status = bContext.Upsert(ref key, ref key); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + var status = bContext.Upsert(TestSpanByteKey.FromPinnedSpan(key), key); ClassicAssert.IsTrue(status.Record.Created, status.ToString()); } bContext.CompletePending(true); @@ -1020,27 +1030,26 @@ unsafe void PopulateAndEvict() [Category(TsavoriteKVTestCategory)] [Category(ReadCacheTestCategory)] [Category(StressTestCategory)] - //[Repeat(300)] - public void SpanByteRcMultiThreadTest([Values] HashModulo modRange, [Values(0, 1, 2, 8)] int numReadThreads, [Values(0, 1, 2, 8)] int numWriteThreads, + public void SpanByteRcMultiThreadTest([Values] HashModulo modRange, [Values(0, 1, 2)] int numReadThreads, [Values(0, 1, 2)] int numWriteThreads, [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) + => SpanByteRcMultiThreadWorker(numReadThreads, numWriteThreads, updateOp); + + internal void SpanByteRcMultiThreadWorker(int numReadThreads, int numWriteThreads, UpdateOp updateOp) { if (numReadThreads == 0 && numWriteThreads == 0) Assert.Ignore("Skipped due to 0 threads for both read and update"); if ((numReadThreads > 2 || numWriteThreads > 2) && IsRunningAzureTests) Assert.Ignore("Skipped because > 2 threads when IsRunningAzureTests"); - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1} ***"); PopulateAndEvict(); const int numIterations = 1; unsafe void runReadThread(int tid) { - using var session = store.NewSession>(new SpanByteFunctions()); + using var session = store.NewSession>(new SpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); + Span key = stackalloc byte[sizeof(long)]; for (var iteration = 0; iteration < numIterations; ++iteration) { @@ -1049,8 +1058,8 @@ unsafe void runReadThread(int tid) { SpanByteAndMemory output = default; - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - var status = bContext.Read(ref key, ref output); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + var status = bContext.Read(TestSpanByteKey.CopySpan(key), ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -1061,7 +1070,7 @@ unsafe void runReadThread(int tid) ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {ii}, {status}, wasPending {false}, pt 1"); ClassicAssert.IsNotNull(output.Memory, $"tid {tid}, key {ii}, wasPending {false}, pt 2"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.ReadOnlySpan); ClassicAssert.AreEqual(ii, value % ValueAdd, $"tid {tid}, key {ii}, wasPending {false}, pt 3"); output.Memory.Dispose(); } @@ -1078,14 +1087,12 @@ unsafe void runReadThread(int tid) status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; // Note: do NOT overwrite 'key' here - long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.AsReadOnlySpan()); - - ClassicAssert.AreEqual(completedOutputs.Current.RecordMetadata.Address == Constants.kInvalidAddress, status.Record.CopiedToReadCache, $"key {keyLong}: {status}"); + long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.KeyBytes); - ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {keyLong}, {status}, wasPending {true}, pt 1"); - ClassicAssert.IsNotNull(output.Memory, $"tid {tid}, key {keyLong}, wasPending {true}, pt 2"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); - ClassicAssert.AreEqual(keyLong, value % ValueAdd, $"tid {tid}, key {keyLong}, wasPending {true}, pt 3"); + ClassicAssert.IsTrue(status.Found, $"pending: tid {tid}, key {keyLong}, {status}, wasPending {true}, pt 1"); + ClassicAssert.IsNotNull(output.Memory, $"pending: tid {tid}, key {keyLong}, wasPending {true}, pt 2"); + long value = BitConverter.ToInt64(output.ReadOnlySpan); + ClassicAssert.AreEqual(keyLong, value % ValueAdd, $"pending: tid {tid}, key {keyLong}, wasPending {true}, pt 3"); output.Memory.Dispose(); } } @@ -1097,13 +1104,12 @@ unsafe void runReadThread(int tid) unsafe void runUpdateThread(int tid) { - using var session = store.NewSession>(new RmwSpanByteFunctions()); + using var session = store.NewSession>(new RmwSpanByteFunctions()); var bContext = session.BasicContext; - Span keyVec = stackalloc byte[sizeof(long)]; - var key = SpanByte.FromPinnedSpan(keyVec); - Span inputVec = stackalloc byte[sizeof(long)]; - var input = SpanByte.FromPinnedSpan(inputVec); + Span key = stackalloc byte[sizeof(long)]; + Span input = stackalloc byte[sizeof(long)]; + var pinnedInputSpan = PinnedSpanByte.FromPinnedSpan(input); for (var iteration = 0; iteration < numIterations; ++iteration) { @@ -1112,11 +1118,11 @@ unsafe void runUpdateThread(int tid) { SpanByteAndMemory output = default; - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(keyVec, ii)); - ClassicAssert.IsTrue(BitConverter.TryWriteBytes(inputVec, ii + ValueAdd)); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(key, ii)); + ClassicAssert.IsTrue(BitConverter.TryWriteBytes(input, ii + ValueAdd)); var status = updateOp == UpdateOp.RMW - ? bContext.RMW(ref key, ref input, ref output) - : bContext.Upsert(ref key, ref input, ref input, ref output); + ? bContext.RMW(TestSpanByteKey.CopySpan(key), ref pinnedInputSpan, ref output) + : bContext.Upsert(TestSpanByteKey.CopySpan(key), ref pinnedInputSpan, input, ref output); var numPending = ii - numCompleted; if (status.IsPending) @@ -1130,10 +1136,10 @@ unsafe void runUpdateThread(int tid) if (updateOp == UpdateOp.RMW) // Upsert will not try to find records below HeadAddress, but it may find them in-memory ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {ii}, {status}"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.ReadOnlySpan); ClassicAssert.AreEqual(ii + ValueAdd, value, $"tid {tid}, key {ii}, wasPending {false}"); - output.Memory?.Dispose(); + output.Dispose(); } if (numPending > 0 && ((numPending % RcTestGlobals.PendingMod == 0) || ii == NumKeys - 1)) @@ -1148,15 +1154,15 @@ unsafe void runUpdateThread(int tid) status = completedOutputs.Current.Status; output = completedOutputs.Current.Output; // Note: do NOT overwrite 'key' here - long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.AsReadOnlySpan()); + long keyLong = BitConverter.ToInt64(completedOutputs.Current.Key.KeyBytes); if (updateOp == UpdateOp.RMW) // Upsert will not try to find records below HeadAddress, but it may find them in-memory ClassicAssert.IsTrue(status.Found, $"tid {tid}, key {keyLong}, {status}"); - long value = BitConverter.ToInt64(output.AsReadOnlySpan()); + long value = BitConverter.ToInt64(output.ReadOnlySpan); ClassicAssert.AreEqual(keyLong + ValueAdd, value, $"tid {tid}, key {keyLong}, wasPending {true}"); - output.Memory?.Dispose(); + output.Dispose(); } } } diff --git a/libs/storage/Tsavorite/cs/test/test.session/Tsavorite.test.session.csproj b/libs/storage/Tsavorite/cs/test/test.session/Tsavorite.test.session.csproj new file mode 100644 index 00000000000..ff6e0695c0b --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.session/Tsavorite.test.session.csproj @@ -0,0 +1,36 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + false + + + + + + + diff --git a/libs/storage/Tsavorite/cs/test/test.stress/ReadCacheStressTests.cs b/libs/storage/Tsavorite/cs/test/test.stress/ReadCacheStressTests.cs new file mode 100644 index 00000000000..5efbc154eee --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.stress/ReadCacheStressTests.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test; +using NUnit.Framework; +using static Tsavorite.test.TestUtils; + +namespace Tsavorite.test.stress +{ + [TestFixture] + class LongStressChainTests : TestBase + { + private readonly ReadCacheTests.LongStressChainTests worker = new(); + + [SetUp] + public void Setup() => worker.Setup(); + + [TearDown] + public void TearDown() => worker.TearDown(); + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(ReadCacheTestCategory)] + [Category(StressTestCategory)] +#pragma warning disable IDE0060 // Remove unused parameter (modRange is used by worker.Setup()) + public void LongRcMultiThreadStressTest([Values] HashModulo modRange, [Values(1, 8)] int numReadThreads, [Values(1, 8)] int numWriteThreads, + [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) +#pragma warning restore IDE0060 // Remove unused parameter + => worker.LongRcMultiThreadWorker(numReadThreads, numWriteThreads, updateOp); + } + + [TestFixture] + class SpanByteStressChainTests : TestBase + { + private readonly ReadCacheTests.SpanByteStressChainTests worker = new(); + + [SetUp] + public void Setup() => worker.Setup(); + + [TearDown] + public void TearDown() => worker.TearDown(); + + [Test] + [Category(TsavoriteKVTestCategory)] + [Category(ReadCacheTestCategory)] + [Category(StressTestCategory)] +#pragma warning disable IDE0060 // Remove unused parameter (modRange is used by worker.Setup()) + public void SpanByteRcMultiThreadStressTest([Values] HashModulo modRange, [Values(1, 8)] int numReadThreads, [Values(1, 8)] int numWriteThreads, + [Values(UpdateOp.Upsert, UpdateOp.RMW)] UpdateOp updateOp) +#pragma warning restore IDE0060 // Remove unused parameter + => worker.SpanByteRcMultiThreadWorker(numReadThreads, numWriteThreads, updateOp); + } +} \ No newline at end of file diff --git a/libs/storage/Tsavorite/cs/test/test.stress/Tsavorite.test.stress.csproj b/libs/storage/Tsavorite/cs/test/test.stress/Tsavorite.test.stress.csproj new file mode 100644 index 00000000000..b749916d7c1 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/test.stress/Tsavorite.test.stress.csproj @@ -0,0 +1,33 @@ + + + + true + ../../../../../../Garnet.snk + false + + + + 1701;1702;1591;IDE0130;IDE0065;IDE0007;IDE0048 + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + false + + + diff --git a/libs/storage/Tsavorite/cs/test/tsavorite.runsettings b/libs/storage/Tsavorite/cs/test/tsavorite.runsettings new file mode 100644 index 00000000000..c8703b53003 --- /dev/null +++ b/libs/storage/Tsavorite/cs/test/tsavorite.runsettings @@ -0,0 +1,7 @@ + + + + + 0 + + diff --git a/main/GarnetServer/Extensions/DeleteIfMatch.cs b/main/GarnetServer/Extensions/DeleteIfMatch.cs index 537d7de5505..63c90fb88a6 100644 --- a/main/GarnetServer/Extensions/DeleteIfMatch.cs +++ b/main/GarnetServer/Extensions/DeleteIfMatch.cs @@ -20,20 +20,20 @@ namespace Garnet sealed class DeleteIfMatchCustomCommand : CustomRawStringFunctions { /// - public override bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new InvalidOperationException(); /// - public override bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref RespMemoryWriter writer) + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) => false; /// - public override int GetInitialLength(ref RawStringInput input) + public override int GetInitialLength(ref StringInput input) => throw new InvalidOperationException(); /// - public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new InvalidOperationException(); /// - public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { var expectedVal = GetFirstArg(ref input); if (value.SequenceEqual(expectedVal)) @@ -46,18 +46,18 @@ public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput i } /// - public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) + public override bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) { var expectedVal = GetFirstArg(ref input); return oldValue.SequenceEqual(expectedVal); } /// - public override int GetLength(ReadOnlySpan value, ref RawStringInput input) + public override int GetLength(ReadOnlySpan value, ref StringInput input) => value.Length; /// - public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { rmwInfo.Action = RMWAction.ExpireAndStop; Debug.Assert(oldValue.Length == newValue.Length); diff --git a/main/GarnetServer/Extensions/GetTwoKeysNoTxn.cs b/main/GarnetServer/Extensions/GetTwoKeysNoTxn.cs index 3eb64518f92..f0b88f0f314 100644 --- a/main/GarnetServer/Extensions/GetTwoKeysNoTxn.cs +++ b/main/GarnetServer/Extensions/GetTwoKeysNoTxn.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet { @@ -37,8 +38,8 @@ public override void Finalize(TGarnetApi api, ref CustomProcedureInp var key1 = GetNextArg(ref procInput, ref offset); var key2 = GetNextArg(ref procInput, ref offset); - api.GET(key1, out var value1); - api.GET(key2, out var value2); + api.GET(key1, out PinnedSpanByte value1); + api.GET(key2, out PinnedSpanByte value2); // Return the two keys as an array of bulk strings WriteBulkStringArray(ref output, value1, value2); diff --git a/main/GarnetServer/Extensions/MGetIfPM.cs b/main/GarnetServer/Extensions/MGetIfPM.cs index e0321376609..2e34e205c07 100644 --- a/main/GarnetServer/Extensions/MGetIfPM.cs +++ b/main/GarnetServer/Extensions/MGetIfPM.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet { @@ -39,11 +40,11 @@ public override void Finalize(TGarnetApi api, ref CustomProcedureInp var prefix = GetNextArg(ref procInput, ref offset); // Read key, check condition, add to output - ArgSlice key; - List values = []; + PinnedSpanByte key; + List values = []; while ((key = GetNextArg(ref procInput, ref offset)).Length > 0) { - if (api.GET(key, out var value) == GarnetStatus.OK) + if (api.GET(key, out PinnedSpanByte value) == GarnetStatus.OK) { if (value.ReadOnlySpan.StartsWith(prefix.ReadOnlySpan)) { diff --git a/main/GarnetServer/Extensions/MSetPx.cs b/main/GarnetServer/Extensions/MSetPx.cs index 7886a437fcf..d547c494fb8 100644 --- a/main/GarnetServer/Extensions/MSetPx.cs +++ b/main/GarnetServer/Extensions/MSetPx.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet { @@ -38,7 +39,7 @@ public override void Finalize(TGarnetApi api, ref CustomProcedureInp var expiryMs = GetNextArg(ref procInput, ref offset); // Read and set key-value pairs with expiry - ArgSlice key, value; + PinnedSpanByte key, value; while ((key = GetNextArg(ref procInput, ref offset)).Length > 0) { value = GetNextArg(ref procInput, ref offset); diff --git a/main/GarnetServer/Extensions/MyDictGet.cs b/main/GarnetServer/Extensions/MyDictGet.cs index dd738d3499a..b91e2a76785 100644 --- a/main/GarnetServer/Extensions/MyDictGet.cs +++ b/main/GarnetServer/Extensions/MyDictGet.cs @@ -10,7 +10,7 @@ namespace Garnet { public class MyDictGet : CustomObjectFunctions { - public override bool Reader(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + public override bool Reader(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) { Debug.Assert(value is MyDict); diff --git a/main/GarnetServer/Extensions/MyDictObject.cs b/main/GarnetServer/Extensions/MyDictObject.cs index 44cd15450ef..4fd0ccfa7ce 100644 --- a/main/GarnetServer/Extensions/MyDictObject.cs +++ b/main/GarnetServer/Extensions/MyDictObject.cs @@ -22,7 +22,7 @@ class MyDict : CustomObjectBase readonly Dictionary dict; public MyDict(byte type) - : base(type, 0, MemoryUtils.DictionaryOverhead) + : base(type, MemoryUtils.DictionaryOverhead) { dict = new(ByteArrayComparer.Instance); } @@ -32,8 +32,8 @@ public MyDict(byte type, BinaryReader reader) { dict = new(ByteArrayComparer.Instance); - int count = reader.ReadInt32(); - for (int i = 0; i < count; i++) + var count = reader.ReadInt32(); + for (var i = 0; i < count; i++) { var key = reader.ReadBytes(reader.ReadInt32()); var value = reader.ReadBytes(reader.ReadInt32()); @@ -69,18 +69,12 @@ public override void Dispose() { } /// Returns the items from this object using a cursor to indicate the start of the scan, /// a pattern to filter out the items to return, and a count to indicate the number of items to return. /// - /// - /// - /// - /// - /// - /// /// public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = null, int patternLength = 0, bool isNoValue = false) { cursor = start; - items = new(); - int index = 0; + items = []; + var index = 0; if (dict.Count < start) { @@ -96,7 +90,7 @@ public override unsafe void Scan(long start, out List items, out long cu continue; } - bool addToList = false; + var addToList = false; if (patternLength == 0) { items.Add(item.Key); @@ -132,9 +126,7 @@ public override unsafe void Scan(long start, out List items, out long cu public bool Set(byte[] key, byte[] value) { if (dict.TryGetValue(key, out var oldValue)) - { UpdateSize(key, oldValue, false); - } dict[key] = value; UpdateSize(key, value); @@ -143,15 +135,19 @@ public bool Set(byte[] key, byte[] value) private void UpdateSize(byte[] key, byte[] value, bool add = true) { - var size = Utility.RoundUp(key.Length, IntPtr.Size) + Utility.RoundUp(value.Length, IntPtr.Size) + var memorySize = Utility.RoundUp(key.Length, IntPtr.Size) + Utility.RoundUp(value.Length, IntPtr.Size) + (2 * MemoryUtils.ByteArrayOverhead) + MemoryUtils.DictionaryEntryOverhead; - this.Size += add ? size : -size; - Debug.Assert(this.Size >= MemoryUtils.DictionaryOverhead); + + if (add) + HeapMemorySize += memorySize; + else + { + HeapMemorySize -= memorySize; + Debug.Assert(HeapMemorySize >= MemoryUtils.DictionaryOverhead); + } } public bool TryGetValue(byte[] key, [MaybeNullWhen(false)] out byte[] value) - { - return dict.TryGetValue(key, out value); - } + => dict.TryGetValue(key, out value); } } \ No newline at end of file diff --git a/main/GarnetServer/Extensions/MyDictSet.cs b/main/GarnetServer/Extensions/MyDictSet.cs index f42cc51cd3b..e5ade27609b 100644 --- a/main/GarnetServer/Extensions/MyDictSet.cs +++ b/main/GarnetServer/Extensions/MyDictSet.cs @@ -10,9 +10,9 @@ namespace Garnet { public class MyDictSet : CustomObjectFunctions { - public override bool NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput input, ref RespMemoryWriter writer) => true; + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) => true; - public override bool Updater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool Updater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { Debug.Assert(value is MyDict); diff --git a/main/GarnetServer/Extensions/ReadWriteTxn.cs b/main/GarnetServer/Extensions/ReadWriteTxn.cs index 57fbeae7ed4..445a9ce4412 100644 --- a/main/GarnetServer/Extensions/ReadWriteTxn.cs +++ b/main/GarnetServer/Extensions/ReadWriteTxn.cs @@ -21,11 +21,11 @@ sealed class ReadWriteTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - api.GET(GetNextArg(ref procInput, ref offset), out var key1); + api.GET(GetNextArg(ref procInput, ref offset), out PinnedSpanByte key1); if (key1.ReadOnlySpan.SequenceEqual("wrong_string"u8)) return false; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); return true; } @@ -36,7 +36,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var key2 = GetNextArg(ref procInput, ref offset); var key3 = GetNextArg(ref procInput, ref offset); - var status = api.GET(key1, out var result); + var status = api.GET(key1, out PinnedSpanByte result); if (status == GarnetStatus.OK) { api.SET(key2, result); diff --git a/main/GarnetServer/Extensions/SampleDeleteTxn.cs b/main/GarnetServer/Extensions/SampleDeleteTxn.cs index 01866fdd820..0ec6a07a19b 100644 --- a/main/GarnetServer/Extensions/SampleDeleteTxn.cs +++ b/main/GarnetServer/Extensions/SampleDeleteTxn.cs @@ -32,21 +32,17 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var offset = 0; var mainStoreKey = GetNextArg(ref procInput, ref offset); - AddKey(mainStoreKey, LockType.Exclusive, false); + AddKey(mainStoreKey, LockType.Exclusive, StoreType.Main); var sortedSet1Key = GetNextArg(ref procInput, ref offset); if (sortedSet1Key.Length > 0) - { - AddKey(sortedSet1Key, LockType.Exclusive, true); - } - - GetNextArg(ref procInput, ref offset); // sortedSet1Entry + AddKey(sortedSet1Key, LockType.Exclusive, StoreType.Object); + GetNextArg(ref procInput, ref offset); // sortedSet1Entry must be retrieved but is not used var sortedSet2Key = GetNextArg(ref procInput, ref offset); if (sortedSet2Key.Length > 0) - { - AddKey(sortedSet2Key, LockType.Exclusive, true); - } + AddKey(sortedSet2Key, LockType.Exclusive, StoreType.Object); + // sortedSet2Entry is not used return true; } @@ -55,25 +51,18 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p { var offset = 0; - var mainStoreKey = GetNextArg(ref procInput, ref offset); - - api.DELETE(mainStoreKey, StoreType.Main); + var stringKey = GetNextArg(ref procInput, ref offset); + api.DELETE(stringKey); var sortedSet1Key = GetNextArg(ref procInput, ref offset); var sortedSet1Entry = GetNextArg(ref procInput, ref offset); - if (sortedSet1Key.Length > 0) - { api.SortedSetRemove(sortedSet1Key, sortedSet1Entry, out _); - } var sortedSet2Key = GetNextArg(ref procInput, ref offset); var sortedSet2Entry = GetNextArg(ref procInput, ref offset); - if (sortedSet2Key.Length > 0) - { api.SortedSetRemove(sortedSet2Key, sortedSet2Entry, out _); - } WriteSimpleString(ref output, "SUCCESS"); } diff --git a/main/GarnetServer/Extensions/SampleUpdateTxn.cs b/main/GarnetServer/Extensions/SampleUpdateTxn.cs index ac132ef507b..d22557ed4ba 100644 --- a/main/GarnetServer/Extensions/SampleUpdateTxn.cs +++ b/main/GarnetServer/Extensions/SampleUpdateTxn.cs @@ -32,24 +32,19 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var offset = 0; var mainStoreKey = GetNextArg(ref procInput, ref offset); - GetNextArg(ref procInput, ref offset); // mainStoreValue - - AddKey(mainStoreKey, LockType.Exclusive, false); + GetNextArg(ref procInput, ref offset); // mainStoreValue must be retrieved but is not used + AddKey(mainStoreKey, LockType.Exclusive, StoreType.Main); var sortedSet1Key = GetNextArg(ref procInput, ref offset); if (sortedSet1Key.Length > 0) - { - AddKey(sortedSet1Key, LockType.Exclusive, true); - } - - GetNextArg(ref procInput, ref offset); // sortedSet1Entry - GetNextArg(ref procInput, ref offset); // sortedSetScore + AddKey(sortedSet1Key, LockType.Exclusive, StoreType.Object); + GetNextArg(ref procInput, ref offset); // sortedSet1Entry must be retrieved but is not used + GetNextArg(ref procInput, ref offset); // sortedSet1Score must be retrieved but is not used var sortedSet2Key = GetNextArg(ref procInput, ref offset); if (sortedSet2Key.Length > 0) - { - AddKey(sortedSet2Key, LockType.Exclusive, true); - } + AddKey(sortedSet2Key, LockType.Exclusive, StoreType.Object); + // sortedSet2Entry and sortedSet2Score are not used return true; } @@ -60,27 +55,19 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var mainStoreKey = GetNextArg(ref procInput, ref offset); var mainStoreValue = GetNextArg(ref procInput, ref offset); - api.SET(mainStoreKey, mainStoreValue); var sortedSet1Key = GetNextArg(ref procInput, ref offset); var sortedSet1Entry = GetNextArg(ref procInput, ref offset); var sortedSet1EntryScore = GetNextArg(ref procInput, ref offset); - - if (sortedSet1Key.Length > 0) - { api.SortedSetAdd(sortedSet1Key, sortedSet1EntryScore, sortedSet1Entry, out _); - } var sortedSet2Key = GetNextArg(ref procInput, ref offset); var sortedSet2Entry = GetNextArg(ref procInput, ref offset); var sortedSet2EntryScore = GetNextArg(ref procInput, ref offset); - if (sortedSet2Key.Length > 0) - { api.SortedSetAdd(sortedSet2Key, sortedSet2EntryScore, sortedSet2Entry, out _); - } WriteSimpleString(ref output, "SUCCESS"); } diff --git a/main/GarnetServer/Extensions/SetIfPM.cs b/main/GarnetServer/Extensions/SetIfPM.cs index 019388b477a..ebb4f709a8a 100644 --- a/main/GarnetServer/Extensions/SetIfPM.cs +++ b/main/GarnetServer/Extensions/SetIfPM.cs @@ -20,20 +20,20 @@ namespace Garnet sealed class SetIfPMCustomCommand : CustomRawStringFunctions { /// - public override bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new InvalidOperationException(); /// - public override bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref RespMemoryWriter writer) + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) => false; /// - public override int GetInitialLength(ref RawStringInput input) + public override int GetInitialLength(ref StringInput input) => throw new InvalidOperationException(); /// - public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new InvalidOperationException(); /// - public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { var offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -49,7 +49,7 @@ public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput i } /// - public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) + public override bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) { var offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -58,11 +58,11 @@ public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput i } /// - public override int GetLength(ReadOnlySpan value, ref RawStringInput input) + public override int GetLength(ReadOnlySpan value, ref StringInput input) => GetFirstArg(ref input).Length; /// - public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { var newVal = GetFirstArg(ref input); Debug.Assert(newVal.Length == newValue.Length); diff --git a/main/GarnetServer/Extensions/SetWPIfPGT.cs b/main/GarnetServer/Extensions/SetWPIfPGT.cs index fa172c5b8c0..ae837841d55 100644 --- a/main/GarnetServer/Extensions/SetWPIfPGT.cs +++ b/main/GarnetServer/Extensions/SetWPIfPGT.cs @@ -20,21 +20,21 @@ sealed class SetWPIFPGTCustomCommand : CustomRawStringFunctions public const string PrefixError = "Invalid prefix length, should be 8 bytes"; /// - public override int GetInitialLength(ref RawStringInput input) + public override int GetInitialLength(ref StringInput input) { var newVal = GetFirstArg(ref input); return newVal.Length + 8; } /// - public override int GetLength(ReadOnlySpan value, ref RawStringInput input) + public override int GetLength(ReadOnlySpan value, ref StringInput input) { var newVal = GetFirstArg(ref input); return newVal.Length + 8; } /// - public override bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref RespMemoryWriter writer) + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) { int offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -48,7 +48,7 @@ public override bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInpu } /// - public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) + public override bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) { int offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -62,7 +62,7 @@ public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput i } /// - public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { int offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -75,7 +75,7 @@ public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput i } /// - public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { int offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -105,7 +105,7 @@ public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput i } /// - public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { int offset = 0; var newVal = GetNextArg(ref input, ref offset); @@ -118,7 +118,7 @@ public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput inpu } /// - public override bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new InvalidOperationException(); } } \ No newline at end of file diff --git a/main/GarnetServer/Extensions/Sum.cs b/main/GarnetServer/Extensions/Sum.cs index 1fdf67ba2e0..46baf607db7 100644 --- a/main/GarnetServer/Extensions/Sum.cs +++ b/main/GarnetServer/Extensions/Sum.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet { @@ -12,11 +13,11 @@ public override bool Execute(TGarnetApi garnetApi, ref CustomProcedu { var offset = 0; var sum = 0; - ArgSlice key; + PinnedSpanByte key; while ((key = GetNextArg(ref procInput, ref offset)).Length > 0) { - if (garnetApi.GET(key, out var value) == GarnetStatus.OK) + if (garnetApi.GET(key, out PinnedSpanByte value) == GarnetStatus.OK) { // Sum the values if (int.TryParse(value.ToString(), out var intValue)) diff --git a/main/GarnetServer/GarnetServer.csproj b/main/GarnetServer/GarnetServer.csproj index 5d537a73192..56c7d3046bb 100644 --- a/main/GarnetServer/GarnetServer.csproj +++ b/main/GarnetServer/GarnetServer.csproj @@ -19,6 +19,11 @@ + + + + + PreserveNewest diff --git a/metrics/HdrHistogram/Utilities/WriterReaderPhaser.cs b/metrics/HdrHistogram/Utilities/WriterReaderPhaser.cs index ac5f65a2bd8..fc2d182e03c 100644 --- a/metrics/HdrHistogram/Utilities/WriterReaderPhaser.cs +++ b/metrics/HdrHistogram/Utilities/WriterReaderPhaser.cs @@ -29,12 +29,12 @@ namespace HdrHistogram.Utilities /// "writers" are wait free, "readers" block for other "readers", and "readers" are only blocked by "writers" whose critical was entered before the reader's attempt. /// /// When used to protect an actively recording data structure, the assumptions on how readers and writers act are: - ///
      - ///
    1. There are two sets of data structures("active" and "inactive")
      2. - ///
    2. Writing is done to the perceived active version(as perceived by the writer), and only within critical sections delineated by and ).
      3. - ///
    3. Only readers switch the perceived roles of the active and inactive data structures. - /// They do so only while under , and only before calling .
      4. - ///
    + /// + /// There are two sets of data structures("active" and "inactive") + /// Writing is done to the perceived active version(as perceived by the writer), and only within critical sections delineated by and ). + /// Only readers switch the perceived roles of the active and inactive data structures. + /// They do so only while under , and only before calling . + /// /// When the above assumptions are met, guarantees that the inactive data structures are not being modified by any writers while being read while under protection after a operation. ///     /// diff --git a/modules/GarnetJSON/GarnetJSON.csproj b/modules/GarnetJSON/GarnetJSON.csproj index 2a46ab3872e..721427387cd 100644 --- a/modules/GarnetJSON/GarnetJSON.csproj +++ b/modules/GarnetJSON/GarnetJSON.csproj @@ -19,6 +19,7 @@ + diff --git a/modules/GarnetJSON/GarnetJsonObject.cs b/modules/GarnetJSON/GarnetJsonObject.cs index 5ec9928122b..e511dbfab09 100644 --- a/modules/GarnetJSON/GarnetJsonObject.cs +++ b/modules/GarnetJSON/GarnetJsonObject.cs @@ -9,6 +9,7 @@ using System.Text.Json.Nodes; using Garnet.server; using GarnetJSON.JSONPath; +using Tsavorite.core; namespace GarnetJSON { @@ -41,10 +42,10 @@ public override CustomObjectBase Deserialize(byte type, BinaryReader reader) public class GarnetJsonObject : CustomObjectBase { private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = - new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; + new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }; private static readonly JsonSerializerOptions IndentedJsonSerializerOptions = - new JsonSerializerOptions { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, WriteIndented = true }; + new() { Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, WriteIndented = true }; private static readonly byte[] OpenBoxBracket = Encoding.UTF8.GetBytes("["); private static readonly byte[] CloseBoxBracket = Encoding.UTF8.GetBytes("]"); @@ -61,7 +62,7 @@ public class GarnetJsonObject : CustomObjectBase ///     /// The type of the object. public GarnetJsonObject(byte type) - : base(type, 0, MemoryUtils.DictionaryOverhead) + : base(type, MemoryUtils.DictionaryOverhead) { } @@ -101,8 +102,8 @@ public GarnetJsonObject(GarnetJsonObject obj) /// The binary writer to serialize to. public override void SerializeObject(BinaryWriter writer) { - if (rootNode == null) return; - + if (rootNode == null) + return; writer.Write(rootNode.ToJsonString()); } @@ -128,22 +129,18 @@ public override unsafe void Scan(long start, out List items, out long cu /// The string to use for new lines. /// The string to use for spaces. /// True if the operation is successful; otherwise, false. - public bool TryGet(ReadOnlySpan paths, List output, out ReadOnlySpan errorMessage, + public bool TryGet(ReadOnlySpan paths, List output, out ReadOnlySpan errorMessage, string? indent = null, string? newLine = null, string? space = null) { if (paths.Length == 1) - { return TryGet(paths[0].ReadOnlySpan, output, out errorMessage, indent, newLine, space); - } output.Add(OpenCurlyBracket); var isFirst = true; foreach (var item in paths) { if (!isFirst) - { output.Add(Comma); - } isFirst = false; @@ -152,11 +149,8 @@ public bool TryGet(ReadOnlySpan paths, List output, out ReadOn output.Add(DoubleQuotesColon); if (!TryGet(item.ReadOnlySpan, output, out errorMessage, indent, newLine, space)) - { return false; - } } - output.Add(CloseCurlyBracket); errorMessage = default; @@ -181,9 +175,7 @@ public bool TryGet(ReadOnlySpan path, List output, out ReadOnlySpa { errorMessage = default; if (rootNode is null) - { return true; - } if (path.Length == 0) { @@ -202,9 +194,7 @@ indent is null && newLine is null && space is null foreach (var item in result) { if (!isFirst) - { output.Add(Comma); - } isFirst = false; @@ -234,8 +224,8 @@ indent is null && newLine is null && space is null /// The error message if the operation fails. /// The result of the set operation. /// Thrown when there is an error in JSON processing. - public SetResult Set(ReadOnlySpan path, ReadOnlySpan value, ExistOptions existOptions, - out ReadOnlySpan errorMessage) + /// TODO: This currently does not update . + public SetResult Set(ReadOnlySpan path, ReadOnlySpan value, ExistOptions existOptions, out ReadOnlySpan errorMessage) { try { @@ -255,15 +245,13 @@ public SetResult Set(ReadOnlySpan path, ReadOnlySpan value, ExistOpt } // Need ToArray to avoid modifying collection while iterating - JsonPath jsonPath = new JsonPath(pathStr); + var jsonPath = new JsonPath(pathStr); var result = jsonPath.Evaluate(rootNode, rootNode, null).ToArray(); if (result.Length == 0) { if (existOptions == ExistOptions.XX) - { return SetResult.ConditionNotMet; - } if (!jsonPath.IsStaticPath()) { @@ -275,33 +263,23 @@ public SetResult Set(ReadOnlySpan path, ReadOnlySpan value, ExistOpt var parentNode = rootNode.SelectNodes(GetParentPath(pathStr, out var pathParentOffset)) .FirstOrDefault(); if (parentNode is null) - { return SetResult.ConditionNotMet; - } var childNode = JsonNode.Parse(value); var itemPropName = GetPropertyName(pathStr, pathParentOffset); if (parentNode is JsonObject matchObject) - { matchObject.Add(itemPropName.ToString(), childNode); - } else if (parentNode is JsonArray matchArray && int.TryParse(itemPropName, out var index)) - { matchArray.Insert(index, childNode); - } else - { return SetResult.ConditionNotMet; - } return SetResult.Success; } if (existOptions == ExistOptions.NX) - { return SetResult.ConditionNotMet; - } foreach (var match in result.ToList()) { @@ -332,14 +310,10 @@ private static string GetParentPath(string path, out int pathOffset) pathOffset = pathSpan[..^1].LastIndexOfAny('.', ']'); if (pathOffset == -1) - { return "$"; - } if (pathSpan[pathOffset] == ']') - { pathOffset++; - } return path.Substring(0, pathOffset); } @@ -348,20 +322,14 @@ private static ReadOnlySpan GetPropertyName(string path, int pathOffset) { var pathSpan = path.AsSpan(); if (pathSpan[pathOffset] is '.') - { pathOffset++; - } var propertSpan = pathSpan[pathOffset..]; if (propertSpan[0] is '[') - { propertSpan = propertSpan[1..^1]; - } if (propertSpan[0] is '"' or '\'') - { propertSpan = propertSpan[1..^1]; - } return propertSpan; } diff --git a/modules/GarnetJSON/JSONPath/JsonPath.cs b/modules/GarnetJSON/JSONPath/JsonPath.cs index abef99938f3..118755b85d7 100644 --- a/modules/GarnetJSON/JSONPath/JsonPath.cs +++ b/modules/GarnetJSON/JSONPath/JsonPath.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + #region License // Copyright (c) 2007 James Newton-King diff --git a/modules/GarnetJSON/JsonCommands.cs b/modules/GarnetJSON/JsonCommands.cs index 1bfa9e7134a..ff179e95137 100644 --- a/modules/GarnetJSON/JsonCommands.cs +++ b/modules/GarnetJSON/JsonCommands.cs @@ -30,8 +30,7 @@ public class JsonSET : CustomObjectFunctions /// The input data. /// The output data. /// Always returns true. - public override bool NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput input, - ref RespMemoryWriter writer) => true; + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) => true; /// /// Updates the JSON object with the specified key and input. @@ -42,8 +41,7 @@ public override bool NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput /// The output data. /// Additional information for the update. /// True if the update is successful, otherwise false. - public override bool Updater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject jsonObject, - ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + public override bool Updater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject jsonObject, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { Debug.Assert(jsonObject is GarnetJsonObject); @@ -106,8 +104,7 @@ public class JsonGET : CustomObjectFunctions /// The output data. /// Additional information for the read operation. /// True if the read is successful, otherwise false. - public override bool Reader(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject jsonObject, - ref RespMemoryWriter writer, ref ReadInfo readInfo) + public override bool Reader(ReadOnlySpan key, ref ObjectInput input, IGarnetObject jsonObject, ref RespMemoryWriter writer, ref ReadInfo readInfo) { Debug.Assert(jsonObject is GarnetJsonObject); var garnetJsonObject = jsonObject as GarnetJsonObject; @@ -125,7 +122,7 @@ public override bool Reader(ReadOnlyMemory key, ref ObjectInput input, IGa } else { - ReadOnlySpan paths = default; + ReadOnlySpan paths = default; var offset = 0; string? indent = null; string? newLine = null; diff --git a/modules/NoOpModule/DummyObject.cs b/modules/NoOpModule/DummyObject.cs index 06cd534d989..ae3ead40eac 100644 --- a/modules/NoOpModule/DummyObject.cs +++ b/modules/NoOpModule/DummyObject.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. using Garnet.server; +using Tsavorite.core; namespace NoOpModule { @@ -26,7 +27,7 @@ public class DummyObject : CustomObjectBase { /// public DummyObject(byte type) - : base(type, 0, MemoryUtils.DictionaryOverhead) + : base(type, MemoryUtils.DictionaryOverhead) { } diff --git a/modules/NoOpModule/DummyObjectNoOpRMW.cs b/modules/NoOpModule/DummyObjectNoOpRMW.cs index acdb3cfafd9..41723579e2c 100644 --- a/modules/NoOpModule/DummyObjectNoOpRMW.cs +++ b/modules/NoOpModule/DummyObjectNoOpRMW.cs @@ -13,11 +13,11 @@ namespace NoOpModule public class DummyObjectNoOpRMW : CustomObjectFunctions { /// - public override bool NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput input, + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) => true; /// - public override bool Updater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, + public override bool Updater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) { return true; diff --git a/modules/NoOpModule/DummyObjectNoOpRead.cs b/modules/NoOpModule/DummyObjectNoOpRead.cs index eef7dc28339..3e93074da27 100644 --- a/modules/NoOpModule/DummyObjectNoOpRead.cs +++ b/modules/NoOpModule/DummyObjectNoOpRead.cs @@ -13,7 +13,7 @@ namespace NoOpModule public class DummyObjectNoOpRead : CustomObjectFunctions { /// - public override bool Reader(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, + public override bool Reader(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) { return true; diff --git a/modules/NoOpModule/NoOpCommandRMW.cs b/modules/NoOpModule/NoOpCommandRMW.cs index f17b63bf618..6523d042164 100644 --- a/modules/NoOpModule/NoOpCommandRMW.cs +++ b/modules/NoOpModule/NoOpCommandRMW.cs @@ -13,37 +13,37 @@ namespace NoOpModule public class NoOpCommandRMW : CustomRawStringFunctions { /// - public override bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new InvalidOperationException(); /// - public override bool NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref RespMemoryWriter writer) + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) => false; /// - public override int GetInitialLength(ref RawStringInput input) + public override int GetInitialLength(ref StringInput input) => throw new InvalidOperationException(); /// - public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new InvalidOperationException(); /// - public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => true; /// - public override bool NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, + public override bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) => false; /// - public override int GetLength(ReadOnlySpan value, ref RawStringInput input) + public override int GetLength(ReadOnlySpan value, ref StringInput input) => 0; /// - public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => true; } } \ No newline at end of file diff --git a/modules/NoOpModule/NoOpCommandRead.cs b/modules/NoOpModule/NoOpCommandRead.cs index c8721834171..4e468baf571 100644 --- a/modules/NoOpModule/NoOpCommandRead.cs +++ b/modules/NoOpModule/NoOpCommandRead.cs @@ -13,29 +13,29 @@ namespace NoOpModule public class NoOpCommandRead : CustomRawStringFunctions { /// - public override int GetInitialLength(ref RawStringInput input) => throw new NotImplementedException(); + public override int GetInitialLength(ref StringInput input) => throw new NotImplementedException(); /// - public override int GetLength(ReadOnlySpan value, ref RawStringInput input) => + public override int GetLength(ReadOnlySpan value, ref StringInput input) => throw new NotImplementedException(); /// - public override bool InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); /// - public override bool InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); /// - public override bool CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); /// - public override bool Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) { return true; diff --git a/playground/ClusterStress/ClusterOptions.cs b/playground/ClusterStress/ClusterOptions.cs deleted file mode 100644 index c82fbc9ffdd..00000000000 --- a/playground/ClusterStress/ClusterOptions.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using CommandLine; - -namespace Resp.benchmark -{ - public partial class Options - { - [Option("cluster", Required = false, Default = false, HelpText = "Cluster mode benchmark enable")] - public bool Cluster { get; set; } - - [Option("shard", Required = false, Default = -1, HelpText = "Restrict benchmark to specific shard")] - public int Shard { get; set; } - - [Option("replica-reads", Required = false, Default = false, HelpText = "Allow replica reads for cluster mode.")] - public bool ReplicaReads { get; set; } - - [Option("migrate-freq", Required = false, Default = 0, HelpText = "Used to control frequency of a task that issues migrate command (Only for cluster option).")] - public int MigrateSlotsFreq { get; set; } - - [Option("migrate-batch", Required = false, Default = 100, HelpText = "Max number of slots picked to migrate from one node to another from background task that executes migrate (Only for cluster option).")] - public int MigrateBatch { get; set; } - - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ClusterStress.csproj b/playground/ClusterStress/ClusterStress.csproj deleted file mode 100644 index 5a77e2b49de..00000000000 --- a/playground/ClusterStress/ClusterStress.csproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - Exe - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/playground/ClusterStress/OnlineReqGen.cs b/playground/ClusterStress/OnlineReqGen.cs deleted file mode 100644 index 3023bb468b7..00000000000 --- a/playground/ClusterStress/OnlineReqGen.cs +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using System.Text; - -namespace Resp.benchmark -{ - public unsafe partial class OnlineReqGen - { - int[] slotPrefixes; - - public OnlineReqGen(int thread_id, int DbSize, bool randomGen = true, bool zipf = false, int keyLen = default, int valueLen = default, int objectDbSize = -1, bool cluster = true) - { - this.randomGen = randomGen; - this.DbSize = DbSize; - this.zipf = zipf; - if (objectDbSize == -1) - { - this.ObjectDbSize = DbSize; - } - else - { - this.ObjectDbSize = objectDbSize; - } - - this.keyLen = Math.Max(NumUtils.NumDigits(DbSize), keyLen); - this.valueLen = valueLen == default ? 8 : valueLen; - valueBuffer = new byte[this.valueLen]; - keyBuffer = GC.AllocateArray(this.keyLen, true); - keyBufferPtr = (byte*)Unsafe.AsPointer(ref keyBuffer[0]); - - InitializeRNGCluster(31337 + thread_id, 41337 + thread_id); - } - - private void InitializeRNGCluster(int keySeed = -1, int valueSeed = -1) - { - if (zipf) - zipfg = new ZipfGenerator(new RandomGenerator(), DbSize, 0.99); - keyRandomGen = keySeed == -1 ? new Random(Guid.NewGuid().GetHashCode()) : new Random(keySeed); - valueRandomGen = valueSeed == -1 ? new Random(Guid.NewGuid().GetHashCode()) : new Random(valueSeed); - - GenerateCRCPrefixesForAllSlots(); - } - - private void GenerateCRCPrefixesForAllSlots() - { - HashSet slots = new(); - for (int i = 0; i < 16384; i++) - slots.Add(i); - slotPrefixes = new int[16384]; - while (slots.Count > 0) - { - int keyPrefix = keyRandomGen.Next(0, int.MaxValue); - int slot = Garnet.common.HashSlotUtils.HashSlot(Encoding.ASCII.GetBytes(keyPrefix.ToString())); - if (slots.Contains(slot)) - { - slotPrefixes[slot] = keyPrefix; - slots.Remove(slot); - } - } - } - - private byte[] GetClusterKeyBytes(int key) - { - string keyStr = "{" + key.ToString() + "}"; - return Encoding.ASCII.GetBytes(keyStr.PadRight(keyLen, 'X')); - } - - public byte[] GenerateKeyBytes(out int slot) - { - int key = randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(DbSize)) : (keyIndex++ % DbSize); - slot = Garnet.common.HashSlotUtils.HashSlot(Encoding.ASCII.GetBytes(key.ToString())); - byte[] keyBytes = GetClusterKeyBytes(key); -#if DEBUG - int _slot = Garnet.common.HashSlotUtils.HashSlot(keyBytes); - System.Diagnostics.Debug.Assert(_slot == slot, $"GenerateKeyBytes slot number incosistence {_slot}:{slot}"); -#endif - return keyBytes; - } - - public string GenerateKey(out int slot) - { - int key = randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(DbSize)) : (keyIndex++ % DbSize); - slot = Garnet.common.HashSlotUtils.HashSlot(Encoding.ASCII.GetBytes(key.ToString())); - byte[] keyBytes = GetClusterKeyBytes(key); -#if DEBUG - int _slot = Garnet.common.HashSlotUtils.HashSlot(keyBytes); - System.Diagnostics.Debug.Assert(_slot == slot, $"GenerateKeyBytes slot number incosistence {_slot}:{slot}"); -#endif - return Encoding.ASCII.GetString(keyBytes); - } - - public string GenerateKeyInSlot(out int slot) - { - slot = (randomGen ? (zipf ? zipfg.Next() : keyRandomGen.Next(DbSize)) : (keyIndex++ % DbSize)) & 16383; - string keyStr = "{" + slotPrefixes[slot] + "}" + PadRandom(keyLen); - return keyStr; - } - } -} \ No newline at end of file diff --git a/playground/ClusterStress/Program.cs b/playground/ClusterStress/Program.cs deleted file mode 100644 index a6d943c708f..00000000000 --- a/playground/ClusterStress/Program.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Linq; -using CommandLine; -using Garnet.common; -using Microsoft.Extensions.Logging; -using Resp.benchmark; -using StackExchange.Redis; - -namespace ClusterStress -{ - class Program - { - public static IConnectionMultiplexer redis; - public static ILoggerFactory loggerFactory; - - static ILoggerFactory CreateLoggerFactory(Options opts) - { - return LoggerFactory.Create(builder => - { - if (!opts.DisableConsoleLogger) - { - builder.AddProvider(new BenchmarkLoggerProvider(Console.Out)); - } - - // Optional: Flush log output to file. - if (opts.FileLogger != null) - builder.AddFile(opts.FileLogger); - builder.SetMinimumLevel(opts.LogLevel); - }); - } - - static void Main(string[] args) - { - ParserResult result = Parser.Default.ParseArguments(args); - if (result.Tag == ParserResultType.NotParsed) return; - var opts = result.MapResult(o => o, xs => new Options()); - - loggerFactory = CreateLoggerFactory(opts); - - if (opts.Client == Resp.benchmark.ClientType.SERedis) - redis = ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost)); - - if (opts.Cluster) - RunShardedBasicCommandsBenchmark(opts); - } - - static void RunShardedBasicCommandsBenchmark(Options opts) - { - if (opts.Online) - { - if (opts.SkipLoad) - throw new Exception("Skipload not supported with --online"); - var bench = new ShardedRespOnlineBench(opts, runDuration: opts.RunTime == -1 ? int.MaxValue : opts.RunTime, loggerFactory: loggerFactory); - bench.Run(); - } - else - { - var bench = new ShardedRespPerfBench(opts, 0); - if (!opts.SkipLoad) - bench.LoadData(keyLen: opts.KeyLength, valueLen: opts.ValueLength, BatchSize: opts.BatchSize.First()); - - int[] threadBench = [.. opts.NumThreads]; - int keyLen = opts.KeyLength; - int valueLen = opts.ValueLength; - foreach (int BatchSize in opts.BatchSize) - bench.Run( - opts.Op, - opts.DbSize, - threadBench, - runTime: TimeSpan.FromSeconds(opts.RunTime), - keyLen: keyLen, - valueLen: valueLen, - BatchSize: BatchSize, - ttl: opts.Ttl); - } - } - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ReqGenForCluster.cs b/playground/ClusterStress/ReqGenForCluster.cs deleted file mode 100644 index 02b6b38d775..00000000000 --- a/playground/ClusterStress/ReqGenForCluster.cs +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Text; -using System.Threading; -using StackExchange.Redis; - -namespace Resp.benchmark -{ - public unsafe partial class ReqGen - { - readonly List[] databaseKeys; - readonly int[] interleavedSlots = new int[16384]; - readonly int shard; - readonly ClusterConfiguration clusterConfig; - int currSlot = 0; - int currKeyInSlot = 0; - - public ReqGen( - int Start, - int DbSize, - int NumOps, - int BatchSize, - OpType opType, - ClusterConfiguration clusterConfig, - int shard = -1, - bool randomGen = true, - bool randomServe = true, - int keyLen = default, - int valueLen = default, - bool numericValue = false, - bool verbose = true, - bool zipf = false, - bool flatBufferClient = false, - int ttl = 0) - { - this.shard = shard; - this.clusterConfig = clusterConfig; - - databaseKeys = new List[16384]; - - for (int i = 0; i < databaseKeys.Length; i++) - databaseKeys[i] = new List(); - - this.NumBuffs = NumOps / BatchSize; - if (NumBuffs > MaxBatches && verbose) - { - Console.WriteLine($"Restricting #buffers to {MaxBatches} instead of {NumBuffs}"); - NumBuffs = MaxBatches; - } - this.buffers = new byte[NumBuffs][]; - - this.flatRequestBuffer = flatBufferClient ? new List>() : null; - this.lens = new int[NumBuffs]; - this.BatchCount = BatchSize; - this.opType = opType; - this.seqNo = 0; - this.randomGen = randomGen; - this.randomServe = randomServe; - this.DbSize = DbSize; - this.Start = Start; - this.flatBufferClient = flatBufferClient; - this.ttl = ttl; - - if (zipf) - { - this.zipf = zipf; - zipfg = new ZipfGenerator(new RandomGenerator(), DbSize, 0.99); - } - - this.keyLen = keyLen == default ? NumUtils.NumDigits(DbSize) : keyLen; - this.valueLen = valueLen == default ? 8 : valueLen; - valueBuffer = new byte[this.valueLen]; - - this.numericValue = numericValue; - this.verbose = verbose; - - int _hllDstMergeKeyCount = (int)(((double)(DbSize)) * this.hllDstMergeKeyFraction); - this.hllDstMergeKeyCount = hllDstMergeKeyCount == 0 ? this.hllDstMergeKeyCount : _hllDstMergeKeyCount; - - this.ttl = ttl; - } - - public void GenerateForCluster() - { - if (verbose) - { - Console.WriteLine(); - Console.WriteLine($"Generating {NumBuffs} {opType} request batches of size {BatchCount} each; total {NumBuffs * BatchCount} ops"); - - if (opType == OpType.PFMERGE) - { - Console.WriteLine("PFMERGE config > mergeDstKeyCount:{0}, hllDstMergeKeyFraction:{1}", hllDstMergeKeyCount, hllDstMergeKeyFraction); - } - } - - // Prepare the cluster sharded keys and slots - var elapsed = Stopwatch.StartNew(); - GenerateShardedKeys(); - InitInterleaveSlots(); - elapsed.Stop(); - if (verbose) - { - Console.WriteLine("Generate keys time: {0} secs", elapsed.ElapsedMilliseconds / 1000.0); - } - - var sw = Stopwatch.StartNew(); - int maxBytesWritten = 0; - while (true) - { - InitializeRNG(); - for (int i = 0; i < NumBuffs; i++) - { - buffers[i] = new byte[BufferSize]; - lens[i] = 0; - - //Reset counters to point to buffer for slot - currSlot = i; - currKeyInSlot = 0; - - switch (opType) - { - case OpType.ZADDREM: - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, 0, BatchCount / 2, OpType.ZADD)) goto resizeBuffer; - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, BatchCount / 2, BatchCount, OpType.ZREM)) goto resizeBuffer; - break; - case OpType.GEOADDREM: - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, 0, BatchCount / 2, OpType.GEOADD)) goto resizeBuffer; - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, BatchCount / 2, BatchCount, OpType.ZREM)) goto resizeBuffer; - break; - case OpType.ZADDCARD: - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, 0, BatchCount / 2, OpType.ZADD)) goto resizeBuffer; - InitializeRNG(keySeed: i); - if (!GenerateBatch(i, BatchCount / 2, BatchCount, OpType.ZCARD)) goto resizeBuffer; - break; - default: - if (!GenerateBatch(i, 0, BatchCount, opType)) goto resizeBuffer; - break; - } - maxBytesWritten = Math.Max(lens[i], maxBytesWritten); - } - break; - - resizeBuffer: - if (verbose) - { - Console.Write("Resizing request buffer from {0}", BufferSize); - BufferSize = BufferSize << 1; - Console.WriteLine(" to {0}", BufferSize); - } - } - sw.Stop(); - if (verbose) - { - Console.WriteLine("Request generation complete"); - Console.WriteLine("maxBytesWritten out of maxBufferSize: {0}/{1}", maxBytesWritten, BufferSize); - Console.WriteLine("Loading time: {0} secs", sw.ElapsedMilliseconds / 1000.0); - } - - if (flatBufferClient) - { - ConvertToSERedisInput(opType); - if (flatRequestBuffer.Count > 0) - { - for (int i = 0; i < NumBuffs; i++) - buffers[i] = null; - } - } - - } - - public byte[] GetRequestInterleaved(ref Random r, out int len, out int slot) - { - int offset; - if (randomServe) - offset = r.Next(NumBuffs); - else - offset = (Interlocked.Increment(ref seqNo) - 1) % NumBuffs; - - slot = interleavedSlots[offset & 16383]; - len = lens[offset]; - return buffers[offset]; - } - - private byte[] GetClusterKeyInterleaved() - { - //currSlot is buffer index, interleaved slots array for all slots to assign all - int slot = interleavedSlots[currSlot & 16383]; - int keyPrefixCount = databaseKeys[slot].Count; - int key = databaseKeys[slot][currKeyInSlot & (keyPrefixCount - 1)]; - - string keyStr = "{" + key.ToString() + "}"; - currKeyInSlot++; - return Encoding.ASCII.GetBytes(keyStr.PadRight(keyLen, numericValue ? '1' : 'X')); - } - - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ReqGenSharded.cs b/playground/ClusterStress/ReqGenSharded.cs deleted file mode 100644 index 15ed40a5a80..00000000000 --- a/playground/ClusterStress/ReqGenSharded.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Resp.benchmark -{ - public unsafe partial class ReqGen - { - private void GenerateRandomKeys() - { - for (int i = 0; i < DbSize; i++) - { - int key = Start + keyRandomGen.Next(DbSize); - int slot = Garnet.common.HashSlotUtils.HashSlot(System.Text.Encoding.ASCII.GetBytes(key.ToString())); - databaseKeys[slot].Add(key); - } - } - - private void ValidateKeysPerSlot(int keysPerSlot) - { - foreach (var keys in databaseKeys) - { - if (keys.Count != keysPerSlot) - throw new Exception($"keysPerSlot not assigned {keysPerSlot} {keys.Count}"); - } - } - - private void GenerateKeysCoverAllSlots() - { - int slotCount = 16384; - int keysPerSlot = ((DbSize - 1) / slotCount) + 1; - - for (int i = 0; i < DbSize; i++) - { - retry: - int key = Start + keyRandomGen.Next(); - int slot = Garnet.common.HashSlotUtils.HashSlot(System.Text.Encoding.ASCII.GetBytes(key.ToString())); - - if (databaseKeys[slot].Count < keysPerSlot) - databaseKeys[slot].Add(key); - else - goto retry; - } - ValidateKeysPerSlot(keysPerSlot); - } - - public void GenerateShardedKeys() - { - InitializeRNG(); - currSlot = 0; - currKeyInSlot = 0; - if (DbSize > (1 << 20)) - GenerateRandomKeys(); - else - GenerateKeysCoverAllSlots(); - } - - /// - /// Create a map of interleaved slots. Used to load data across shards equally independent of db size - /// - private void InitInterleaveSlots() - { - var pNodes = clusterConfig.Nodes.ToList().FindAll(p => !p.IsReplica && p.Slots.Count > 0).ToArray(); - - LinkedList[] shardSlots = new LinkedList[pNodes.Length]; - for (int i = 0; i < shardSlots.Length; i++) - { - shardSlots[i] = new LinkedList(); - var slotRanges = pNodes[i].Slots; - foreach (var slotRange in slotRanges) - { - for (int j = slotRange.From; j <= slotRange.To; j++) - shardSlots[i].AddLast(j); - } - } - - int k = shard == -1 ? 0 : shard; - for (int i = 0; i < interleavedSlots.Length; i++) - { - interleavedSlots[i] = shardSlots[k].First(); - shardSlots[k].RemoveFirst(); - shardSlots[k].AddLast(interleavedSlots[i]); - k = shard == -1 ? (k + 1 < shardSlots.Length ? k + 1 : 0) : (shard); - } - } - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ReqGenUtilsCluster.cs b/playground/ClusterStress/ReqGenUtilsCluster.cs deleted file mode 100644 index 3aa7aa53926..00000000000 --- a/playground/ClusterStress/ReqGenUtilsCluster.cs +++ /dev/null @@ -1,414 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Text; - -namespace Resp.benchmark -{ - public unsafe partial class ReqGen - { - static readonly bool invalidateHLL = false; - - private bool WriteHeader(byte[] headerData, ref byte* curr, byte* vend) - { - byte* save = curr; - if (curr + headerData.Length >= vend) - return false; - for (int i = 0; i < headerData.Length; i++) - *curr++ = headerData[i]; - return true; - } - - private bool WriteBitfieldArgs(ref byte* curr, byte* vend, byte[] bitfieldOpType) - { - int offset = valueRandomGen.Next(0, (valueBuffer.Length << 3) - 64); - int bitCount = valueRandomGen.Next(1, 64); - long vset = RandomIntBitRange(bitCount, true); - byte[] typeData = Encoding.ASCII.GetBytes("i" + bitCount.ToString()); - - WriteStringBytes(ref curr, vend, bitfieldOpType); - WriteStringBytes(ref curr, vend, typeData); - WriteInteger(offset, ref curr, vend); - - if (bitfieldOpType[0] == 'G') return true; - - WriteInteger(vset, ref curr, vend); - - return true; - } - - private bool WriteInteger(int n, ref byte* curr, byte* vend) - { - int nd = NumUtils.NumDigits(n); - int sign = ((n < 0) ? 1 : 0); - - int ndSize = NumUtils.NumDigits(nd + sign); - int totalLen = 1 + ndSize + 2 + (nd + sign) + 2; - if (curr + totalLen >= vend) - return false; - - *curr++ = (byte)'$'; - NumUtils.IntToBytes(nd + sign, ref curr); - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - NumUtils.IntToBytes(n, nd, ref curr); - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - return true; - } - - private bool WriteInteger(long n, ref byte* curr, byte* vend) - { - int nd = NumUtils.NumDigitsInLong(n); - int sign = ((n < 0) ? 1 : 0); - - int ndSize = NumUtils.NumDigits(nd + sign); - int totalLen = 1 + ndSize + 2 + (nd + sign) + 2; - if (curr + totalLen >= vend) - return false; - - *curr++ = (byte)'$'; - NumUtils.IntToBytes(nd + sign, ref curr); - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - NumUtils.LongToBytes(n, nd, ref curr); - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - - return true; - } - - private bool WriteStringBytes(ref byte* curr, byte* vend, byte[] data) - { - int digits = NumUtils.NumDigits(data.Length); - int totalLen = 1 + digits + 2 + data.Length + 2; - if (curr + totalLen >= vend) - return false; - - *curr++ = (byte)'$'; - NumUtils.IntToBytes(data.Length, ref curr); - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - - for (int i = 0; i < data.Length; i++) *curr++ = (byte)data[i]; - *curr++ = (byte)'\r'; - *curr++ = (byte)'\n'; - return true; - } - - private bool WriteOp(ref byte* curr, byte* vend, OpType opType) - { - int n; - - var bitopType = opType switch - { - OpType.BITOP_AND => Encoding.ASCII.GetBytes("AND"), - OpType.BITOP_OR => Encoding.ASCII.GetBytes("OR"), - OpType.BITOP_XOR => Encoding.ASCII.GetBytes("XOR"), - OpType.BITOP_NOT => Encoding.ASCII.GetBytes("NOT"), - _ => null - }; - - byte[] keyData = null; - - //key - switch (opType) - { - case OpType.ZADD: - case OpType.ZREM: - case OpType.ZCARD: - case OpType.GEOADD: - case OpType.PFADD: - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.PFCOUNT: - if (!WriteKey(ref curr, vend, out keyData)) - return false; - - if (invalidateHLL) - { - //Try to invalidate PFCOUNT - if (!WriteHeader(Encoding.ASCII.GetBytes($"*3\r\n$5\r\nPFADD\r\n"), ref curr, vend)) - return false; - - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - } - - break; - case OpType.PFMERGE: - int key = keyRandomGen.Next(0, hllDstMergeKeyCount); - if (!WriteKey(ref curr, vend, key)) - return false; - if (invalidateHLL) - { - //Try to delete merge HLL - if (!WriteHeader(Encoding.ASCII.GetBytes($"*2\r\n$3\r\nDEL\r\n"), ref curr, vend)) - return false; - - if (!WriteKey(ref curr, vend, key)) - return false; - } - break; - case OpType.MSET: - case OpType.INCR: - case OpType.GET: - case OpType.SET: - case OpType.SETEX: - case OpType.MGET: - case OpType.SETBIT: - case OpType.GETBIT: - case OpType.BITCOUNT: - case OpType.BITPOS: - case OpType.SETIFPM: - case OpType.MYDICTSET: - case OpType.MYDICTGET: - if (!WriteKey(ref curr, vend, out keyData)) - return false; - break; - case OpType.BITOP_AND: - case OpType.BITOP_OR: - case OpType.BITOP_XOR: - case OpType.BITOP_NOT: - if (!WriteStringBytes(ref curr, vend, bitopType)) - return false; - break; - case OpType.BITFIELD: - case OpType.BITFIELD_GET: - case OpType.BITFIELD_SET: - case OpType.BITFIELD_INCR: - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.PING: - return true; - default: - break; - } - - //arg1 - switch (opType) - { - case OpType.ZADD: - case OpType.ZREM: - n = Start + r.Next(DbSize); - if (!WriteInteger(n, ref curr, vend)) - return false; - break; - case OpType.PFADD: - case OpType.MSET: - if (valueLen == 0) - { - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - } - else - { - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - } - break; - - case OpType.MYDICTSET: - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - if (valueLen == 0) - { - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - } - else - { - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - } - break; - - case OpType.MYDICTGET: - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - break; - - case OpType.SETIFPM: - if (valueLen == 0) - { - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - } - else - { - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - } - if (valueLen == 0) - { - if (!WriteStringBytes(ref curr, vend, keyData)) - return false; - } - else - { - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - } - break; - case OpType.MPFADD: - case OpType.SET: - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - break; - case OpType.SETEX: - if (!WriteInteger(ttl, ref curr, vend)) - return false; - break; - case OpType.PFMERGE: - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.INCR: - case OpType.GET: - case OpType.MGET: - break; - case OpType.SETBIT: - case OpType.GETBIT: - n = valueRandomGen.Next(0, (valueLen << 3) - 1); - if (!WriteInteger(n, ref curr, vend)) - return false; - break; - case OpType.BITCOUNT: - break; - case OpType.BITPOS: - if (!WriteInteger(valueRandomGen.Next(0, 1), ref curr, vend)) - return false; - break; - case OpType.BITOP_AND: - case OpType.BITOP_OR: - case OpType.BITOP_XOR: - case OpType.BITOP_NOT: - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.BITFIELD: - bitfieldOpCount = 3; - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("SET"))) - return false; - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("INCRBY"))) - return false; - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("GET"))) - return false; - break; - case OpType.BITFIELD_GET: - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("GET"))) - return false; - break; - case OpType.BITFIELD_SET: - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("SET"))) - return false; - break; - case OpType.BITFIELD_INCR: - if (!WriteBitfieldArgs(ref curr, vend, Encoding.ASCII.GetBytes("INCRBY"))) - return false; - break; - case OpType.GEOADD: - if (!WriteStringBytes(ref curr, vend, Encoding.ASCII.GetBytes(GeoUtils.GetValidGeo().lng))) - return false; - break; - default: - break; - } - - //arg2 - switch (opType) - { - case OpType.ZADD: - n = Start + r.Next(DbSize); - if (!WriteInteger(n, ref curr, vend)) - return false; - break; - case OpType.ZREM: - case OpType.PFADD: - case OpType.MSET: - case OpType.INCR: - case OpType.GET: - case OpType.MGET: - break; - case OpType.SETBIT: - n = valueRandomGen.Next(0, 1); - if (!WriteInteger(n, ref curr, vend)) - return false; - break; - case OpType.GETBIT: - case OpType.BITCOUNT: - case OpType.BITPOS: - break; - case OpType.BITOP_AND: - case OpType.BITOP_OR: - case OpType.BITOP_XOR: - for (int i = 0; i < bitOpSrckeyCount; i++) - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.BITOP_NOT: - if (!WriteKey(ref curr, vend)) - return false; - break; - case OpType.BITFIELD: - case OpType.BITFIELD_GET: - case OpType.BITFIELD_SET: - case OpType.BITFIELD_INCR: - case OpType.GEOADD: - if (!WriteStringBytes(ref curr, vend, Encoding.ASCII.GetBytes(GeoUtils.GetValidGeo().lat))) - return false; - break; - case OpType.SETEX: - RandomString(); - if (!WriteStringBytes(ref curr, vend, valueBuffer)) - return false; - break; - default: - break; - } - - // arg3 - switch (opType) - { - case OpType.GEOADD: - n = Start + r.Next(DbSize); - if (!WriteInteger(n, ref curr, vend)) - return false; - break; - default: - break; - } - - return true; - } - - private bool WriteKey(ref byte* curr, byte* vend) - { - byte[] keyData = GetClusterKeyInterleaved(); - return WriteStringBytes(ref curr, vend, keyData); - } - - private bool WriteKey(ref byte* curr, byte* vend, out byte[] keyData) - { - keyData = GetClusterKeyInterleaved(); - return WriteStringBytes(ref curr, vend, keyData); - } - - private bool WriteKey(ref byte* curr, byte* vend, int key) - { - byte[] keyData = Encoding.ASCII.GetBytes(key.ToString().PadLeft(keyLen, 'X')); - return WriteStringBytes(ref curr, vend, keyData); - } - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ShardedRespOnlineBench.cs b/playground/ClusterStress/ShardedRespOnlineBench.cs deleted file mode 100644 index bf2071ce664..00000000000 --- a/playground/ClusterStress/ShardedRespOnlineBench.cs +++ /dev/null @@ -1,864 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Runtime.CompilerServices; -using System.Threading; -using System.Threading.Tasks; -using Garnet.client; -using Garnet.common; -using HdrHistogram; -using Microsoft.Extensions.Logging; -using StackExchange.Redis; - -namespace Resp.benchmark -{ - internal class ShardedRespOnlineBench - { - static readonly long HISTOGRAM_LOWER_BOUND = 1; - static readonly long HISTOGRAM_UPPER_BOUND = TimeStamp.Seconds(100); - const int bufferSizeValue = 1 << 17; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static bool IsValidRange(long value) - => value < HISTOGRAM_UPPER_BOUND && value > HISTOGRAM_LOWER_BOUND; - - readonly int NumThreads; - readonly int BatchSize; - readonly Options opts; - readonly string auth; - - readonly ManualResetEventSlim waiter = new(); - readonly LongHistogram[] thread_histograms; - readonly ManualResetEventSlim WaitForEpochBump = new(); - readonly int runDuration; - readonly int resetInterval; - ulong iteration = 0; - - readonly int[] opPercent; - readonly OpType[] opWorkload; - - readonly ClusterConfiguration clusterConfig; - readonly ushort[] slotMap = new ushort[16384]; - ClusterNode[] primaryNodes; - ClusterNode[] replicaNodes; - - GarnetClient[] gclient = null; - GarnetClientSession[][] gcs = null; - - readonly LightEpoch epoch = new(); - readonly Stopwatch epochWatch = new(); - - readonly ILoggerFactory loggerFactory; - readonly ILogger logger; - - AsyncPool redisPool; - IConnectionMultiplexer redis; - - readonly CancellationTokenSource cts = new(); - volatile int workerCount = 0; - - public ShardedRespOnlineBench(Options opts, int resetInterval = 30, int runDuration = int.MaxValue, ILoggerFactory loggerFactory = null) - { - this.runDuration = runDuration; - this.resetInterval = resetInterval; - this.opts = opts; - this.auth = opts.Auth; - NumThreads = opts.NumThreads.ToArray()[0]; - BatchSize = opts.BatchSize.ToArray()[0]; - thread_histograms = new LongHistogram[NumThreads * 2]; - for (int i = 0; i < thread_histograms.Length; i++) - thread_histograms[i] = new LongHistogram(HISTOGRAM_LOWER_BOUND, HISTOGRAM_UPPER_BOUND, 2); - - - opPercent = opts.OpPercent?.ToArray(); - opWorkload = opts.OpWorkload?.ToArray(); - - this.loggerFactory = loggerFactory; - this.logger = loggerFactory?.CreateLogger("sonline"); - - if (opPercent != null && opWorkload != null) - { - if (opPercent.Length != opWorkload.Length) - throw new Exception($"opPercent {opWorkload.Length} and opWorkload {opWorkload.Length} mismatch!"); - for (int i = 1; i < opPercent.Length; i++) - { - opPercent[i] += opPercent[i - 1]; - } - if (opPercent[^1] != 100) - throw new Exception($"opPercent must sum to 100, distribution: {String.Join(',', opPercent)}"); - } - - clusterConfig = GetClusterConfig(); - } - - OpType SelectOpType(int percent) - { - for (int i = 0; i < opPercent.Length; i++) - if (percent <= opPercent[i]) - return opWorkload[i]; - throw new Exception($"Invalid percent range {percent}"); - } - - private void RecordValue(int thread_id, long elapsed) - { - try - { - epoch.Resume(); - var _offset = (int)(this.iteration & 0x1); - if (IsValidRange(elapsed)) - thread_histograms[_offset * NumThreads + thread_id].RecordValue(elapsed); - else - thread_histograms[_offset * NumThreads + thread_id].RecordValue(HISTOGRAM_UPPER_BOUND); - } - finally - { - epoch.Suspend(); - } - } - - private double WaitForMonitorIterationSwitch(out int prev_iter) - { - prev_iter = (int)((this.iteration++)); - epochWatch.Start(); - try - { - epoch.Resume(); - epoch.BumpCurrentEpoch(WaitForEpochBump.Set); - } - catch (Exception ex) - { - Console.WriteLine(ex.ToString()); - WaitForEpochBump.Reset(); - return 0; - } - finally - { - epoch.Suspend(); - } - WaitForEpochBump.Wait(); - WaitForEpochBump.Reset(); - epochWatch.Stop(); - return epochWatch.ElapsedMilliseconds; - } - - private void PrintClusterConfig() - { - Console.WriteLine("Cluster Retrieved Configuration..."); - var nodes = clusterConfig.Nodes.ToArray(); - Array.Sort(nodes, (x, y) => ((IPEndPoint)x.EndPoint).Address.ToString().CompareTo(((IPEndPoint)y.EndPoint).Address.ToString())); - foreach (var node in nodes) - { - var endpoint = (IPEndPoint)node.EndPoint; - Console.Write($"host: {endpoint.Address}:{endpoint.Port}, "); - Console.Write($"role: {((node.IsReplica || node.Slots.Count == 0) ? "REPLICA" : "PRIMARY")}, "); - var slotRanges = node.Slots; - - Console.Write("slotRanges: "); - foreach (var slotRange in slotRanges) - Console.Write($"{slotRange.From} - {slotRange.To} "); - Console.WriteLine(""); - } - Console.WriteLine("<--------------------------------------------->"); - } - - private ClusterConfiguration GetClusterConfig() - { - using var redis = ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost, allowAdmin: true)); - var clusterConfig = redis.GetServer(opts.Address + ":" + opts.Port).ClusterNodes(); - - UpdateSlotMap(clusterConfig); - - return clusterConfig; - } - - private void UpdateSlotMap(ClusterConfiguration clusterConfig) - { - var nodes = clusterConfig.Nodes.ToArray(); - primaryNodes = [.. nodes.ToList().FindAll(p => !p.IsReplica)]; - replicaNodes = [.. nodes.ToList().FindAll(p => p.IsReplica)]; - ushort j = 0; - foreach (var node in nodes) - { - var slotRanges = node.Slots; - foreach (var slotRange in slotRanges) - { - for (int i = slotRange.From; i <= slotRange.To; i++) - { - slotMap[i] = j; - } - } - j++; - } - } - - private void InitClients(ClusterNode[] nodes) - { - switch (opts.Client) - { - case ClientType.GarnetClient: - gclient = new GarnetClient[nodes.Length]; - for (int i = 0; i < nodes.Length; i++) - { - var endpoint = (IPEndPoint)nodes[i].EndPoint; - gclient[i] = new GarnetClient(endpoint, - opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null, - recordLatency: opts.ClientHistogram); - gclient[i].Connect(); - } - break; - case ClientType.GarnetClientSession: - gcs = new GarnetClientSession[NumThreads][]; - for (int j = 0; j < NumThreads; j++) - { - gcs[j] = new GarnetClientSession[nodes.Length]; - for (int i = 0; i < nodes.Length; i++) - { - var endpoint = (IPEndPoint)nodes[i].EndPoint; - gcs[j][i] = new GarnetClientSession( - endpoint, - new(Math.Max(bufferSizeValue, opts.IntraThreadParallelism * opts.ValueLength)), - tlsOptions: opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null); - gcs[j][i].Connect(); - if (auth != null) - { - gcs[j][i].Execute("AUTH", auth); - gcs[j][i].CompletePending(); - } - } - } - break; - case ClientType.SERedis: - if (opts.Pool) - { - redisPool = new AsyncPool(opts.NumThreads.First(), () => - { - return ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost)); - }); - } - else - { - redis = ClusterStress.Program.redis; - } - break; - default: - throw new Exception($"ClientType {opts.Client} not supported"); - } - } - - private Thread[] InitializeThreadWorkers() - { - Thread[] workers = new Thread[NumThreads]; - for (int idx = 0; idx < NumThreads; ++idx) - { - int x = idx; - switch (opts.Client) - { - case ClientType.GarnetClientSession: - if (opts.IntraThreadParallelism > 1) - { - if (idx == 0) Console.WriteLine("Using OpRunnerGarnetClientSessionParallel..."); - workers[idx] = new Thread(() => OpRunnerGarnetClientSessionParallel(x, opts.IntraThreadParallelism)); - } - else - { - if (idx == 0) Console.WriteLine("Using OpRunnerGarnetClientSession..."); - workers[idx] = new Thread(() => OpRunnerGarnetClientSession(x)); - } - break; - case ClientType.GarnetClient: - if (opts.IntraThreadParallelism > 1) - { - if (idx == 0) Console.WriteLine("Using OpRunnerGarnetClientParallel..."); - workers[idx] = new Thread(() => OpRunnerGarnetClientParallel(x, opts.IntraThreadParallelism)); - } - else - { - if (idx == 0) Console.WriteLine("Using OpRunnerGarnetClient..."); - workers[idx] = new Thread(() => OpRunnerGarnetClient(x)); - } - break; - case ClientType.SERedis: - if (opts.IntraThreadParallelism > 1) - { - if (idx == 0) Console.WriteLine("Using OpRunnerSERedisParallel..."); - workers[idx] = new Thread(() => OpRunnerSERedisParallel(x, opts.IntraThreadParallelism)); - } - else - { - if (idx == 0) Console.WriteLine("Using OpRunnerSERedis..."); - workers[idx] = new Thread(() => OpRunnerSERedis(x)); - } - break; - default: - throw new Exception($"ClientType {opts.Client} not supported"); - } - } - return workers; - } - - private async void PeriodicConfigUpdate() - { - using var redis = ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost, allowAdmin: true)); - while (true) - { - await Task.Delay(TimeSpan.FromSeconds(1)); - if (cts.IsCancellationRequested) return; - var clusterConfig = redis.GetServer(opts.Address + ":" + opts.Port).ClusterNodes(); - UpdateSlotMap(clusterConfig); - } - } - - private async void MigrationBgTask() - { - using var redis = ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost, allowAdmin: true)); - Random r = new(7638); - try - { - while (true) - { - //Check if cancellation is requested - await Task.Delay(TimeSpan.FromSeconds(opts.MigrateSlotsFreq)); - if (cts.IsCancellationRequested) return; - - //Retrieve latest cluster config and update client slotMap - var clusterConfig = redis.GetServer(opts.Address + ":" + opts.Port).ClusterNodes(); - UpdateSlotMap(clusterConfig); - - //Initiate a migration operation between nodes - InitiateMigration(redis, r); - } - } - catch (Exception ex) - { - logger?.LogError(ex, "Migration bg task failed"); - } - } - - private void InitiateMigration(ConnectionMultiplexer redis, Random r) - { - List migratingSlots = new(); - //Initiate a migration operation between nodes - int source = r.Next(0, primaryNodes.Length); - int target = r.Next(0, primaryNodes.Length); - - //Choose target different from source - while (target == source) - { - target = r.Next(0, primaryNodes.Length); - if (cts.IsCancellationRequested) return; - } - - //Retrieve source and target info - var sourceNode = primaryNodes[source]; - var targetNode = primaryNodes[target]; - var sourceNodeEndpoint = (IPEndPoint)sourceNode.EndPoint; - var targetNodeEndpoint = (IPEndPoint)targetNode.EndPoint; - - //Pick randomly a few number of slots to migrate - foreach (var slotRange in sourceNode.Slots) - { - for (int i = slotRange.From; i < slotRange.To; i++) - if (r.Next(0, 2) > 0 && migratingSlots.Count < opts.MigrateBatch) - migratingSlots.Add(i); - } - - Console.WriteLine($"{sourceNodeEndpoint.Address}:{sourceNodeEndpoint.Port} > {targetNodeEndpoint.Address}:{targetNodeEndpoint.Port} slots:{migratingSlots.Count}"); - //Initiate migration - if (migratingSlots.Count > 0) - MigrateSlots(redis, sourceNodeEndpoint, targetNodeEndpoint, migratingSlots); - - //Clear migration list - migratingSlots.Clear(); - } - - public static void MigrateSlots(ConnectionMultiplexer redis, IPEndPoint source, IPEndPoint target, List slots, bool range = false, ILogger logger = null) - { - //MIGRATE host port destination-db timeout [COPY] [REPLACE] [[AUTH password] | [AUTH2 username password]] [KEYS key [key...]] - var server = redis.GetServer(source); - List args = new() - { - target.Address.ToString(), - target.Port, - "", - 0, - -1, - range ? "SLOTSRANGE": "SLOTS" - }; - foreach (var slot in slots) - args.Add(slot); - - try - { - var resp = server.Execute("migrate", args); - if (!resp.Equals("OK")) - logger?.LogError("{errorMessage}", resp.ToString()); - } - catch (Exception ex) - { - logger?.LogError(ex, "An error has occurred"); - } - } - - public void Run() - { - PrintClusterConfig(); - Console.WriteLine($"Running benchmark using {opts.Client} client type"); - - // Initialize clients to nodes using the retrieved configuration - InitClients([.. clusterConfig.Nodes]); - Thread[] workers = InitializeThreadWorkers(); - - // Start threads. - foreach (Thread worker in workers) - worker.Start(); - - var summary = new LongHistogram(HISTOGRAM_LOWER_BOUND, HISTOGRAM_UPPER_BOUND, 2); - bool printHeader = true; - long last_iter_ops = 0; - Stopwatch swatch = new(); - swatch.Start(); - waiter.Set(); - const int pad = -15; - ulong resetInterval = (ulong)this.resetInterval; - - //Pull config to update slotmap periodically - Task.Run(PeriodicConfigUpdate); - - if (opts.MigrateSlotsFreq > 0) - Task.Run(MigrationBgTask); - - while (true) - { - Thread.Sleep(2000); - var epochElapsedMs = WaitForMonitorIterationSwitch(out var prev_iter); - if (printHeader) - { - printHeader = false; - if (opts.DisableConsoleLogger && opts.FileLogger == null) - { - Console.WriteLine( - $"{"min (us);",pad}" + - $"{"5th (us);",pad}" + - $"{"median (us);",pad}" + - $"{"avg (us);",pad}" + - $"{"95th (us);",pad}" + - $"{"99th (us);",pad}" + - $"{"99.9th (us);",pad}" + - $"{"total_ops;",pad}" + - $"{"iter_tops;",pad}" + - $"{"tpt (Kops/sec)",pad}"); - } - else - { - var histogramHeader = $"{"min (us);",pad}" + - $"{"5th (us);",pad}" + - $"{"median (us);",pad}" + - $"{"avg (us);",pad}" + - $"{"95th (us);",pad}" + - $"{"99th (us);",pad}" + - $"{"99.9th (us);",pad}" + - $"{"total_ops;",pad}" + - $"{"iter_tops;",pad}" + - $"{"tpt (Kops/sec)",pad}"; - logger.Log(LogLevel.Information, "{msg}", histogramHeader); - } - } - - var offset = prev_iter & 0x1; - for (int i = offset * NumThreads; i < offset * NumThreads + NumThreads; i++) - { - summary.Add(thread_histograms[i]); - thread_histograms[i].Reset(); - } - - //find operation perform during polling period - long curr_iter_ops = summary.TotalCount - last_iter_ops; - //if more than one operation per latency recording - if (opts.IntraThreadParallelism > 1 && !opts.SyncMode) - curr_iter_ops *= opts.IntraThreadParallelism; - - swatch.Stop(); - double elapsedSecs = (double)swatch.ElapsedMilliseconds - epochElapsedMs; - if (opts.DisableConsoleLogger && opts.FileLogger == null) - { - Console.WriteLine( - $"{Math.Round(summary.GetValueAtPercentile(0) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(5) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(50) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetMean() / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(95) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(99) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(99.9) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{summary.TotalCount,pad}" + - $"{curr_iter_ops,pad}" + - $"{Math.Round(BatchSize * curr_iter_ops / elapsedSecs, 2),pad}"); - } - else - { - var histogramOutput = $"{Math.Round(summary.GetValueAtPercentile(0) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(5) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(50) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetMean() / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(95) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(99) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{Math.Round(summary.GetValueAtPercentile(99.9) / OutputScalingFactor.TimeStampToMicroseconds, 2),pad}" + - $"{summary.TotalCount,pad}" + - $"{curr_iter_ops,pad}" + - $"{Math.Round(BatchSize * curr_iter_ops / elapsedSecs, 2),pad}"; - logger.Log(LogLevel.Information, "{msg}", histogramOutput); - } - - last_iter_ops = summary.TotalCount; - swatch.Reset(); - swatch.Start(); - - if (iteration % resetInterval == 0) - { - summary.Reset(); - last_iter_ops = 0; - } - - if ((ulong)runDuration == iteration) - break; - } - - while (workerCount > 0) - { - cts.Cancel(); - Thread.Yield(); - } - epoch.Dispose(); - } - - public async void OpRunnerGarnetClient(int thread_id) - { - Interlocked.Increment(ref workerCount); - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - - waiter.Wait(); - while (true) - { - if (cts.IsCancellationRequested) break; - var rand = r.Next(100); - OpType op = SelectOpType(rand); - byte[] reqKey = req.GenerateKeyBytes(out int slot); - Memory valueData = req.GenerateValueBytes(); - int clientIdx = slotMap[slot]; - - long startTimestamp = Stopwatch.GetTimestamp(); - switch (op) - { - case OpType.GET: - await gclient[clientIdx].StringGetAsMemoryAsync(reqKey); - break; - case OpType.SET: - await gclient[clientIdx].StringSetAsync(reqKey, valueData); - break; - case OpType.DEL: - await gclient[clientIdx].KeyDeleteAsync(reqKey); - break; - default: - throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"); - } - - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - RecordValue(thread_id, elapsed); - } - Interlocked.Decrement(ref workerCount); - } - - public async void OpRunnerGarnetClientParallel(int thread_id, int parallel) - { - Interlocked.Increment(ref workerCount); - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - Task[] tasks = new Task[parallel]; - int offset = 0; - - waiter.Wait(); - while (true) - { - if (cts.IsCancellationRequested) break; - var rand = r.Next(100); - OpType op = SelectOpType(rand); - byte[] reqKey = req.GenerateKeyBytes(out int slot); - Memory valueData = req.GenerateValueBytes(); - int clientIdx = slotMap[slot]; - - long startTimestamp = Stopwatch.GetTimestamp(); - tasks[offset++] = op switch - { - OpType.GET => gclient[clientIdx].StringGetAsMemoryAsync(reqKey), - OpType.SET => gclient[clientIdx].StringSetAsync(reqKey, valueData), - OpType.DEL => gclient[clientIdx].KeyDeleteAsync(reqKey), - _ => throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"), - }; - - if (offset == parallel) - { - await Task.WhenAll(tasks); - offset = 0; - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - RecordValue(thread_id, elapsed); - } - } - Interlocked.Decrement(ref workerCount); - } - - public async void OpRunnerGarnetClientSession(int thread_id) - { - Interlocked.Increment(ref workerCount); - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - var _gcs = gcs[thread_id]; - - waiter.Wait(); - while (true) - { - if (cts.IsCancellationRequested) break; - var rand = r.Next(100); - OpType op = SelectOpType(rand); - string reqKey = req.GenerateKeyInSlot(out int slot); - string valueData = req.GenerateValue(); - int clientIdx = slotMap[slot]; - - long startTimestamp = Stopwatch.GetTimestamp(); - try - { - switch (op) - { - case OpType.GET: - await _gcs[clientIdx].ExecuteAsync(["GET", reqKey]); - break; - case OpType.SET: - await _gcs[clientIdx].ExecuteAsync(["SET", reqKey, valueData]); - break; - case OpType.DEL: - await _gcs[clientIdx].ExecuteAsync(["DEL", reqKey]); - break; - default: - throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"); - } - } - catch (Exception e) - { - //if(e.ToString().Contains()) - if (e.ToString().Contains("CLUSTERDOWN")) - logger?.LogError(e, "An error has occurred"); - } - - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - RecordValue(thread_id, elapsed); - } - Interlocked.Decrement(ref workerCount); - } - - public void OpRunnerGarnetClientSessionParallel(int thread_id, int parallel) - { - Interlocked.Increment(ref workerCount); - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - int offset = 0; - var _gcs = gcs[thread_id]; - - waiter.Wait(); - while (true) - { - if (cts.IsCancellationRequested) break; - var rand = r.Next(100); - OpType op = SelectOpType(rand); - string reqKey = req.GenerateKey(out int slot); - string valueData = req.GenerateValue(); - int clientIdx = slotMap[slot]; - - long startTimestamp = Stopwatch.GetTimestamp(); - switch (op) - { - case OpType.GET: - _gcs[clientIdx].ExecuteBatch(["GET", reqKey]); - break; - case OpType.SET: - _gcs[clientIdx].ExecuteBatch(["SET", reqKey, valueData]); - break; - case OpType.DEL: - _gcs[clientIdx].ExecuteBatch(["DEL", reqKey]); - break; - default: - throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"); - } - - offset++; - if (offset == parallel) - { - for (int i = 0; i < _gcs.Length; i++) _gcs[i].CompletePending(false); - if (!opts.Burst) - for (int i = 0; i < _gcs.Length; i++) _gcs[i].Wait(); - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - offset = 0; - RecordValue(thread_id, elapsed); - } - } - Interlocked.Decrement(ref workerCount); - } - - public async void OpRunnerSERedis(int thread_id) - { - Interlocked.Increment(ref workerCount); - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - - waiter.Wait(); - while (true) - { - if (cts.IsCancellationRequested) break; - var rand = r.Next(100); - OpType op = SelectOpType(rand); - long startTimestamp = Stopwatch.GetTimestamp(); - - var rd = opts.Pool ? redisPool.Get() : redis; - var db = rd.GetDatabase(0); - switch (op) - { - case OpType.GET: - await db.StringGetAsync(req.GenerateKey()); - break; - case OpType.SET: - await db.StringSetAsync(req.GenerateKey(), req.GenerateValue()); - break; - case OpType.SETEX: - await db.StringSetAsync(req.GenerateKey(), req.GenerateValue(), TimeSpan.FromSeconds(opts.Ttl)); - break; - case OpType.DEL: - await db.KeyDeleteAsync(req.GenerateKey()); - break; - case OpType.ZADD: - { - var key = req.GenerateKey(); - var sskey = opts.SortedSetCardinality > 0 ? $"sskey{Math.Abs(HashUtils.StableHash(key)) % opts.SortedSetCardinality}" : "sskey"; - await db.SortedSetAddAsync(sskey, key, 1.0); - if (opts.Ttl > 0) - { - await db.KeyExpireAsync(sskey, TimeSpan.FromSeconds(opts.Ttl)); - } - } - break; - case OpType.ZREM: - { - var key = req.GenerateKey(); - var sskey = opts.SortedSetCardinality > 0 ? $"sskey{Math.Abs(HashUtils.StableHash(key)) % opts.SortedSetCardinality}" : "sskey"; - await db.SortedSetRemoveAsync(sskey, key); - if (opts.Ttl > 0) - { - await db.KeyExpireAsync(sskey, TimeSpan.FromSeconds(opts.Ttl)); - } - } - break; - default: - throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"); - } - if (opts.Pool) - { - redisPool.Return(rd); - } - - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - RecordValue(thread_id, elapsed); - } - Interlocked.Decrement(ref workerCount); - } - - public async void OpRunnerSERedisParallel(int thread_id, int parallel) - { - if (opts.BatchSize.First() != 1) - throw new Exception("Only batch size 1 supported for online bench"); - var req = new OnlineReqGen(thread_id, opts.DbSize, true, opts.Zipf, opts.KeyLength, opts.ValueLength, cluster: true); - Random r = new(thread_id + 100); - Task[] tasks = new Task[parallel]; - int offset = 0; - - waiter.Wait(); - while (true) - { - var rand = r.Next(100); - OpType op = SelectOpType(rand); - long startTimestamp = Stopwatch.GetTimestamp(); - - var rd = opts.Pool ? await redisPool.GetAsync() : redis; - var db = rd.GetDatabase(0); - switch (op) - { - case OpType.GET: - tasks[offset++] = db.StringGetAsync(req.GenerateKey()); - break; - case OpType.SET: - tasks[offset++] = db.StringSetAsync(req.GenerateKey(), req.GenerateValue()); - break; - case OpType.SETEX: - tasks[offset++] = db.StringSetAsync(req.GenerateKey(), req.GenerateValue(), TimeSpan.FromSeconds(opts.Ttl)); - break; - case OpType.DEL: - tasks[offset++] = db.KeyDeleteAsync(req.GenerateKey()); - break; - case OpType.ZADD: - { - var key = req.GenerateKey(); - var sskey = opts.SortedSetCardinality > 0 ? $"sskey{Math.Abs(HashUtils.StableHash(key)) % opts.SortedSetCardinality}" : "sskey"; - tasks[offset++] = Task.Run(async () => - { - await db.SortedSetAddAsync(sskey, key, 1.0); - if (opts.Ttl > 0) - { - await db.KeyExpireAsync(sskey, TimeSpan.FromSeconds(opts.Ttl)); - } - }); - } - break; - case OpType.ZREM: - { - var key = req.GenerateKey(); - var sskey = opts.SortedSetCardinality > 0 ? $"sskey{Math.Abs(HashUtils.StableHash(key)) % opts.SortedSetCardinality}" : "sskey"; - tasks[offset++] = Task.Run(async () => - { - await db.SortedSetRemoveAsync(sskey, key); - if (opts.Ttl > 0) - { - await db.KeyExpireAsync(sskey, TimeSpan.FromSeconds(opts.Ttl)); - } - }); - } - break; - default: - throw new Exception($"opType: {op} benchmark not supported with {opts.Client} ClientType!"); - } - if (opts.Pool) - { - redisPool.Return(rd); - } - - if (offset == parallel) - { - await Task.WhenAll(tasks); - offset = 0; - long elapsed = Stopwatch.GetTimestamp() - startTimestamp; - RecordValue(thread_id, elapsed); - } - } - } - } -} \ No newline at end of file diff --git a/playground/ClusterStress/ShardedRespPerfBench.cs b/playground/ClusterStress/ShardedRespPerfBench.cs deleted file mode 100644 index 1550f966973..00000000000 --- a/playground/ClusterStress/ShardedRespPerfBench.cs +++ /dev/null @@ -1,428 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Diagnostics; -using System.Linq; -using System.Net; -using System.Threading; -using Garnet.common; -using StackExchange.Redis; - -namespace Resp.benchmark -{ - internal struct ClusterShardConnection - { - public LightClient primary; - public LightClient[] replicas; - public int count; - } - - /// - /// Dummy clients issuing commands as fast as possible, with varying number of - /// threads, to stress server side. - /// - public class ShardedRespPerfBench - { - readonly Options opts; - readonly int Start; - readonly ManualResetEventSlim waiter = new(); - - readonly ClusterConfiguration clusterConfig; - readonly ushort[] slotMap = new ushort[16384]; - - volatile bool done = false; - long total_ops_done = 0; - - public ShardedRespPerfBench(Options opts, int Start) - { - this.opts = opts; - this.Start = Start; - clusterConfig = GetClusterConfig(); - InitSlotMap(); - PrintClusterConfig(); - CheckAllSlotsCovered(); - } - - private ClusterConfiguration GetClusterConfig() - { - using var redis = ConnectionMultiplexer.Connect(BenchUtils.GetConfig(opts.Address, opts.Port, useTLS: opts.EnableTLS, tlsHost: opts.TlsHost, allowAdmin: true)); - var clusterConfig = redis.GetServer(opts.Address + ":" + opts.Port).ClusterNodes(); - return clusterConfig; - } - - private void CheckAllSlotsCovered() - { - var nodes = clusterConfig.Nodes.ToArray(); - - var slotMap = new byte[16384]; - foreach (var node in nodes) - { - var slotRanges = node.Slots; - foreach (var slotRange in slotRanges) - { - for (int i = slotRange.From; i <= slotRange.To; i++) - slotMap[i] = 1; - } - } - - for (int i = 0; i < slotMap.Length; i++) - { - if (slotMap[i] == 0) - { - throw new Exception($"Slot {i} not covered"); - } - } - } - - private void InitSlotMap() - { - var nodes = clusterConfig.Nodes.ToArray(); - ushort j = 0; - foreach (var node in nodes) - { - var slotRanges = node.Slots; - foreach (var slotRange in slotRanges) - for (int i = slotRange.From; i <= slotRange.To; i++) - slotMap[i] = j; - j++; - } - } - - private unsafe ClusterShardConnection[] InitConnections(OpType opType, int bufferSize) - { - var lighClientOnResponseDelegate = new LightClient.OnResponseDelegateUnsafe(ReqGen.OnResponse); - var pNodes = clusterConfig.Nodes.ToList().FindAll(p => !p.IsReplica).ToArray(); - ClusterShardConnection[] clusterShards = new ClusterShardConnection[pNodes.Length]; - for (int i = 0; i < pNodes.Length; i++) - { - clusterShards[i].count = 1 + (opts.ReplicaReads ? pNodes[i].Children.Count : 0); - IPEndPoint pEndpoint = (IPEndPoint)pNodes[i].EndPoint; - clusterShards[i].primary = new LightClient( - pEndpoint, - (int)opType, - lighClientOnResponseDelegate, - bufferSize, - opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null); - clusterShards[i].primary.Connect(); - clusterShards[i].primary.Authenticate(opts.Auth); - - if (opts.ReplicaReads) - { - clusterShards[i].replicas = new LightClient[pNodes[i].Children.Count]; - for (int j = 0; j < clusterShards[i].replicas.Length; j++) - { - IPEndPoint rEndpoint = (IPEndPoint)pNodes[i].Children[j].EndPoint; - clusterShards[i].replicas[j] = new LightClient( - rEndpoint, - (int)opType, - lighClientOnResponseDelegate, - bufferSize, - opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null); - clusterShards[i].replicas[j].Connect(); - clusterShards[i].replicas[j].Authenticate(opts.Auth); - clusterShards[i].replicas[j].ReadOnly(); - } - } - } - return clusterShards; - } - - private ref LightClient GetConnection(ref Random r, ref ClusterShardConnection[] clusterShards, int slot, bool readOnly = false) - { - int shardIdx = slotMap[slot]; - if (opts.ReplicaReads) - { - int nodeIdx = readOnly ? r.Next(0, clusterShards[shardIdx].count) : 0; - if (nodeIdx == 0) - return ref clusterShards[shardIdx].primary; - else - return ref clusterShards[shardIdx].replicas[nodeIdx - 1]; - } - else - { - return ref clusterShards[shardIdx].primary; - } - } - - private static bool IsReadOnly(OpType opType) - { - return opType switch - { - OpType.MSET => false, - OpType.GET => true, - _ => throw new Exception($"{opType} not supported for cluster benchmark!") - }; - } - - private void PrintClusterConfig() - { - Console.WriteLine("Cluster Retrieved Configuration..."); - var nodes = clusterConfig.Nodes.ToArray(); - Array.Sort(nodes, (x, y) => ((IPEndPoint)x.EndPoint).Address.ToString().CompareTo(((IPEndPoint)y.EndPoint).Address.ToString())); - foreach (var node in nodes) - { - var endpoint = (IPEndPoint)node.EndPoint; - Console.Write($"host: {endpoint.Address}:{endpoint.Port}, "); - Console.Write($"role: {((node.IsReplica || node.Slots.Count == 0) ? "REPLICA" : "PRIMARY")}, "); - var slotRanges = node.Slots; - - Console.Write("slotRanges: "); - foreach (var slotRange in slotRanges) - Console.Write($"{slotRange.From} - {slotRange.To} "); - Console.WriteLine(""); - } - Console.WriteLine("<--------------------------------------------->"); - } - - /// - /// Load DB with DbSize keys, starting from Start; specified #threads - /// e.g., "0" => "0", "1" => "1", and so on - /// - /// - /// - /// - /// - /// - public void LoadData( - int loadDbThreads = 8, - int BatchSize = 1 << 12, - int keyLen = default, - int valueLen = default, - bool numericValue = false) - { - loadDbThreads = opts.DbSize / loadDbThreads > BatchSize ? loadDbThreads : 1; - var rg = new ReqGen( - Start, - opts.DbSize, - NumOps: opts.DbSize, - BatchSize: BatchSize, - opType: OpType.MSET, - clusterConfig: clusterConfig, - shard: opts.Shard, - randomGen: false, - randomServe: false, - keyLen, - valueLen, - numericValue, - verbose: true, - flatBufferClient: false, - ttl: opts.Ttl); - rg.GenerateForCluster(); - - LightOperate( - opType: OpType.MSET, - TotalOps: opts.DbSize, - BatchSize: BatchSize, - NumThreads: loadDbThreads, - OpsPerThread: opts.DbSize / loadDbThreads, - runTime: default, - rg: rg, - randomGen: false, - randomServe: false, - keyLen: keyLen, - valueLen: valueLen); - } - - public void Run( - OpType opType, - int TotalOps, - int[] NumThreads, - int BatchSize = 1 << 12, - TimeSpan runTime = default, - bool randomGen = true, - bool randomServe = true, - int keyLen = default, - int valueLen = default, - int ttl = 0) - { - var rg = new ReqGen( - Start, - opts.DbSize, - TotalOps, - BatchSize, - opType, - clusterConfig, - shard: opts.Shard, - randomGen, - randomServe, - keyLen, - valueLen, - ttl: ttl); - rg.GenerateForCluster(); - - foreach (var numThread in NumThreads) - { - GC.Collect(); - GC.WaitForFullGCComplete(); - LightOperate(opType, TotalOps, BatchSize, numThread, 0, runTime, rg, randomGen, randomServe, burst: opts.Burst); - } - } - - public ReqGen LightOperate( - OpType opType, - int TotalOps, - int BatchSize, - int NumThreads, - int OpsPerThread = 0, - TimeSpan runTime = default, - ReqGen rg = null, - bool randomGen = true, - bool randomServe = true, - int keyLen = default, - int valueLen = default, - bool numericValue = false, - bool verbose = true, - bool burst = false) - { - if (rg == null) - { - rg = new ReqGen( - Start, - opts.DbSize, - TotalOps, - BatchSize, - opType, - clusterConfig: clusterConfig, - shard: opts.Shard, - randomGen, - randomServe, - keyLen, - valueLen, - numericValue, - verbose, - flatBufferClient: (opts.Client == ClientType.SERedis || opts.Client == ClientType.GarnetClientSession), - ttl: opts.Ttl); - rg.GenerateForCluster(); - } - - if (verbose) - { - Console.WriteLine(); - Console.WriteLine($"Operation type: {opType}"); - Console.WriteLine($"Num threads: {NumThreads}"); - } - - // Query database - Thread[] workers = new Thread[NumThreads]; - - // Run the experiment. - for (int idx = 0; idx < NumThreads; ++idx) - { - int x = idx; - workers[idx] = opts.Client switch - { - ClientType.LightClient => opts.ReplicaReads ? new Thread(() => LightOperateThreadRunnerClusterReplicaReads(x, OpsPerThread, opType, rg, burst)) : - new Thread(() => LightOperateThreadRunnerCluster(x, OpsPerThread, opType, rg, burst)), - _ => throw new Exception($"ClientType {opts.Client} not supported"), - }; - } - - // Start threads. - foreach (Thread worker in workers) - worker.Start(); - - waiter.Set(); - - Stopwatch swatch = new(); - swatch.Start(); - if (OpsPerThread == 0) - { - if (runTime == default) runTime = TimeSpan.FromSeconds(15); // default - Thread.Sleep(runTime); - done = true; - } - foreach (Thread worker in workers) - worker.Join(); - - swatch.Stop(); - - double seconds = swatch.ElapsedMilliseconds / 1000.0; - double opsPerSecond = total_ops_done / seconds; - - if (verbose) - { - Console.WriteLine($"Total time: {swatch.ElapsedMilliseconds:N2}ms for {total_ops_done:N2} ops"); - Console.WriteLine($"Throughput: {opsPerSecond:N2} ops/sec"); - } - - done = false; - total_ops_done = 0; - waiter.Reset(); - - return rg; - } - - private unsafe void LightOperateThreadRunnerCluster(int threadid, int NumOps, OpType opType, ReqGen rg, bool burst) - { - var lighClientOnResponseDelegate = new LightClient.OnResponseDelegateUnsafe(ReqGen.OnResponse); - var nodes = clusterConfig?.Nodes.ToArray(); - LightClient[] clients = new LightClient[clusterConfig.Nodes.Count]; - for (int i = 0; i < clients.Length; i++) - { - var endpoint = (IPEndPoint)nodes[i].EndPoint; - clients[i] = new LightClient( - endpoint, - (int)opType, - lighClientOnResponseDelegate, - rg.GetBufferSize(), - opts.EnableTLS ? BenchUtils.GetTlsOptions(opts.TlsHost, opts.CertFileName, opts.CertPassword) : null); - - clients[i].Connect(); - clients[i].Authenticate(opts.Auth); - } - - int maxReqs = (NumOps / rg.BatchCount); - int numReqs = 0; - - waiter.Wait(); - - Random r = new(threadid); - Stopwatch sw = new(); - sw.Start(); - while (!done) - { - byte[] buf = rg.GetRequestInterleaved(ref r, out var len, out var slot); - int clientIdx = slotMap[slot]; - - if (burst) clients[clientIdx].CompletePendingRequests(); - clients[clientIdx].Send(buf, len, (opType == OpType.MSET || opType == OpType.MPFADD) ? 1 : rg.BatchCount); - if (!burst) clients[clientIdx].CompletePendingRequests(); - numReqs++; - if (numReqs == maxReqs) break; - } - sw.Stop(); - - Interlocked.Add(ref total_ops_done, numReqs * rg.BatchCount); - } - - private unsafe void LightOperateThreadRunnerClusterReplicaReads(int threadid, int NumOps, OpType opType, ReqGen rg, bool burst) - { - var clusterShards = InitConnections(opType, rg.GetBufferSize()); - int maxReqs = (NumOps / rg.BatchCount); - int numReqs = 0; - //Console.WriteLine($"{NumOps}:{rg.BatchCount}:{maxReqs}"); - - waiter.Wait(); - - Random r = new(threadid); - Stopwatch sw = new(); - sw.Start(); - while (!done) - { - //byte[] buf = rg.GetRequest(out var len, out var slot); - byte[] buf = rg.GetRequestInterleaved(ref r, out var len, out var slot); - ref LightClient client = ref GetConnection(ref r, ref clusterShards, slot, readOnly: IsReadOnly(opType)); - - if (burst) client.CompletePendingRequests(); - client.Send(buf, len, (opType == OpType.MSET || opType == OpType.MPFADD) ? 1 : rg.BatchCount); - if (!burst) client.CompletePendingRequests(); - numReqs++; - if (numReqs == maxReqs) break; - } - sw.Stop(); - - Interlocked.Add(ref total_ops_done, numReqs * rg.BatchCount); - } - } -} \ No newline at end of file diff --git a/playground/CommandInfoUpdater/GarnetCommandsDocs.json b/playground/CommandInfoUpdater/GarnetCommandsDocs.json index 2ce569c4f83..29d78cf4328 100644 --- a/playground/CommandInfoUpdater/GarnetCommandsDocs.json +++ b/playground/CommandInfoUpdater/GarnetCommandsDocs.json @@ -1066,7 +1066,7 @@ { "Command": "GETIFNOTMATCH", "Name": "GETIFNOTMATCH", - "Summary": "Gets the ETag and value if the key\u0027s current etag does not match the given etag.", + "Summary": "Gets the ETag and value if the key's current etag does not match the given etag.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -1236,7 +1236,7 @@ { "Command": "SET", "Name": "SET", - "Summary": "Sets the string value of a key, ignoring its type. The key is created if it doesn\u0027t exist.", + "Summary": "Sets the string value of a key, ignoring its type. The key is created if it doesn't exist.", "Group": "String", "Complexity": "O(1)", "Arguments": [ @@ -1452,6 +1452,50 @@ } ] }, + { + "Command": "SETWITHETAG", + "Name": "SETWITHETAG", + "Summary": "Sets a key-value pair with an ETag. If the key already exists, the value is overwritten and the ETag is incremented. Returns the ETag.", + "Group": "String", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE", + "DisplayText": "value", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "EXPIRATION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "SECONDS", + "DisplayText": "seconds", + "Type": "Integer", + "Token": "EX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MILLISECONDS", + "DisplayText": "milliseconds", + "Type": "Integer", + "Token": "PX" + } + ] + } + ] + }, { "Command": "CLUSTER", "Name": "CLUSTER", @@ -1988,5 +2032,317 @@ ] } ] + }, + { + "Command": "RICREATE", + "Name": "RI.CREATE", + "Summary": "Creates a new RangeIndex backed by a BfTree. Returns an error if the key already exists.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "BACKEND", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "DISK", + "DisplayText": "disk", + "Type": "PureToken", + "Token": "DISK" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMORY", + "DisplayText": "memory", + "Type": "PureToken", + "Token": "MEMORY" + } + ] + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "CACHESIZE", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "CACHESIZE", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MINRECORD", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MINRECORD", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MAXRECORD", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MAXRECORD", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MAXKEYLEN", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "MAXKEYLEN", + "ArgumentFlags": "Optional" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "PAGESIZE", + "DisplayText": "bytes", + "Type": "Integer", + "Token": "PAGESIZE", + "ArgumentFlags": "Optional" + } + ] + }, + { + "Command": "RICONFIG", + "Name": "RI.CONFIG", + "Summary": "Returns the configuration of a RangeIndex as alternating field-value pairs.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIDEL", + "Summary": "Deletes an entry from a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + } + ] + }, + { + "Command": "RIEXISTS", + "Name": "RI.EXISTS", + "Summary": "Checks whether a key exists and is a RangeIndex. Returns 1 if true, 0 otherwise.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIGET", + "Summary": "Gets the value of an entry in a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + } + ] + }, + { + "Command": "RIMETRICS", + "Name": "RI.METRICS", + "Summary": "Returns runtime metrics for a RangeIndex including tree handle status and flags.", + "Group": "Generic", + "Complexity": "O(1)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + } + ] + }, + { + "Command": "RIRANGE", + "Summary": "Returns entries in a key range from a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N + M) where M is the number of entries returned", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "START", + "DisplayText": "start", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "END", + "DisplayText": "end", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS_OPTION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Token": "FIELDS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KEY_ONLY", + "DisplayText": "KEY", + "Type": "PureToken", + "Token": "KEY" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE_ONLY", + "DisplayText": "VALUE", + "Type": "PureToken", + "Token": "VALUE" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "BOTH", + "DisplayText": "BOTH", + "Type": "PureToken", + "Token": "BOTH" + } + ] + } + ] + }, + { + "Command": "RISCAN", + "Name": "RI.SCAN", + "Summary": "Scans entries starting from a key in a RangeIndex with a count limit.", + "Group": "Generic", + "Complexity": "O(log N + M) where M is the count", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "START", + "DisplayText": "start", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "COUNT", + "DisplayText": "count", + "Type": "Integer", + "Token": "COUNT" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FIELDS_OPTION", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Token": "FIELDS", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KEY_ONLY", + "DisplayText": "KEY", + "Type": "PureToken", + "Token": "KEY" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE_ONLY", + "DisplayText": "VALUE", + "Type": "PureToken", + "Token": "VALUE" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "BOTH", + "DisplayText": "BOTH", + "Type": "PureToken", + "Token": "BOTH" + } + ] + } + ] + }, + { + "Command": "RISET", + "Name": "RI.SET", + "Summary": "Inserts or updates an entry in a RangeIndex.", + "Group": "Generic", + "Complexity": "O(log N)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FIELD", + "DisplayText": "field", + "Type": "String" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "VALUE", + "DisplayText": "value", + "Type": "String" + } + ] } ] \ No newline at end of file diff --git a/playground/CommandInfoUpdater/GarnetCommandsInfo.json b/playground/CommandInfoUpdater/GarnetCommandsInfo.json index afb17f2c2e5..720de6fb55c 100644 --- a/playground/CommandInfoUpdater/GarnetCommandsInfo.json +++ b/playground/CommandInfoUpdater/GarnetCommandsInfo.json @@ -25,19 +25,6 @@ "Tips": null, "KeySpecifications": null, "SubCommands": [ - { - "Command": "CLUSTER_AOFSYNC", - "Name": "CLUSTER|AOFSYNC", - "IsInternal": true, - "Arity": 3, - "Flags": "Admin, NoScript, NoMulti", - "FirstKey": 0, - "LastKey": 0, - "Step": 0, - "AclCategories": "Admin, Dangerous, Slow, Garnet", - "KeySpecifications": null, - "SubCommands": null - }, { "Command": "CLUSTER_APPENDLOG", "Name": "CLUSTER|APPENDLOG", @@ -216,15 +203,15 @@ "SubCommands": null }, { - "Command": "CLUSTER_RESERVE", - "Name": "CLUSTER|RESERVE", + "Command": "CLUSTER_MLOG_KEY_TIME", + "Name": "CLUSTER|MLOG_KEY_TIME", "IsInternal": true, - "Arity": 4, + "Arity": -2, "Flags": "Admin, NoScript, NoMulti", "FirstKey": 0, "LastKey": 0, "Step": 0, - "AclCategories": "Admin, Dangerous, Garnet", + "AclCategories": "Admin, Dangerous, Slow, Garnet", "KeySpecifications": null, "SubCommands": null }, @@ -317,6 +304,19 @@ "KeySpecifications": null, "SubCommands": null }, + { + "Command": "CLUSTER_SNAPSHOT_DATA", + "Name": "CLUSTER|SNAPSHOT_DATA", + "IsInternal": true, + "Arity": 6, + "Flags": "Admin, NoScript, NoMulti", + "FirstKey": 0, + "LastKey": 0, + "Step": 0, + "AclCategories": "Admin, Dangerous, Slow, Garnet", + "KeySpecifications": null, + "SubCommands": null + }, { "Command": "CLUSTER_SETSLOTSRANGE", "Name": "CLUSTER|SETSLOTSRANGE", @@ -1076,6 +1076,32 @@ } ] }, + { + "Command": "SETWITHETAG", + "Name": "SETWITHETAG", + "IsInternal": false, + "Arity": -3, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, String, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert, Update" + } + ] + }, { "Command": "SPUBLISH", "Name": "SPUBLISH", @@ -1426,5 +1452,255 @@ "Flags": "RO, Access" } ] + }, + { + "Command": "RICREATE", + "Name": "RI.CREATE", + "IsInternal": false, + "Arity": -2, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Write, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert" + } + ], + "StoreType": "Main" + }, + { + "Command": "RICONFIG", + "Name": "RI.CONFIG", + "IsInternal": false, + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIDEL", + "Name": "RI.DEL", + "IsInternal": false, + "Arity": 3, + "Flags": "Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Write, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Delete" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIEXISTS", + "Name": "RI.EXISTS", + "IsInternal": false, + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIGET", + "IsInternal": false, + "Arity": 3, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIMETRICS", + "Name": "RI.METRICS", + "IsInternal": false, + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RIRANGE", + "IsInternal": false, + "Arity": -4, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RISCAN", + "Name": "RI.SCAN", + "IsInternal": false, + "Arity": -5, + "Flags": "ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Slow, Read, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ], + "StoreType": "Main" + }, + { + "Command": "RISET", + "Name": "RI.SET", + "IsInternal": false, + "Arity": 4, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, Write, Garnet", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RW, Insert" + } + ], + "StoreType": "Main" } ] \ No newline at end of file diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 20b2ed74a77..c33b0c29807 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -59,7 +59,7 @@ public class SupportedCommand [ new("CLUSTER|ADDSLOTS", RespCommand.CLUSTER_ADDSLOTS), new("CLUSTER|ADDSLOTSRANGE", RespCommand.CLUSTER_ADDSLOTSRANGE), - new("CLUSTER|AOFSYNC", RespCommand.CLUSTER_AOFSYNC), + new("CLUSTER|ADVANCE_TIME", RespCommand.CLUSTER_ADVANCE_TIME), new("CLUSTER|APPENDLOG", RespCommand.CLUSTER_APPENDLOG), new("CLUSTER|ATTACH_SYNC", RespCommand.CLUSTER_ATTACH_SYNC), new("CLUSTER|BANLIST", RespCommand.CLUSTER_BANLIST), @@ -93,15 +93,16 @@ public class SupportedCommand new("CLUSTER|REPLICAS", RespCommand.CLUSTER_REPLICAS), new("CLUSTER|REPLICATE", RespCommand.CLUSTER_REPLICATE), new("CLUSTER|RESET", RespCommand.CLUSTER_RESET), - new("CLUSTER|RESERVE", RespCommand.CLUSTER_RESERVE), new("CLUSTER|SEND_CKPT_FILE_SEGMENT", RespCommand.CLUSTER_SEND_CKPT_FILE_SEGMENT), new("CLUSTER|SEND_CKPT_METADATA", RespCommand.CLUSTER_SEND_CKPT_METADATA), + new("CLUSTER|SNAPSHOT_DATA", RespCommand.CLUSTER_SNAPSHOT_DATA), new("CLUSTER|SET-CONFIG-EPOCH", RespCommand.CLUSTER_SETCONFIGEPOCH), new("CLUSTER|SETSLOT", RespCommand.CLUSTER_SETSLOT), new("CLUSTER|SETSLOTSRANGE", RespCommand.CLUSTER_SETSLOTSRANGE), new("CLUSTER|SHARDS", RespCommand.CLUSTER_SHARDS), new("CLUSTER|SLOTS", RespCommand.CLUSTER_SLOTS), new("CLUSTER|SLOTSTATE", RespCommand.CLUSTER_SLOTSTATE), + new("CLUSTER|MLOG_KEY_TIME", RespCommand.CLUSTER_MLOG_KEY_TIME), new("CLUSTER|SYNC", RespCommand.CLUSTER_SYNC), ]), new("COMMAND", RespCommand.COMMAND, StoreType.None, @@ -255,6 +256,15 @@ public class SupportedCommand new("RESTORE", RespCommand.RESTORE, StoreType.All), new("RENAMENX", RespCommand.RENAMENX, StoreType.All), new("REPLICAOF", RespCommand.REPLICAOF), + new("RI.CONFIG", RespCommand.RICONFIG, StoreType.Main), + new("RI.CREATE", RespCommand.RICREATE, StoreType.Main), + new("RI.DEL", RespCommand.RIDEL, StoreType.Main), + new("RI.EXISTS", RespCommand.RIEXISTS, StoreType.Main), + new("RI.GET", RespCommand.RIGET, StoreType.Main), + new("RI.METRICS", RespCommand.RIMETRICS, StoreType.Main), + new("RI.RANGE", RespCommand.RIRANGE, StoreType.Main), + new("RI.SCAN", RespCommand.RISCAN, StoreType.Main), + new("RI.SET", RespCommand.RISET, StoreType.Main), new("ROLE", RespCommand.ROLE), new("RPOP", RespCommand.RPOP, StoreType.Object), new("RPOPLPUSH", RespCommand.RPOPLPUSH, StoreType.Object), @@ -274,6 +284,7 @@ public class SupportedCommand new("SETEX", RespCommand.SETEX, StoreType.Main), new("SETIFMATCH", RespCommand.SETIFMATCH, StoreType.Main), new("SETIFGREATER", RespCommand.SETIFGREATER, StoreType.Main), + new("SETWITHETAG", RespCommand.SETWITHETAG, StoreType.Main), new("SETNX", RespCommand.SETNX, StoreType.Main), new("SETRANGE", RespCommand.SETRANGE, StoreType.Main), new("SISMEMBER", RespCommand.SISMEMBER, StoreType.Object), diff --git a/playground/TstRunner/TstRunner.csproj b/playground/TstRunner/TstRunner.csproj index 758be9e7dea..248b5ba8863 100644 --- a/playground/TstRunner/TstRunner.csproj +++ b/playground/TstRunner/TstRunner.csproj @@ -1,4 +1,4 @@ - + Exe @@ -9,8 +9,6 @@ - - @@ -20,8 +18,10 @@ - - + + + + diff --git a/samples/ETag/Caching.cs b/samples/ETag/Caching.cs index 74cc75c6843..1df28e864c1 100644 --- a/samples/ETag/Caching.cs +++ b/samples/ETag/Caching.cs @@ -112,7 +112,7 @@ static async Task SeedCache(Dictionary localApplicatio string key = i.ToString(); MovieReview movieReview = MovieReview.CreateRandomReview(random); string value = JsonSerializer.Serialize(movieReview); - long etag = (long)await db.ExecuteAsync("SET", key, value, "WITHETAG"); + long etag = (long)await db.ExecuteAsync("SETWITHETAG", key, value); localApplicationState.Add(i, (etag, movieReview)); Console.WriteLine($"Seeded {i}"); } diff --git a/samples/ETag/OccSimulation.cs b/samples/ETag/OccSimulation.cs index 2419a0dc6d3..6f67530449c 100644 --- a/samples/ETag/OccSimulation.cs +++ b/samples/ETag/OccSimulation.cs @@ -62,7 +62,7 @@ public static async Task RunSimulation() string serializedUserInfo = JsonSerializer.Serialize(userInfo); // Seed the item in the database - long initialEtag = (long)await db.ExecuteAsync("SET", userKey, serializedUserInfo, "WITHETAG"); + long initialEtag = (long)await db.ExecuteAsync("SETWITHETAG", userKey, serializedUserInfo); // Cancellation token is used to exit program on end of interactive repl var cts = new CancellationTokenSource(); diff --git a/samples/MetricsMonitor/Options.cs b/samples/MetricsMonitor/Options.cs index e64e4fd7fa5..06db489e403 100644 --- a/samples/MetricsMonitor/Options.cs +++ b/samples/MetricsMonitor/Options.cs @@ -26,7 +26,7 @@ public class Options [Option("latency-metrics-type", Required = false, Default = LatencyMetricsType.NET_RS_LAT, HelpText = "Latency metrics types to track (NET_RS_LAT)")] public LatencyMetricsType LatencyEvent { get; set; } - [Option("info-metrics-type", Required = false, Default = InfoMetricsType.STATS, HelpText = "Info metrics types to track (SERVER, MEMORY, CLUSTER, STATS, STORE, OBJECTSTORE, ALL)")] + [Option("info-metrics-type", Required = false, Default = InfoMetricsType.STATS, HelpText = "Info metrics types to track (SERVER, MEMORY, CLUSTER, STATS, STORE, ALL)")] public InfoMetricsType infoType { get; set; } [Option("metrics", Required = false, Default = Metric.INFO, HelpText = "What type of server side metrics to retrieve (LATENCY, INFO)")] diff --git a/test/Allure/GenerateAllureReport.ps1 b/test/Allure/GenerateAllureReport.ps1 deleted file mode 100644 index 5ba985c8f84..00000000000 --- a/test/Allure/GenerateAllureReport.ps1 +++ /dev/null @@ -1,56 +0,0 @@ -#Requires -Version 7 - -<# -.SYNOPSIS - This script is called after all the Allure data is merged into one location and generates the Allure report. - - It is getting the data from the test/Allure/CombinedResults directory and generating the report into the test/Allure/allure-report directory. - - NOTE: Preserving history between runs is handled in the GitHub Actions workflow by downloading and uploading the history folder to allure_data_history branch (test/Allure/history). -#> - -$OFS = "`r`n" - -# Get base path since paths can differ from machine to machine -$pathstring = $pwd.Path -if ($pathstring.Contains("test")) { - $position = $pathString.IndexOf("test") - $basePath = $pathstring.Substring(0,$position-1) # take off slash off end as well -} else { - $basePath = $pathstring # already in base and not in test - Set-Location .\test\Allure\ -} - -# Location of all the allure results -$allureResultsCombinedDir = "$basePath/test/Allure/CombinedResults" - -# Double check combined results dir exists -if (-not (Test-Path -Path $allureResultsCombinedDir)) { - Write-Error -Message "The Combined results directory $allureResultsCombinedDir does not exist. " -Category ObjectNotFound - exit 1 -} - -# Copy categories.json to the CombinedResults directory -Write-Host "Copying categories.json to $allureResultsCombinedDir" -Copy-Item -Path "$basePath/test/Allure/categories.json" -Destination "$allureResultsCombinedDir/categories.json" - -# Generate the report -Write-Host "Generate the Allure report from $allureResultsCombinedDir" -allure generate CombinedResults -o allure-report --clean - -# verify report generated -$reportDir = "$basePath/test/Allure/allure-report" -if (-not (Test-Path -Path $reportDir)) { - Write-Error -Message "The Allure report directory $reportDir did not get created." -Category ObjectNotFound - exit 1 -} -else { - Write-Host "Allure report generated successfully at $reportDir. Use 'allure open allure-report' to view it locally." -} - -Write-Output "************************" -Write-Output "**" -Write-Output "** Done!" -Write-Output "**" -Write-Output "************************" - diff --git a/test/Allure/categories.json b/test/Allure/categories.json deleted file mode 100644 index 9cb55af5042..00000000000 --- a/test/Allure/categories.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "name": "Assertion Failures", - "matchedStatuses": ["assert"], - "messageRegex": ".*expected.*but found.*" - }, - { - "name": "Connection Failures", - "matchedStatuses": ["failed"], - "messageRegex": ".*Failed to connect within.*" - }, - { - "name": "All Other Failures", - "matchedStatuses": ["failed"], - "messageRegex": ".*" - }, - { - "name": "All Broken", - "matchedStatuses": ["broken"], - "messageRegex": ".*" - }, - { - "name": "All Skipped", - "matchedStatuses": ["skipped"], - "messageRegex": ".*" - }, - { - "name": "All Unknown", - "matchedStatuses": ["unknown"], - "messageRegex": ".*" - } -] \ No newline at end of file diff --git a/test/BfTreeInterop.test/BfTreeInterop.test.csproj b/test/BfTreeInterop.test/BfTreeInterop.test.csproj new file mode 100644 index 00000000000..18ca2259b69 --- /dev/null +++ b/test/BfTreeInterop.test/BfTreeInterop.test.csproj @@ -0,0 +1,20 @@ + + + + true + 1701;1702;1591 + false + true + + + + + + + + + + + + + diff --git a/test/BfTreeInterop.test/BfTreeInteropTests.cs b/test/BfTreeInterop.test/BfTreeInteropTests.cs new file mode 100644 index 00000000000..060fb7a0065 --- /dev/null +++ b/test/BfTreeInterop.test/BfTreeInteropTests.cs @@ -0,0 +1,579 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Garnet.server.BfTreeInterop; +using NUnit.Framework; + +namespace BfTreeInterop.test +{ + /// + /// Integration tests for the bftree-garnet native FFI interop layer. + /// Tests all core BfTree APIs: lifecycle, point operations, scans, and snapshot/recovery. + /// + [TestFixture] + public class BfTreeInteropTests + { + private BfTreeService _tree; + private string _treePath; + + [SetUp] + public void Setup() + { + _treePath = Path.Combine( + Path.GetTempPath(), $"bftree_test_{Guid.NewGuid():N}.bftree"); + _tree = new BfTreeService( + filePath: _treePath, + cbMinRecordSize: 4); + } + + [TearDown] + public void TearDown() + { + _tree?.Dispose(); + if (_treePath != null && File.Exists(_treePath)) + File.Delete(_treePath); + } + + // --------------------------------------------------------------- + // Lifecycle tests + // --------------------------------------------------------------- + + [Test] + public void CreateAndDispose() + { + var path = Path.Combine(Path.GetTempPath(), $"bftree_t_{Guid.NewGuid():N}.bftree"); + try + { + using var tree = new BfTreeService(filePath: path, cbMinRecordSize: 4); + Assert.Pass(); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + [Test] + public void CreateWithCustomConfig() + { + var path = Path.Combine(Path.GetTempPath(), $"bftree_t_{Guid.NewGuid():N}.bftree"); + try + { + using var tree = new BfTreeService( + filePath: path, + cbSizeByte: 16 * 1024 * 1024, + cbMinRecordSize: 8, + cbMaxRecordSize: 4096, + cbMaxKeyLen: 128, + leafPageSize: 16384); + Assert.Pass(); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + [Test] + public void CreateMemoryOnly() + { + using var tree = new BfTreeService( + storageBackend: StorageBackendType.Memory, + cbMinRecordSize: 4); + var insertResult = tree.Insert("testkey"u8, "testval"u8); + Assert.That(insertResult, Is.EqualTo(BfTreeInsertResult.Success)); + } + + [Test] + public void CreateDiskBacked_MissingPath_Throws() + { + Assert.Throws(() => new BfTreeService(filePath: null)); + } + + [Test] + public void DoubleDispose_DoesNotThrow() + { + var path = Path.Combine(Path.GetTempPath(), $"bftree_t_{Guid.NewGuid():N}.bftree"); + try + { + var tree = new BfTreeService(filePath: path, cbMinRecordSize: 4); + tree.Dispose(); + Assert.DoesNotThrow(() => tree.Dispose()); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + // --------------------------------------------------------------- + // Insert tests + // --------------------------------------------------------------- + + [Test] + public void InsertAndRead_BasicRoundTrip() + { + var key = "user:1001"u8; + var value = "Alice"u8; + + var insertResult = _tree.Insert(key, value); + Assert.That(insertResult, Is.EqualTo(BfTreeInsertResult.Success)); + + var readResult = _tree.Read(key, out var readValue); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Found)); + Assert.That(readValue, Is.EqualTo(value.ToArray())); + } + + [Test] + public void InsertOverwrite_ReturnsUpdatedValue() + { + var key = "mykey"u8; + + _tree.Insert(key, "value1"u8); + _tree.Insert(key, "value2"u8); + + var readResult = _tree.Read(key, out var value); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Found)); + Assert.That(value, Is.EqualTo("value2"u8.ToArray())); + } + + [Test] + public void InsertMultiple_AllReadable() + { + for (int i = 0; i < 100; i++) + { + var key = Encoding.UTF8.GetBytes($"key:{i:D4}"); + var value = Encoding.UTF8.GetBytes($"value:{i}"); + var result = _tree.Insert(key, value); + Assert.That(result, Is.EqualTo(BfTreeInsertResult.Success)); + } + + for (int i = 0; i < 100; i++) + { + var key = Encoding.UTF8.GetBytes($"key:{i:D4}"); + var expectedValue = Encoding.UTF8.GetBytes($"value:{i}"); + var readResult = _tree.Read(key, out var readValue); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Found)); + Assert.That(readValue, Is.EqualTo(expectedValue)); + } + } + + // --------------------------------------------------------------- + // Read tests + // --------------------------------------------------------------- + + [Test] + public void ReadNotFound() + { + var readResult = _tree.Read("nonexistent"u8, out var value); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.NotFound)); + Assert.That(value, Is.Empty); + } + + [Test] + public void ReadAfterDelete_ReturnsDeleted() + { + var key = "deleteme"u8; + _tree.Insert(key, "value"u8); + _tree.Delete(key); + + var readResult = _tree.Read(key, out var value); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Deleted)); + Assert.That(value, Is.Empty); + } + + [Test] + public void ReadIntoSpan_ZeroAlloc() + { + var key = "spankey"u8; + var expected = "spanvalue"u8; + _tree.Insert(key, expected); + + Span buffer = stackalloc byte[256]; + var result = _tree.Read(key, buffer, out int bytesWritten); + Assert.That(result, Is.EqualTo(BfTreeReadResult.Found)); + Assert.That(bytesWritten, Is.EqualTo(expected.Length)); + Assert.That(buffer[..bytesWritten].SequenceEqual(expected), Is.True); + } + + [Test] + public void ReadIntoSpan_NotFound() + { + Span buffer = stackalloc byte[256]; + var result = _tree.Read("nope"u8, buffer, out int bytesWritten); + Assert.That(result, Is.EqualTo(BfTreeReadResult.NotFound)); + Assert.That(bytesWritten, Is.EqualTo(0)); + } + + // --------------------------------------------------------------- + // Delete tests + // --------------------------------------------------------------- + + [Test] + public void DeleteExistingKey() + { + var key = "toremove"u8; + _tree.Insert(key, "data"u8); + _tree.Delete(key); + + var readResult = _tree.Read(key, out _); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Deleted)); + } + + [Test] + public void DeleteNonExistentKey_DoesNotThrow() + { + Assert.DoesNotThrow(() => _tree.Delete("ghost"u8)); + } + + // --------------------------------------------------------------- + // Scan with count tests + // --------------------------------------------------------------- + + [Test] + public void ScanWithCount_ReturnsCorrectCount() + { + InsertTestData(10); + + var results = _tree.ScanWithCount("key:"u8, 5); + Assert.That(results, Has.Count.EqualTo(5)); + } + + [Test] + public void ScanWithCount_ReturnsKeyAndValue() + { + InsertTestData(5); + + var results = _tree.ScanWithCount("key:"u8, 10, ScanReturnField.KeyAndValue); + Assert.That(results, Has.Count.EqualTo(5)); + + foreach (var r in results) + { + Assert.That(r.Key.Length, Is.GreaterThan(0)); + Assert.That(r.Value.Length, Is.GreaterThan(0)); + } + } + + [Test] + public void ScanWithCount_KeyOnly() + { + InsertTestData(5); + + var results = _tree.ScanWithCount("key:"u8, 10, ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(5)); + + foreach (var r in results) + { + Assert.That(r.Key.Length, Is.GreaterThan(0)); + Assert.That(r.Value.Length, Is.EqualTo(0)); + } + } + + [Test] + public void ScanWithCount_ValueOnly() + { + InsertTestData(5); + + var results = _tree.ScanWithCount("key:"u8, 10, ScanReturnField.Value); + Assert.That(results, Has.Count.EqualTo(5)); + + foreach (var r in results) + { + Assert.That(r.Key.Length, Is.EqualTo(0)); + Assert.That(r.Value.Length, Is.GreaterThan(0)); + } + } + + [Test] + public void ScanWithCount_Ordering() + { + // Insert keys out of order, verify scan returns them sorted + _tree.Insert("key:C"u8, "3"u8); + _tree.Insert("key:A"u8, "1"u8); + _tree.Insert("key:B"u8, "2"u8); + + var results = _tree.ScanWithCount("key:"u8, 10, ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(3)); + + var keys = results.Select(r => Encoding.UTF8.GetString(r.Key.Span)).ToList(); + Assert.That(keys, Is.EqualTo(new[] { "key:A", "key:B", "key:C" })); + } + + [Test] + public void ScanWithCount_StartKeyInMiddle() + { + InsertTestData(10); // key:0000 through key:0009 + + // Start from key:0005, should get key:0005 through key:0009 + var results = _tree.ScanWithCount( + Encoding.UTF8.GetBytes("key:0005"), 10, ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(5)); + + var firstKey = Encoding.UTF8.GetString(results[0].Key.Span); + Assert.That(firstKey, Is.EqualTo("key:0005")); + } + + [Test] + public void ScanWithCount_EmptyTree() + { + var results = _tree.ScanWithCount("key:"u8, 10); + Assert.That(results, Is.Empty); + } + + // --------------------------------------------------------------- + // Scan with end key tests + // --------------------------------------------------------------- + + [Test] + public void ScanWithEndKey_InclusiveRange() + { + InsertTestData(10); // key:0000 through key:0009 + + var results = _tree.ScanWithEndKey( + Encoding.UTF8.GetBytes("key:0002"), + Encoding.UTF8.GetBytes("key:0005"), + ScanReturnField.Key); + + var keys = results.Select(r => Encoding.UTF8.GetString(r.Key.Span)).ToList(); + Assert.That(keys, Has.Count.GreaterThanOrEqualTo(3)); + Assert.That(keys[0], Is.EqualTo("key:0002")); + } + + [Test] + public void ScanWithEndKey_AllEntries() + { + InsertTestData(5); + + var results = _tree.ScanWithEndKey( + "key:0000"u8.ToArray(), + "key:9999"u8.ToArray(), + ScanReturnField.KeyAndValue); + Assert.That(results, Has.Count.EqualTo(5)); + } + + [Test] + public void ScanWithEndKey_EmptyRange() + { + InsertTestData(5); // key:0000 through key:0004 + + var results = _tree.ScanWithEndKey( + "zzz:0000"u8.ToArray(), + "zzz:9999"u8.ToArray()); + Assert.That(results, Is.Empty); + } + + // --------------------------------------------------------------- + // ScanAll tests + // --------------------------------------------------------------- + + [Test] + public void ScanAll_ReturnsAllEntries() + { + InsertTestData(20); + + var results = _tree.ScanAll(); + Assert.That(results, Has.Count.EqualTo(20)); + + // Verify ordering + var keys = results.Select(r => Encoding.UTF8.GetString(r.Key.Span)).ToList(); + var sorted = keys.OrderBy(k => k, StringComparer.Ordinal).ToList(); + Assert.That(keys, Is.EqualTo(sorted)); + } + + [Test] + public void ScanAll_EmptyTree() + { + var results = _tree.ScanAll(); + Assert.That(results, Is.Empty); + } + + [Test] + public void ScanAll_KeyOnly() + { + InsertTestData(5); + + var results = _tree.ScanAll(ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(5)); + + foreach (var r in results) + { + Assert.That(r.Key.Length, Is.GreaterThan(0)); + Assert.That(r.Value.Length, Is.EqualTo(0)); + } + } + + // --------------------------------------------------------------- + // Zero-alloc callback scan tests + // --------------------------------------------------------------- + + [Test] + public void ScanWithCallback_ZeroAlloc() + { + InsertTestData(10); + + var keys = new List(); + Span scanBuf = stackalloc byte[8192]; + int count = _tree.ScanWithCount("key:"u8, 100, scanBuf, + (key, value) => + { + keys.Add(Encoding.UTF8.GetString(key)); + return true; + }); + + Assert.That(count, Is.EqualTo(10)); + Assert.That(keys, Has.Count.EqualTo(10)); + Assert.That(keys[0], Is.EqualTo("key:0000")); + } + + [Test] + public void ScanWithCallback_EarlyStop() + { + InsertTestData(10); + + int seen = 0; + Span scanBuf = stackalloc byte[8192]; + int count = _tree.ScanWithCount("key:"u8, 100, scanBuf, + (key, value) => + { + seen++; + return seen < 3; // stop after 3 records + }); + + Assert.That(count, Is.EqualTo(3)); + } + + // --------------------------------------------------------------- + // Snapshot / Recovery tests (disk-backed) + // --------------------------------------------------------------- + + [Test] + public void SnapshotAndRecover_RoundTrip() + { + InsertTestData(20); + _tree.Snapshot(); + _tree.Dispose(); + + // Recover from the same file + _tree = BfTreeService.RecoverFromSnapshot(_treePath, cbMinRecordSize: 4); + + for (int i = 0; i < 20; i++) + { + var key = Encoding.UTF8.GetBytes($"key:{i:D4}"); + var expectedValue = Encoding.UTF8.GetBytes($"val:{i}"); + var readResult = _tree.Read(key, out var readValue); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.Found), + $"Key key:{i:D4} not found after recovery"); + Assert.That(readValue, Is.EqualTo(expectedValue)); + } + } + + [Test] + public void SnapshotAndRecover_ScanAfterRestore() + { + InsertTestData(10); + _tree.Snapshot(); + _tree.Dispose(); + + _tree = BfTreeService.RecoverFromSnapshot(_treePath, cbMinRecordSize: 4); + + var results = _tree.ScanWithCount("key:"u8, 100, ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(10)); + } + + [Test] + public void RecoverNonExistentFile_CreatesEmpty() + { + var path = Path.Combine( + Path.GetTempPath(), $"bftree_noexist_{Guid.NewGuid():N}.bftree"); + try + { + using var tree = BfTreeService.RecoverFromSnapshot(path, cbMinRecordSize: 4); + var readResult = tree.Read("anything"u8, out _); + Assert.That(readResult, Is.EqualTo(BfTreeReadResult.NotFound)); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + [Test] + public void MemoryOnly_SnapshotThrows_PendingBfTreeSupport() + { + using var memTree = new BfTreeService( + storageBackend: StorageBackendType.Memory, + cbMinRecordSize: 4); + memTree.Insert("testkey"u8, "testval"u8); + + var snapshotPath = Path.Combine( + Path.GetTempPath(), $"bftree_memsnap_{Guid.NewGuid():N}.bftree"); + // FFI stub returns -1, C# surfaces as NotSupportedException + Assert.Throws(() => memTree.Snapshot(snapshotPath)); + } + + [Test] + public void MemoryOnly_RecoverThrows_PendingBfTreeSupport() + { + // FFI stub returns null, C# surfaces as NotSupportedException + Assert.Throws(() => + BfTreeService.RecoverFromSnapshot( + "/tmp/nonexistent.bftree", + storageBackend: StorageBackendType.Memory, + cbMinRecordSize: 4)); + } + + // --------------------------------------------------------------- + // Disposed object tests + // --------------------------------------------------------------- + + [Test] + public void OperationsOnDisposedTree_Throw() + { + var path = Path.Combine(Path.GetTempPath(), $"bftree_t_{Guid.NewGuid():N}.bftree"); + var tree = new BfTreeService(filePath: path, cbMinRecordSize: 4); + tree.Dispose(); + + try + { + Assert.Throws(() => tree.Insert("k"u8, "v"u8)); + Assert.Throws(() => tree.Read("k"u8, out _)); + Assert.Throws(() => tree.Delete("k"u8)); + Assert.Throws(() => tree.ScanWithCount("k"u8, 1)); + Assert.Throws(() => tree.ScanWithEndKey("a"u8, "z"u8)); + Assert.Throws(() => tree.Snapshot()); + } + finally { if (File.Exists(path)) File.Delete(path); } + } + + // --------------------------------------------------------------- + // Large data tests + // --------------------------------------------------------------- + + [Test] + public void LargeInsertAndScan() + { + const int count = 1000; + for (int i = 0; i < count; i++) + { + var key = Encoding.UTF8.GetBytes($"large:{i:D6}"); + var value = Encoding.UTF8.GetBytes($"payload_{i}_{new string('x', 100)}"); + _tree.Insert(key, value); + } + + var results = _tree.ScanWithCount("large:"u8, count + 1, ScanReturnField.Key); + Assert.That(results, Has.Count.EqualTo(count)); + } + + // --------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------- + + private void InsertTestData(int count) + { + InsertTestDataInto(_tree, count); + } + + private static void InsertTestDataInto(BfTreeService tree, int count) + { + for (int i = 0; i < count; i++) + { + var key = Encoding.UTF8.GetBytes($"key:{i:D4}"); + var value = Encoding.UTF8.GetBytes($"val:{i}"); + tree.Insert(key, value); + } + } + } +} diff --git a/test/Garnet.fuzz/Targets/GarnetEndToEnd.cs b/test/Garnet.fuzz/Targets/GarnetEndToEnd.cs index 5217e269fe5..a9847666b17 100644 --- a/test/Garnet.fuzz/Targets/GarnetEndToEnd.cs +++ b/test/Garnet.fuzz/Targets/GarnetEndToEnd.cs @@ -358,7 +358,7 @@ private static EmbeddedRespServer CreateServer() { ThreadPoolMinThreads = 100, SegmentSize = "1g", - ObjectStoreSegmentSize = "1g", + ObjectLogSegmentSize = "1g", EnableStorageTier = true, LogDir = LogDir.FullName, CheckpointDir = CheckpointDir.FullName, @@ -367,16 +367,14 @@ private static EmbeddedRespServer CreateServer() DisableObjects = false, EnableDebugCommand = ConnectionProtectionOption.Yes, Recover = false, - IndexSize = "1m", - ObjectStoreIndexSize = "16k", + IndexMemorySize = "1m", EnableCluster = true, CleanClusterConfig = true, ClusterTimeout = -1, QuietMode = true, EnableAOF = true, - MemorySize = "1g", + LogMemorySize = "1g", GossipDelay = 5, - EnableFastCommit = true, MetricsSamplingFrequency = 0, TlsOptions = null, DeviceFactoryCreator = new LocalStorageNamedDeviceFactoryCreator(), @@ -384,7 +382,6 @@ private static EmbeddedRespServer CreateServer() AofMemorySize = "64m", OnDemandCheckpoint = true, CommitFrequencyMs = -1, - EnableIncrementalSnapshots = true, AuthSettings = null, ClusterUsername = "cluster-user", ClusterPassword = "cluster-pass", diff --git a/test/Garnet.test.cluster/Garnet.test.cluster.csproj b/test/Garnet.test.cluster/Garnet.test.cluster.csproj deleted file mode 100644 index e150abe8284..00000000000 --- a/test/Garnet.test.cluster/Garnet.test.cluster.csproj +++ /dev/null @@ -1,64 +0,0 @@ - - - - true - ../../Garnet.snk - false - - - - 1701;1702;1591 - - - - - - - - - - - - - - - PreserveNewest - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - PreserveNewest - - - - - - false - - diff --git a/test/Garnet.test/AllureTestBase.cs b/test/Garnet.test/AllureTestBase.cs deleted file mode 100644 index 811f1155e26..00000000000 --- a/test/Garnet.test/AllureTestBase.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -using System; -using System.Linq; -using System.Reflection; -using System.Runtime.InteropServices; -using System.Runtime.Versioning; -using Allure.Net.Commons; -using NUnit.Framework; - -namespace Garnet.test -{ - - /// - /// Used as base class for Allure tests to label environment - /// - public abstract class AllureTestBase - { - [SetUp] - public void LabelEnvironment() - { - var os = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? nameof(OSPlatform.Linux) : - RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? nameof(OSPlatform.Windows) : - RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? nameof(OSPlatform.OSX) : - "unknown"; - - var frameworkAttr = Assembly.GetExecutingAssembly() - .GetCustomAttribute(); - var framework = frameworkAttr?.FrameworkName.Split(',').LastOrDefault()?.Replace("Version=v", "net") ?? "unknown"; - - var config = Assembly.GetExecutingAssembly() - .GetCustomAttribute()?.Configuration ?? "unknown"; - var timestamp = DateTime.Now.ToString("M/d/yyyy"); - var fullName = $"[{os}, {framework}, {config}]"; - var namespaceName = GetType().Namespace ?? "UnknownNamespace"; - - AllureLifecycle.Instance.UpdateTestCase(x => - { - // Remove any default suite/subSuite labels that NUnit added - x.labels.RemoveAll(l => l.name == "suite" || l.name == "subSuite"); - - // apply your custom hierarchy - x.labels.Add(Label.ParentSuite($"{namespaceName} - {timestamp}")); - x.labels.Add(Label.Suite(os)); - x.labels.Add(Label.SubSuite($"{framework} | {config}")); - }); - - // allows to separate out tests based on config but still hold history - AllureApi.AddTestParameter("env", fullName); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/AttributeExtractorTests.cs b/test/Garnet.test/AttributeExtractorTests.cs deleted file mode 100644 index 10e8a83d814..00000000000 --- a/test/Garnet.test/AttributeExtractorTests.cs +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Text; -using Allure.NUnit; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; - -namespace Garnet.test -{ - /// - /// Tests for — the raw-byte JSON field extractor - /// used by the filter expression VM to resolve selectors on demand. - /// - [AllureNUnit] - [TestFixture] - public class AttributeExtractorTests : AllureTestBase - { - /// - /// Extract a field from JSON using the new byte-span API. - /// - private static ExprToken Extract(byte[] jsonBytes, ReadOnlySpan field) - => AttributeExtractor.ExtractField(jsonBytes, field); - - /// - /// Get the string value from an ExprToken that references into json bytes. - /// - private static string GetStr(byte[] jsonBytes, ExprToken token) - { - if (token.TokenType != ExprTokenType.Str) return null; - return Encoding.UTF8.GetString(jsonBytes, token.Utf8Start, token.Utf8Length); - } - - // ======================== Number extraction ======================== - - [Test] - public void ExtractField_Integer() - { - var json = Encoding.UTF8.GetBytes("{\"year\":1980}"); - var token = Extract(json, "year"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1980.0, token.Num); - } - - [Test] - public void ExtractField_NegativeInteger() - { - var json = Encoding.UTF8.GetBytes("{\"temp\":-42}"); - var token = Extract(json, "temp"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(-42.0, token.Num); - } - - [Test] - public void ExtractField_Decimal() - { - var json = Encoding.UTF8.GetBytes("{\"rating\":4.5}"); - var token = Extract(json, "rating"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(4.5, token.Num, 0.001); - } - - [Test] - public void ExtractField_ScientificNotation() - { - var json = Encoding.UTF8.GetBytes("{\"val\":1.5e3}"); - var token = Extract(json, "val"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1500.0, token.Num); - } - - [Test] - public void ExtractField_Zero() - { - var json = Encoding.UTF8.GetBytes("{\"val\":0}"); - var token = Extract(json, "val"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(0.0, token.Num); - } - - // ======================== String extraction ======================== - - [Test] - public void ExtractField_SimpleString() - { - var json = Encoding.UTF8.GetBytes("{\"genre\":\"action\"}"); - var token = Extract(json, "genre"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsFalse(token.HasEscape, "Non-escaped strings should not have escape flag set"); - ClassicAssert.AreEqual("action", GetStr(json, token)); - } - - [Test] - public void ExtractField_EmptyString() - { - var json = Encoding.UTF8.GetBytes("{\"name\":\"\"}"); - var token = Extract(json, "name"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.AreEqual("", GetStr(json, token)); - } - - [Test] - public void ExtractField_StringWithEscapedQuote() - { - var json = Encoding.UTF8.GetBytes("{\"name\":\"hello\\\"world\"}"); - var token = Extract(json, "name"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsTrue(token.HasEscape, "Escaped strings should have escape flag set"); - // The raw bytes contain the escape sequences; the test verifies we get the right byte range - var raw = GetStr(json, token); - ClassicAssert.IsTrue(raw.Contains("hello") && raw.Contains("world")); - } - - [Test] - public void ExtractField_StringWithEscapedBackslash() - { - var json = Encoding.UTF8.GetBytes("{\"path\":\"c:\\\\temp\"}"); - var token = Extract(json, "path"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsTrue(token.HasEscape); - // Raw bytes include the escape sequences - var raw = GetStr(json, token); - ClassicAssert.IsTrue(raw.Contains("c:") && raw.Contains("temp")); - } - - [Test] - public void ExtractField_StringWithEscapedNewline() - { - var json = Encoding.UTF8.GetBytes("{\"text\":\"line1\\nline2\"}"); - var token = Extract(json, "text"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsTrue(token.HasEscape); - var raw = GetStr(json, token); - ClassicAssert.IsTrue(raw.Contains("line1") && raw.Contains("line2")); - } - - [Test] - public void ExtractField_StringWithEscapedTab() - { - var json = Encoding.UTF8.GetBytes("{\"text\":\"col1\\tcol2\"}"); - var token = Extract(json, "text"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsTrue(token.HasEscape); - var raw = GetStr(json, token); - ClassicAssert.IsTrue(raw.Contains("col1") && raw.Contains("col2")); - } - - [Test] - public void ExtractField_StringWithSlashEscape() - { - var json = Encoding.UTF8.GetBytes("{\"url\":\"http:\\/\\/example.com\"}"); - var token = Extract(json, "url"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.IsTrue(token.HasEscape); - var raw = GetStr(json, token); - ClassicAssert.IsTrue(raw.Contains("http") && raw.Contains("example.com")); - } - - // ======================== Boolean extraction ======================== - - [Test] - public void ExtractField_True() - { - var json = Encoding.UTF8.GetBytes("{\"active\":true}"); - var token = Extract(json, "active"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1.0, token.Num); - } - - [Test] - public void ExtractField_False() - { - var json = Encoding.UTF8.GetBytes("{\"deleted\":false}"); - var token = Extract(json, "deleted"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(0.0, token.Num); - } - - // ======================== Null extraction ======================== - - [Test] - public void ExtractField_Null() - { - var json = Encoding.UTF8.GetBytes("{\"value\":null}"); - var token = Extract(json, "value"u8); - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - // ======================== Array extraction ======================== - // NOTE: In the current refactored API, AttributeExtractor.ParseArrayToken returns Null - // for JSON arrays (array extraction via tuple pool is not yet re-added). - // These tests verify the current behavior: arrays are treated as Null. - - [Test] - public void ExtractField_StringArray_ReturnsNull() - { - var json = Encoding.UTF8.GetBytes("{\"tags\":[\"classic\",\"popular\"]}"); - var token = Extract(json, "tags"u8); - // Arrays are currently returned as Null by the extractor - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - [Test] - public void ExtractField_NumericArray_ReturnsNull() - { - var json = Encoding.UTF8.GetBytes("{\"scores\":[1,2,3]}"); - var token = Extract(json, "scores"u8); - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - [Test] - public void ExtractField_MixedArray_ReturnsNull() - { - var json = Encoding.UTF8.GetBytes("{\"data\":[1,\"two\",true,null]}"); - var token = Extract(json, "data"u8); - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - [Test] - public void ExtractField_EmptyArray_ReturnsNull() - { - var json = Encoding.UTF8.GetBytes("{\"items\":[]}"); - var token = Extract(json, "items"u8); - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - // ======================== Multiple fields ======================== - - [Test] - public void ExtractField_FirstField() - { - var json = Encoding.UTF8.GetBytes("{\"a\":1,\"b\":2,\"c\":3}"); - var token = Extract(json, "a"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1.0, token.Num); - } - - [Test] - public void ExtractField_MiddleField() - { - var json = Encoding.UTF8.GetBytes("{\"a\":1,\"b\":2,\"c\":3}"); - var token = Extract(json, "b"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(2.0, token.Num); - } - - [Test] - public void ExtractField_LastField() - { - var json = Encoding.UTF8.GetBytes("{\"a\":1,\"b\":2,\"c\":3}"); - var token = Extract(json, "c"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(3.0, token.Num); - } - - [Test] - public void ExtractField_SkipsValuesOfDifferentTypes() - { - // Ensure the extractor correctly skips strings, arrays, objects, booleans, nulls, and numbers - // when seeking a later field - var json = Encoding.UTF8.GetBytes("{\"s\":\"hello\",\"a\":[1,2],\"o\":{\"nested\":true},\"b\":false,\"n\":null,\"target\":42}"); - var token = Extract(json, "target"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(42.0, token.Num); - } - - // ======================== Missing / not found ======================== - - [Test] - public void ExtractField_MissingField_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"year\":1980}"); - var token = Extract(json, "rating"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_EmptyObject_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{}"); - var token = Extract(json, "anything"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - // ======================== Whitespace handling ======================== - - [Test] - public void ExtractField_WithWhitespace() - { - var json = Encoding.UTF8.GetBytes(" { \"year\" : 1980 } "); - var token = Extract(json, "year"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1980.0, token.Num); - } - - [Test] - public void ExtractField_WithNewlines() - { - var json = Encoding.UTF8.GetBytes("{\n \"year\": 1980,\n \"rating\": 4.5\n}"); - var token = Extract(json, "rating"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(4.5, token.Num, 0.001); - } - - // ======================== Nested objects (skipped) ======================== - - [Test] - public void ExtractField_NestedObject_ReturnsNone() - { - // Nested objects are not supported as values — should return IsNone - var json = Encoding.UTF8.GetBytes("{\"meta\":{\"key\":\"val\"}}"); - var token = Extract(json, "meta"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_SkipsNestedObjectToFindLaterField() - { - var json = Encoding.UTF8.GetBytes("{\"meta\":{\"key\":\"val\"},\"year\":2020}"); - var token = Extract(json, "year"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(2020.0, token.Num); - } - - [Test] - public void ExtractField_SkipsDeeplyNestedObject() - { - var json = Encoding.UTF8.GetBytes("{\"deep\":{\"a\":{\"b\":{\"c\":1}}},\"target\":99}"); - var token = Extract(json, "target"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(99.0, token.Num); - } - - // ======================== Malformed / non-JSON input ======================== - - [Test] - public void ExtractField_NotJson_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("this is not json"); - var token = Extract(json, "year"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_EmptyInput_ReturnsNone() - { - var token = AttributeExtractor.ExtractField([], "year"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_ArrayAtRoot_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("[1,2,3]"); - var token = Extract(json, "year"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_TruncatedJson_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"year\":"); - var token = Extract(json, "year"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_MissingColon_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"year\" 1980}"); - var token = Extract(json, "year"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_UnterminatedString_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"name\":\"hello}"); - var token = Extract(json, "name"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_UnterminatedKey_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"name:\"hello\"}"); - var token = Extract(json, "name"u8); - // The key "name will match to :, parsing should fail gracefully - ClassicAssert.IsTrue(token.IsNone); - } - - // ======================== Edge cases ======================== - - [Test] - public void ExtractField_StringValueContainingBraces() - { - var json = Encoding.UTF8.GetBytes("{\"data\":\"{not an object}\"}"); - var token = Extract(json, "data"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.AreEqual("{not an object}", GetStr(json, token)); - } - - [Test] - public void ExtractField_StringValueContainingBrackets() - { - var json = Encoding.UTF8.GetBytes("{\"data\":\"[not an array]\"}"); - var token = Extract(json, "data"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.AreEqual("[not an array]", GetStr(json, token)); - } - - [Test] - public void ExtractField_StringValueContainingComma() - { - var json = Encoding.UTF8.GetBytes("{\"msg\":\"hello, world\"}"); - var token = Extract(json, "msg"u8); - ClassicAssert.AreEqual(ExprTokenType.Str, token.TokenType); - ClassicAssert.AreEqual("hello, world", GetStr(json, token)); - } - - [Test] - public void ExtractField_FieldNameCaseSensitive() - { - var json = Encoding.UTF8.GetBytes("{\"Year\":2020}"); - var token = Extract(json, "year"u8); - ClassicAssert.IsTrue(token.IsNone); // Case mismatch - - var token2 = Extract(json, "Year"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token2.TokenType); - ClassicAssert.AreEqual(2020.0, token2.Num); - } - - [Test] - public void ExtractField_FieldWithHyphen() - { - // Hyphens in JSON keys are valid - var json = Encoding.UTF8.GetBytes("{\"my-field\":42}"); - var token = Extract(json, "my-field"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(42.0, token.Num); - } - - [Test] - public void ExtractField_FieldWithUnderscore() - { - var json = Encoding.UTF8.GetBytes("{\"my_field\":42}"); - var token = Extract(json, "my_field"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(42.0, token.Num); - } - - [Test] - public void ExtractField_FieldWithDigits() - { - var json = Encoding.UTF8.GetBytes("{\"field123\":99}"); - var token = Extract(json, "field123"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(99.0, token.Num); - } - - [Test] - public void ExtractField_BooleanLiteralNotFollowedByDelimiter_ReturnsNone() - { - // "trueish" should not match as true - var json = Encoding.UTF8.GetBytes("{\"val\":trueish}"); - var token = Extract(json, "val"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_NullLiteralNotFollowedByDelimiter_ReturnsNone() - { - var json = Encoding.UTF8.GetBytes("{\"val\":nullify}"); - var token = Extract(json, "val"u8); - ClassicAssert.IsTrue(token.IsNone); - } - - [Test] - public void ExtractField_ArrayWithNestedArrays_ReturnsNull() - { - // Arrays are currently returned as Null by the extractor - var json = Encoding.UTF8.GetBytes("{\"matrix\":[[1,2],[3,4]]}"); - var token = Extract(json, "matrix"u8); - ClassicAssert.AreEqual(ExprTokenType.Null, token.TokenType); - } - - [Test] - public void ExtractField_LargeNumberOfFields() - { - // Ensure we can skip many fields to find the target - var sb = new StringBuilder("{"); - for (var i = 0; i < 100; i++) - { - if (i > 0) sb.Append(','); - sb.Append($"\"field{i}\":{i}"); - } - sb.Append('}'); - - var json = Encoding.UTF8.GetBytes(sb.ToString()); - var token = Extract(json, "field99"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(99.0, token.Num); - } - - [Test] - public void ExtractField_SkipsArrayWithStringsContainingQuotes() - { - // Ensure the array skipper handles escaped quotes inside string elements - var json = Encoding.UTF8.GetBytes("{\"arr\":[\"he\\\"llo\",\"world\"],\"target\":1}"); - var token = Extract(json, "target"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(1.0, token.Num); - } - - [Test] - public void ExtractField_SkipsStringWithEscapedBackslashBeforeClosingQuote() - { - // The value is the string: ends_with_backslash\ (the JSON encodes \\ at the end) - // This tests that \\\" is parsed as \\ + " (close quote), not \ + \" - var json = Encoding.UTF8.GetBytes("{\"a\":\"ends_with_backslash\\\\\",\"b\":2}"); - var token = Extract(json, "b"u8); - ClassicAssert.AreEqual(ExprTokenType.Num, token.TokenType); - ClassicAssert.AreEqual(2.0, token.Num); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/CacheSizeTrackerTests.cs b/test/Garnet.test/CacheSizeTrackerTests.cs deleted file mode 100644 index 3c906416b2a..00000000000 --- a/test/Garnet.test/CacheSizeTrackerTests.cs +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. -using System; -using System.Threading; -using Allure.NUnit; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using StackExchange.Redis; -using Tsavorite.core; - -namespace Garnet.test -{ - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - public class CacheSizeTrackerTests : AllureTestBase - { - GarnetServer server; - TsavoriteKV objStore; - CacheSizeTracker cacheSizeTracker; - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, memorySize: "2k", pageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreHeapMemorySize: "5k"); - server.Start(); - objStore = server.Provider.StoreWrapper.objectStore; - cacheSizeTracker = server.Provider.StoreWrapper.objectStoreSizeTracker; - } - - [TearDown] - public void TearDown() - { - server?.Dispose(); - TestUtils.OnTearDown(); - } - - [Test] - public void HeapSizeValidationTest() - { - ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); - - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - db.HashSet("user:user1", [new HashEntry("Title", "Faster")]); - string r = db.HashGet("user:user1", "Title"); - ClassicAssert.AreEqual("Faster", r); - - ClassicAssert.AreEqual(248, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); - } - - [Test, CancelAfter(40 * 1000)] - public void IncreaseEmptyPageCountTest() - { - ManualResetEventSlim epcEvent = new ManualResetEventSlim(false); - int emptyPageCountIncrements = 0; - cacheSizeTracker.mainLogTracker.PostEmptyPageCountIncrease = (int count) => { emptyPageCountIncrements++; if (emptyPageCountIncrements == 3) epcEvent.Set(); }; - - ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); - ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.logAccessor.EmptyPageCount); - - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - db.HashSet("user:user1", [new HashEntry("Title", "Faster")]); - string r = db.HashGet("user:user1", "Title"); - ClassicAssert.AreEqual("Faster", r); - - ClassicAssert.AreEqual(248, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); - ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.logAccessor.EmptyPageCount); // Ensure empty page count hasn't changed as EPC is still within the min & max limits - - // Have enough records (24 bytes each) to cross a page boundary (512) - for (int i = 2; i <= 24; i++) - { - db.HashSet($"user:user{i}", [new HashEntry("Title", "Faster")]); - } - - ClassicAssert.AreEqual(5952, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); // 24 * 248 for each hashset object - - // Wait for the resizing to happen - bool eventSignaled = epcEvent.Wait( - TimeSpan.FromSeconds(3 * LogSizeTracker.ResizeTaskDelaySeconds)); // Wait for 3x resize task delay - - if (!eventSignaled) - { - Assert.Fail("Timeout occurred. Resizing did not happen within the specified time."); - } - } - - [Test] - public void ReadCacheIncreaseEmptyPageCountTest() - { - server?.Dispose(); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, memorySize: "1k", pageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreReadCacheHeapMemorySize: "1k", enableObjectStoreReadCache: true); - server.Start(); - objStore = server.Provider.StoreWrapper.objectStore; - cacheSizeTracker = server.Provider.StoreWrapper.objectStoreSizeTracker; - - var readCacheEmptyPageCountIncrements = 0; - var readCacheEpcEvent = new ManualResetEventSlim(false); - - cacheSizeTracker.readCacheTracker.PostEmptyPageCountIncrease = (int count) => { readCacheEmptyPageCountIncrements++; readCacheEpcEvent.Set(); }; - - ClassicAssert.AreEqual(0, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); - ClassicAssert.AreEqual(0, cacheSizeTracker.readCacheTracker.logAccessor.EmptyPageCount); - - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); - var db = redis.GetDatabase(0); - - // Have enough records (24 bytes each) to spill over to disk - for (var i = 0; i < 100; i++) - { - db.HashSet($"user:user{i}", [new HashEntry("Title", "Faster")]); - } - - for (var i = 0; i < 25; i++) - { - var value = db.HashGet($"user:user{i}", "Title"); - ClassicAssert.AreEqual("Faster", (string)value, i.ToString()); - } - - ClassicAssert.AreEqual(6200, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); // 25 * 248 for each hashset object - var info = TestUtils.GetStoreAddressInfo(redis.GetServer(TestUtils.EndPoint), includeReadCache: true, isObjectStore: true); - ClassicAssert.AreEqual(632, info.ReadCacheTailAddress); // 25 (records) * 24 (rec size) + 24 (initial) + 8 (page boundary) - - if (!readCacheEpcEvent.Wait(TimeSpan.FromSeconds(3 * 3 * LogSizeTracker.ResizeTaskDelaySeconds))) - ClassicAssert.Fail("Timeout occurred. Resizing did not happen within the specified time."); - - ClassicAssert.AreEqual(1, readCacheEmptyPageCountIncrements); - // 1 page of the read cache has been evicted => 20 records removed (512 pg size - 24 initial - 8 pg boundary = 480. 480/24 = 20 records) - // Leaves 5 records in the read cache. 5 * 248 = 1240 - ClassicAssert.AreEqual(1240, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/ExprCompilerTests.cs b/test/Garnet.test/ExprCompilerTests.cs deleted file mode 100644 index b8722fb06c2..00000000000 --- a/test/Garnet.test/ExprCompilerTests.cs +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Text; -using Allure.NUnit; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; - -namespace Garnet.test -{ - /// - /// Tests for the ExprCompiler (shunting-yard tokenizer + compiler). - /// Verifies tokenization and compilation to flat postfix programs. - /// - [AllureNUnit] - [TestFixture] - public class ExprCompilerTests : AllureTestBase - { - /// - /// Helper to get the string content of a Str/Selector token from the filter bytes. - /// - private static string GetStr(byte[] filterBytes, ExprToken token) - => Encoding.UTF8.GetString(filterBytes, token.Utf8Start, token.Utf8Length); - - /// Test-only wrapper around the new Span-based TryCompile. - private sealed class CompileResult - { - public ExprToken[] Instructions; - public int Length; - public ExprToken[] TuplePool; - public int TuplePoolLength; - } - - private static CompileResult Compile(byte[] filterBytes) - { - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out _); - if (instrCount < 0) return null; - return new CompileResult - { - Instructions = instrBuf[..instrCount].ToArray(), - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount].ToArray(), - TuplePoolLength = tupleCount, - }; - } - - [Test] - public void Compiler_IntegerNumbers() - { - var program = Compile(Encoding.UTF8.GetBytes("42")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(42.0, program.Instructions[0].Num); - } - - [Test] - public void Compiler_DecimalNumbers() - { - var program = Compile(Encoding.UTF8.GetBytes("3.14")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(3.14, program.Instructions[0].Num, 0.001); - } - - [Test] - public void Compiler_NegativeNumbers() - { - var program = Compile(Encoding.UTF8.GetBytes("-5")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(-5.0, program.Instructions[0].Num); - } - - [Test] - public void Compiler_StringLiterals() - { - var bytes = Encoding.UTF8.GetBytes("\"hello\""); - var program = Compile(bytes); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Str, program.Instructions[0].TokenType); - ClassicAssert.AreEqual("hello", GetStr(bytes, program.Instructions[0])); - - bytes = Encoding.UTF8.GetBytes("'world'"); - program = Compile(bytes); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Str, program.Instructions[0].TokenType); - ClassicAssert.AreEqual("world", GetStr(bytes, program.Instructions[0])); - } - - [Test] - public void Compiler_EscapedStringLiterals() - { - var bytes = Encoding.UTF8.GetBytes("\"hello\\\"world\""); - var program = Compile(bytes); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Str, program.Instructions[0].TokenType); - ClassicAssert.IsTrue(program.Instructions[0].HasEscape); - // The raw bytes include the escape sequences; verify it contains the key parts - var raw = GetStr(bytes, program.Instructions[0]); - ClassicAssert.IsTrue(raw.Contains("hello") && raw.Contains("world")); - } - - [Test] - public void Compiler_UnterminatedStringReturnsFalse() - { - var program = Compile(Encoding.UTF8.GetBytes("\"hello")); - ClassicAssert.IsNull(program); - } - - [Test] - public void Compiler_SubtractionNotConfusedWithNegative() - { - // ".a - 5" → postfix: [SEL:a] [NUM:5] [OP:Sub] - var program = Compile(Encoding.UTF8.GetBytes(".a - 5")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(3, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Selector, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[1].TokenType); - ClassicAssert.AreEqual(5.0, program.Instructions[1].Num); - ClassicAssert.AreEqual(ExprTokenType.Op, program.Instructions[2].TokenType); - ClassicAssert.AreEqual(OpCode.Sub, program.Instructions[2].OpCode); - } - - [Test] - public void Compiler_Selectors() - { - var bytes = Encoding.UTF8.GetBytes(".year"); - var program = Compile(bytes); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Selector, program.Instructions[0].TokenType); - ClassicAssert.AreEqual("year", GetStr(bytes, program.Instructions[0])); - } - - [Test] - public void Compiler_Keywords() - { - // "true and false" → [NUM:1] [NUM:0] [OP:And] - var program = Compile(Encoding.UTF8.GetBytes("true and false")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(3, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(1.0, program.Instructions[0].Num); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[1].TokenType); - ClassicAssert.AreEqual(0.0, program.Instructions[1].Num); - ClassicAssert.AreEqual(ExprTokenType.Op, program.Instructions[2].TokenType); - ClassicAssert.AreEqual(OpCode.And, program.Instructions[2].OpCode); - } - - [Test] - public void Compiler_Booleans() - { - var program = Compile(Encoding.UTF8.GetBytes("true")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(1.0, program.Instructions[0].Num); - - program = Compile(Encoding.UTF8.GetBytes("false")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(0.0, program.Instructions[0].Num); - } - - [Test] - public void Compiler_TwoCharOperators() - { - var program = Compile(Encoding.UTF8.GetBytes("1 == 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Eq, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 != 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Neq, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 >= 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Gte, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 <= 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Lte, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("true && false")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.And, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("true || false")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Or, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("2 ** 3")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Pow, program.Instructions[2].OpCode); - } - - [Test] - public void Compiler_SingleCharOperators() - { - var program = Compile(Encoding.UTF8.GetBytes("1 > 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Gt, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 < 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Lt, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 + 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Add, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 * 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Mul, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 / 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Div, program.Instructions[2].OpCode); - - program = Compile(Encoding.UTF8.GetBytes("1 % 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(OpCode.Mod, program.Instructions[2].OpCode); - } - - [Test] - public void Compiler_Parentheses() - { - var program = Compile(Encoding.UTF8.GetBytes("(.year > 10)")); - ClassicAssert.IsNotNull(program); - // Postfix: [SEL:year] [NUM:10] [OP:Gt] - ClassicAssert.AreEqual(3, program.Length); - } - - [Test] - public void Compiler_ComplexExpression() - { - var program = Compile(Encoding.UTF8.GetBytes(".year > 1950 and .rating >= 4.0")); - ClassicAssert.IsNotNull(program); - // Postfix: [SEL:year] [NUM:1950] [OP:Gt] [SEL:rating] [NUM:4.0] [OP:Gte] [OP:And] - ClassicAssert.AreEqual(7, program.Length); - } - - [Test] - public void Compiler_EmptyInput() - { - var program = Compile(Encoding.UTF8.GetBytes("")); - ClassicAssert.IsNull(program); - - program = Compile(Encoding.UTF8.GetBytes(" ")); - ClassicAssert.IsNull(program); - } - - [Test] - public void Compiler_UnexpectedCharacterReturnsFalse() - { - var program = Compile(Encoding.UTF8.GetBytes("@")); - ClassicAssert.IsNull(program); - } - - [Test] - public void Compiler_NullLiteral() - { - var program = Compile(Encoding.UTF8.GetBytes("null")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Null, program.Instructions[0].TokenType); - } - - [Test] - public void Compiler_TupleLiteral() - { - var program = Compile(Encoding.UTF8.GetBytes("[1, \"foo\", 42]")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Tuple, program.Instructions[0].TokenType); - // For Tuple tokens, Utf8Length is the element count - ClassicAssert.AreEqual(3, program.Instructions[0].Utf8Length); - } - - [Test] - public void Compiler_HyphenInSelector() - { - var bytes = Encoding.UTF8.GetBytes(".my-field"); - var program = Compile(bytes); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(1, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Selector, program.Instructions[0].TokenType); - ClassicAssert.AreEqual("my-field", GetStr(bytes, program.Instructions[0])); - } - - [Test] - public void Compiler_PrecedenceMultiplicationBeforeAddition() - { - // "1 + 2 * 3" → [1] [2] [3] [*] [+] - var program = Compile(Encoding.UTF8.GetBytes("1 + 2 * 3")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(5, program.Length); - ClassicAssert.AreEqual(OpCode.Mul, program.Instructions[3].OpCode); - ClassicAssert.AreEqual(OpCode.Add, program.Instructions[4].OpCode); - } - - [Test] - public void Compiler_PrecedenceAndBeforeOr() - { - // "true or false and true" → [1] [0] [1] [and] [or] - var program = Compile(Encoding.UTF8.GetBytes("true or false and true")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(5, program.Length); - ClassicAssert.AreEqual(OpCode.And, program.Instructions[3].OpCode); - ClassicAssert.AreEqual(OpCode.Or, program.Instructions[4].OpCode); - } - - [Test] - public void Compiler_ParenthesesOverridePrecedence() - { - // "(1 + 2) * 3" → [1] [2] [+] [3] [*] - var program = Compile(Encoding.UTF8.GetBytes("(1 + 2) * 3")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(5, program.Length); - ClassicAssert.AreEqual(OpCode.Add, program.Instructions[2].OpCode); - ClassicAssert.AreEqual(OpCode.Mul, program.Instructions[4].OpCode); - } - - [Test] - public void Compiler_ContainmentOperator() - { - // '"action" in .tags' → [STR:action] [SEL:tags] [OP:In] - var program = Compile(Encoding.UTF8.GetBytes("\"action\" in .tags")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(3, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Str, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(ExprTokenType.Selector, program.Instructions[1].TokenType); - ClassicAssert.AreEqual(OpCode.In, program.Instructions[2].OpCode); - } - - [Test] - public void Compiler_ExponentiationRightAssociative() - { - // "2 ** 3 ** 2" → 2 ** (3 ** 2) = 512 - var program = Compile(Encoding.UTF8.GetBytes("2 ** 3 ** 2")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(5, program.Length); - ClassicAssert.AreEqual(OpCode.Pow, program.Instructions[3].OpCode); - ClassicAssert.AreEqual(OpCode.Pow, program.Instructions[4].OpCode); - - var result = ExprTestHelpers.EvaluateFilter("2 ** 3 ** 2", "{}"); - ClassicAssert.AreEqual(512.0, result.Num); - } - - [Test] - public void Compiler_UnaryNot() - { - // "not true" → [NUM:1] [OP:Not] - var program = Compile(Encoding.UTF8.GetBytes("not true")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(2, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Num, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(ExprTokenType.Op, program.Instructions[1].TokenType); - ClassicAssert.AreEqual(OpCode.Not, program.Instructions[1].OpCode); - } - - [Test] - public void Compiler_ErrorOnMissingClosingParen() - { - var program = Compile(Encoding.UTF8.GetBytes("(1 + 2")); - ClassicAssert.IsNull(program); - } - - [Test] - public void Compiler_ErrorOnUnexpectedToken() - { - var program = Compile(Encoding.UTF8.GetBytes(")")); - ClassicAssert.IsNull(program); - } - - [Test] - public void Compiler_InWithTupleLiteral() - { - var program = Compile(Encoding.UTF8.GetBytes(".director in [\"Spielberg\", \"Nolan\"]")); - ClassicAssert.IsNotNull(program); - ClassicAssert.AreEqual(3, program.Length); - ClassicAssert.AreEqual(ExprTokenType.Selector, program.Instructions[0].TokenType); - ClassicAssert.AreEqual(ExprTokenType.Tuple, program.Instructions[1].TokenType); - // For Tuple tokens, Utf8Length is the element count - ClassicAssert.AreEqual(2, program.Instructions[1].Utf8Length); - ClassicAssert.AreEqual(OpCode.In, program.Instructions[2].OpCode); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/ExprRunnerTests.cs b/test/Garnet.test/ExprRunnerTests.cs deleted file mode 100644 index f9c109cf5f1..00000000000 --- a/test/Garnet.test/ExprRunnerTests.cs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Text; -using Allure.NUnit; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; - -namespace Garnet.test -{ - /// - /// Tests for ExprRunner (stack-based VM) + AttributeExtractor (raw byte JSON extractor). - /// Verifies the compile-once-run-many evaluation pipeline. - /// - [AllureNUnit] - [TestFixture] - public class ExprRunnerTests : AllureTestBase - { - [Test] - public void Runner_Arithmetic() - { - var json = "{}"; - ClassicAssert.AreEqual(5.0, ExprTestHelpers.EvaluateFilter("2 + 3", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("3 - 2", json).Num); - ClassicAssert.AreEqual(6.0, ExprTestHelpers.EvaluateFilter("2 * 3", json).Num); - ClassicAssert.AreEqual(2.5, ExprTestHelpers.EvaluateFilter("5 / 2", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("7 % 3", json).Num); - ClassicAssert.AreEqual(8.0, ExprTestHelpers.EvaluateFilter("2 ** 3", json).Num); - } - - [Test] - public void Runner_SubtractionWithField() - { - var json = "{\"year\":1980}"; - ClassicAssert.AreEqual(1975.0, ExprTestHelpers.EvaluateFilter(".year - 5", json).Num); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".year - 5 > 0", json)); - } - - [Test] - public void Runner_Comparison() - { - var json = "{}"; - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("5 > 3", json).Num); - ClassicAssert.AreEqual(0.0, ExprTestHelpers.EvaluateFilter("3 > 5", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("3 < 5", json).Num); - ClassicAssert.AreEqual(0.0, ExprTestHelpers.EvaluateFilter("5 < 3", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("5 >= 5", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("5 <= 5", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("5 == 5", json).Num); - ClassicAssert.AreEqual(1.0, ExprTestHelpers.EvaluateFilter("5 != 3", json).Num); - ClassicAssert.AreEqual(0.0, ExprTestHelpers.EvaluateFilter("5 != 5", json).Num); - } - - [Test] - public void Runner_LogicalAnd() - { - var json = "{}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("true and true", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("true and false", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("false and true", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("true && true", json)); - } - - [Test] - public void Runner_LogicalOr() - { - var json = "{}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("true or false", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("false or true", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("false or false", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("false || true", json)); - } - - [Test] - public void Runner_LogicalNot() - { - var json = "{}"; - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("not true", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("not false", json)); - } - - [Test] - public void Runner_StringEquality() - { - var json = "{\"genre\":\"action\"}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".genre == \"action\"", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy(".genre == \"drama\"", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".genre != \"drama\"", json)); - } - - [Test] - public void Runner_MemberAccess() - { - var json = "{\"year\":1980,\"rating\":4.5}"; - ClassicAssert.AreEqual(1980.0, ExprTestHelpers.EvaluateFilter(".year", json).Num); - ClassicAssert.AreEqual(4.5, ExprTestHelpers.EvaluateFilter(".rating", json).Num); - } - - [Test] - public void Runner_MissingFieldReturnsFalse() - { - var json = "{\"year\":1980}"; - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy(".missing", json)); - } - - [Test] - public void Runner_InOperatorWithJsonArray() - { - var json = "{\"tags\":[\"classic\",\"popular\"]}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("\"classic\" in .tags", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("\"modern\" in .tags", json)); - } - - [Test] - public void Runner_InOperatorWithNumericJsonArray() - { - var json = "{\"scores\":[1,2,3]}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("2 in .scores", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("5 in .scores", json)); - } - - [Test] - public void Runner_InOperatorWithTupleLiteral() - { - var json = "{\"director\":\"Nolan\"}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".director in [\"Spielberg\", \"Nolan\"]", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy(".director in [\"Spielberg\", \"Kubrick\"]", json)); - } - - [Test] - public void Runner_InOperatorSubstringCheck() - { - var json = "{\"name\":\"barfoobar\"}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("\"foo\" in .name", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("\"xyz\" in .name", json)); - } - - [Test] - public void Runner_ComplexExpression() - { - var json = "{\"year\":1980,\"rating\":4.5,\"genre\":\"action\"}"; - - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy( - ".rating * 2 > 8 and .year >= 1980", json)); - - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy( - "(.year > 2000 or .year < 1970) and .rating >= 4.0", json)); - - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("not (.genre == \"drama\")", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".year / 10 >= 198", json)); - } - - [Test] - public void Runner_BooleanJsonValues() - { - var json = "{\"active\":true,\"deleted\":false}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".active", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy(".deleted", json)); - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".active == true", json)); - } - - [Test] - public void Runner_ArithmeticWithNonNumericString_CoercesToZero() - { - var json = "{\"genre\":\"action\"}"; - ClassicAssert.AreEqual(2.0, ExprTestHelpers.EvaluateFilter(".genre + 2", json).Num); - ClassicAssert.AreEqual(-1.0, ExprTestHelpers.EvaluateFilter(".genre - 1", json).Num); - } - - [Test] - public void Runner_NullLiteral() - { - var json = "{\"year\":1980}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".year != null", json)); - } - - [Test] - public void Runner_NonJsonAttributesExcluded() - { - var filterBytes = Encoding.UTF8.GetBytes(".year > 1950"); - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out _); - ClassicAssert.IsTrue(instrCount > 0); - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - // Compute selector ranges - Span<(int, int)> selectorBuf = stackalloc (int, int)[32]; - var selectorCount = 0; - for (var idx = 0; idx < instrCount; idx++) - { - if (instrBuf[idx].TokenType != ExprTokenType.Selector) continue; - var found = false; - for (var j = 0; j < selectorCount; j++) - if (((ReadOnlySpan)filterBytes.AsSpan(selectorBuf[j].Item1, selectorBuf[j].Item2)).SequenceEqual(filterBytes.AsSpan(instrBuf[idx].Utf8Start, instrBuf[idx].Utf8Length))) { found = true; break; } - if (!found) selectorBuf[selectorCount++] = (instrBuf[idx].Utf8Start, instrBuf[idx].Utf8Length); - } - var selectorRanges = selectorBuf[..selectorCount]; - - Span extractedFields = stackalloc ExprToken[selectorCount > 0 ? selectorCount : 1]; - Span stackBuf2 = stackalloc ExprToken[16]; - var stack = new ExprStack(stackBuf2); - - var nonJson = Encoding.UTF8.GetBytes("this is not json"); - AttributeExtractor.ExtractFields(nonJson, filterBytes, selectorRanges, extractedFields, ref program); - ClassicAssert.IsFalse(ExprRunner.Run(ref program, nonJson, filterBytes, selectorRanges, extractedFields, ref stack)); - - var emptyJson = Encoding.UTF8.GetBytes(""); - AttributeExtractor.ExtractFields(emptyJson, filterBytes, selectorRanges, extractedFields, ref program); - ClassicAssert.IsFalse(ExprRunner.Run(ref program, emptyJson, filterBytes, selectorRanges, extractedFields, ref stack)); - } - - [Test] - public void Runner_ExactNumericEquality() - { - var json = "{}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy("5 == 5", json)); - ClassicAssert.IsFalse(ExprTestHelpers.EvaluateFilterTruthy("5 == 5.0001", json)); - } - - [Test] - public void Runner_HyphenatedField() - { - var json = "{\"my-field\":42}"; - ClassicAssert.AreEqual(42.0, ExprTestHelpers.EvaluateFilter(".my-field", json).Num); - } - - [Test] - public void Runner_JsonEscapeHandling() - { - var json = "{\"name\":\"hello\\\"world\"}"; - ClassicAssert.IsTrue(ExprTestHelpers.EvaluateFilterTruthy(".name == \"hello\\\"world\"", json)); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/ExprTestHelpers.cs b/test/Garnet.test/ExprTestHelpers.cs deleted file mode 100644 index 188a9d10399..00000000000 --- a/test/Garnet.test/ExprTestHelpers.cs +++ /dev/null @@ -1,331 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Text; -using Garnet.server; - -namespace Garnet.test -{ - /// - /// Test helpers for the Redis-style filter pipeline. - /// Compiles filter expressions and runs them against JSON attribute data. - /// - internal static class ExprTestHelpers - { - /// - /// Compile and run a filter expression against JSON, returning the result as an ExprToken. - /// This is useful for testing arithmetic/comparison results. - /// - internal static ExprToken EvaluateFilter(string expression, string json) - { - var filterBytes = Encoding.UTF8.GetBytes(expression); - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out var errpos); - if (instrCount < 0) - throw new InvalidOperationException($"Compilation failed at position {errpos}"); - - var jsonBytes = Encoding.UTF8.GetBytes(json); - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - return RunAndReturnTop(ref program, filterBytes, jsonBytes); - } - - /// - /// Compile and run a filter expression against JSON, returning a boolean result. - /// - internal static bool EvaluateFilterTruthy(string expression, string json) - { - var filterBytes = Encoding.UTF8.GetBytes(expression); - Span instrBuf = stackalloc ExprToken[128]; - Span tuplePoolBuf = stackalloc ExprToken[64]; - Span tokensBuf = stackalloc ExprToken[128]; - Span opsStackBuf = stackalloc ExprToken[128]; - var instrCount = ExprCompiler.TryCompile(filterBytes, instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, out var tupleCount, out var errpos); - if (instrCount < 0) - throw new InvalidOperationException($"Compilation failed at position {errpos}"); - - var jsonBytes = Encoding.UTF8.GetBytes(json); - - Span runtimePoolBuf = stackalloc ExprToken[64]; - var program = new ExprProgram - { - Instructions = instrBuf[..instrCount], - Length = instrCount, - TuplePool = tuplePoolBuf[..tupleCount], - TuplePoolLength = tupleCount, - RuntimePool = runtimePoolBuf, - RuntimePoolLength = 0, - }; - - Span<(int, int)> selectorBuf = stackalloc (int, int)[32]; - var selectorCount = 0; - for (var i = 0; i < instrCount; i++) - { - if (instrBuf[i].TokenType != ExprTokenType.Selector) continue; - var s = instrBuf[i].Utf8Start; - var l = instrBuf[i].Utf8Length; - ReadOnlySpan span = filterBytes.AsSpan(s, l); - var found = false; - for (var j = 0; j < selectorCount; j++) - if (((ReadOnlySpan)filterBytes.AsSpan(selectorBuf[j].Item1, selectorBuf[j].Item2)).SequenceEqual(span)) { found = true; break; } - if (!found) selectorBuf[selectorCount++] = (s, l); - } - var selectorRanges = selectorBuf[..selectorCount]; - - Span extractedFields = stackalloc ExprToken[selectorCount > 0 ? selectorCount : 1]; - AttributeExtractor.ExtractFields(jsonBytes, filterBytes, selectorRanges, extractedFields, ref program); - Span stackBuf = stackalloc ExprToken[16]; - var stack = new ExprStack(stackBuf); - return ExprRunner.Run(ref program, jsonBytes, filterBytes, selectorRanges, extractedFields, ref stack); - } - - /// - /// Execute a compiled program and return the top-of-stack value (for testing). - /// This is a test-only method that mirrors ExprRunner.Run but returns the raw result - /// instead of a boolean, so tests can inspect numeric/string values. - /// - private static ExprToken RunAndReturnTop(ref ExprProgram program, byte[] filterBytes, byte[] jsonBytes) - { - ReadOnlySpan json = jsonBytes; - ReadOnlySpan filter = filterBytes; - var stack = new ExprToken[256]; - var stackLen = 0; - - for (var i = 0; i < program.Length; i++) - { - var inst = program.Instructions[i]; - - if (inst.TokenType == ExprTokenType.Selector) - { - var selectorName = filter.Slice(inst.Utf8Start, inst.Utf8Length); - var extracted = AttributeExtractor.ExtractField(json, selectorName); - if (extracted.IsNone) - return ExprToken.NewNull(); - - stack[stackLen++] = extracted; - continue; - } - - if (inst.TokenType != ExprTokenType.Op) - { - stack[stackLen++] = inst; - continue; - } - - var arity = OpTable.GetArity(inst.OpCode); - ExprToken b = stackLen > 0 ? stack[--stackLen] : default; - ExprToken a = arity == 2 && stackLen > 0 ? stack[--stackLen] : default; - - var result = ExprToken.NewNum(0); - - switch (inst.OpCode) - { - case OpCode.Not: - result.Num = ToBool(b, filterBytes, json) == 0 ? 1 : 0; - break; - case OpCode.Pow: - result.Num = Math.Pow(ToNum(a, filterBytes, json), ToNum(b, filterBytes, json)); - break; - case OpCode.Mul: - result.Num = ToNum(a, filterBytes, json) * ToNum(b, filterBytes, json); - break; - case OpCode.Div: - result.Num = ToNum(a, filterBytes, json) / ToNum(b, filterBytes, json); - break; - case OpCode.Mod: - result.Num = ToNum(a, filterBytes, json) % ToNum(b, filterBytes, json); - break; - case OpCode.Add: - result.Num = ToNum(a, filterBytes, json) + ToNum(b, filterBytes, json); - break; - case OpCode.Sub: - result.Num = ToNum(a, filterBytes, json) - ToNum(b, filterBytes, json); - break; - case OpCode.Gt: - result.Num = ToNum(a, filterBytes, json) > ToNum(b, filterBytes, json) ? 1 : 0; - break; - case OpCode.Gte: - result.Num = ToNum(a, filterBytes, json) >= ToNum(b, filterBytes, json) ? 1 : 0; - break; - case OpCode.Lt: - result.Num = ToNum(a, filterBytes, json) < ToNum(b, filterBytes, json) ? 1 : 0; - break; - case OpCode.Lte: - result.Num = ToNum(a, filterBytes, json) <= ToNum(b, filterBytes, json) ? 1 : 0; - break; - case OpCode.Eq: - result.Num = AreEqual(a, b, program, filterBytes, json) ? 1 : 0; - break; - case OpCode.Neq: - result.Num = !AreEqual(a, b, program, filterBytes, json) ? 1 : 0; - break; - case OpCode.In: - result.Num = EvalIn(a, b, program, filterBytes, json) ? 1 : 0; - break; - case OpCode.And: - result.Num = ToBool(a, filterBytes, json) != 0 && ToBool(b, filterBytes, json) != 0 ? 1 : 0; - break; - case OpCode.Or: - result.Num = ToBool(a, filterBytes, json) != 0 || ToBool(b, filterBytes, json) != 0 ? 1 : 0; - break; - } - - stack[stackLen++] = result; - } - - return stackLen > 0 ? stack[stackLen - 1] : ExprToken.NewNull(); - } - - /// - /// Resolve the UTF-8 bytes for a Str token. Filter-origin tokens reference - /// filterBytes; extracted tokens reference json. - /// - private static ReadOnlySpan GetStrSpan(ExprToken t, ReadOnlySpan filterBytes, ReadOnlySpan json) - { - return t.IsFilterOrigin - ? filterBytes.Slice(t.Utf8Start, t.Utf8Length) - : json.Slice(t.Utf8Start, t.Utf8Length); - } - - private static double ToNum(ExprToken t, ReadOnlySpan filterBytes, ReadOnlySpan json) - { - if (t.IsNone) return 0; - if (t.TokenType == ExprTokenType.Num) return t.Num; - if (t.TokenType == ExprTokenType.Str) - { - var slice = GetStrSpan(t, filterBytes, json); - // Try parsing UTF-8 bytes as a number - if (double.TryParse(Encoding.UTF8.GetString(slice.ToArray()), - System.Globalization.NumberStyles.Float | System.Globalization.NumberStyles.AllowLeadingSign, - System.Globalization.CultureInfo.InvariantCulture, out var result)) - return result; - return 0; - } - return 0; - } - - private static double ToBool(ExprToken t, ReadOnlySpan filterBytes, ReadOnlySpan json) - { - if (t.IsNone) return 0; - if (t.TokenType == ExprTokenType.Num) return t.Num != 0 ? 1 : 0; - if (t.TokenType == ExprTokenType.Str) return t.Utf8Length == 0 ? 0 : 1; - if (t.TokenType == ExprTokenType.Null) return 0; - return 1; - } - - private static bool AreEqual(ExprToken a, ExprToken b, ExprProgram program, - ReadOnlySpan filterBytes, ReadOnlySpan json) - { - if (a.IsNone || b.IsNone) return a.IsNone && b.IsNone; - - if (a.TokenType == ExprTokenType.Str && b.TokenType == ExprTokenType.Str) - { - var aSpan = GetStrSpan(a, filterBytes, json); - var bSpan = GetStrSpan(b, filterBytes, json); - if (!a.HasEscape && !b.HasEscape) - return aSpan.SequenceEqual(bSpan); - return UnescapedEquals(aSpan, a.HasEscape, bSpan, b.HasEscape); - } - - if (a.TokenType == ExprTokenType.Num && b.TokenType == ExprTokenType.Num) - return a.Num == b.Num; - - if (a.TokenType == ExprTokenType.Null || b.TokenType == ExprTokenType.Null) - return a.TokenType == b.TokenType; - - return ToNum(a, filterBytes, json) == ToNum(b, filterBytes, json); - } - - private static bool EvalIn(ExprToken a, ExprToken b, ExprProgram program, - ReadOnlySpan filterBytes, ReadOnlySpan json) - { - if (b.IsNone) return false; - - // Tuple membership: for Tuple tokens, Utf8Start = pool start, Utf8Length = element count - if (b.TokenType == ExprTokenType.Tuple) - { - var poolStart = b.Utf8Start; - var poolLen = b.Utf8Length; - for (var i = 0; i < poolLen; i++) - { - if (AreEqual(a, program.TuplePool[poolStart + i], program, filterBytes, json)) - return true; - } - return false; - } - - // String substring check - if (!a.IsNone && a.TokenType == ExprTokenType.Str && b.TokenType == ExprTokenType.Str) - { - var needle = GetStrSpan(a, filterBytes, json); - var haystack = GetStrSpan(b, filterBytes, json); - if (needle.Length == 0) return true; - if (needle.Length > haystack.Length) return false; - return haystack.IndexOf(needle) >= 0; - } - - return false; - } - - private static bool UnescapedEquals(ReadOnlySpan a, bool aEscaped, ReadOnlySpan b, bool bEscaped) - { - var ai = 0; - var bi = 0; - while (ai < a.Length && bi < b.Length) - { - byte ac, bc; - if (aEscaped && ai < a.Length - 1 && a[ai] == (byte)'\\') - { - ai++; - ac = UnescapeByte(a[ai]); - } - else - { - ac = a[ai]; - } - - if (bEscaped && bi < b.Length - 1 && b[bi] == (byte)'\\') - { - bi++; - bc = UnescapeByte(b[bi]); - } - else - { - bc = b[bi]; - } - - if (ac != bc) return false; - ai++; - bi++; - } - - return ai == a.Length && bi == b.Length; - } - - private static byte UnescapeByte(byte b) => b switch - { - (byte)'n' => (byte)'\n', - (byte)'r' => (byte)'\r', - (byte)'t' => (byte)'\t', - (byte)'\\' => (byte)'\\', - (byte)'"' => (byte)'"', - (byte)'\'' => (byte)'\'', - (byte)'/' => (byte)'/', - _ => b, - }; - } -} \ No newline at end of file diff --git a/test/Garnet.test/Garnet.test.csproj b/test/Garnet.test/Garnet.test.csproj deleted file mode 100644 index e71000d8d2c..00000000000 --- a/test/Garnet.test/Garnet.test.csproj +++ /dev/null @@ -1,79 +0,0 @@ - - - - true - ../../Garnet.snk - false - - - - 1701;1702;1591 - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - - - - - - - - - - - - - - - PreserveNewest - - - - - - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - false - - - diff --git a/test/Garnet.test/GarnetObjectTests.cs b/test/Garnet.test/GarnetObjectTests.cs deleted file mode 100644 index e7a85966d23..00000000000 --- a/test/Garnet.test/GarnetObjectTests.cs +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System.Threading.Tasks; -using Allure.NUnit; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using Tsavorite.core; - -namespace Garnet.test -{ - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - - [AllureNUnit] - [TestFixture] - public class GarnetObjectTests : AllureTestBase - { - TsavoriteKV store; - IDevice logDevice, objectLogDevice; - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - CreateStore(); - } - - [TearDown] - public void TearDown() - { - store.Dispose(); - logDevice.Dispose(); - objectLogDevice.Dispose(); - logDevice = objectLogDevice = null; - TestUtils.OnTearDown(); - } - - [Test] - public void WriteRead() - { - using var session = store.NewSession>(new SimpleSessionFunctions()); - var bContext = session.BasicContext; - - var key = new byte[] { 0 }; - var obj = new SortedSetObject(); - - bContext.Upsert(key, obj); - - IGarnetObject output = null; - var status = bContext.Read(ref key, ref output); - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.AreEqual(obj, output); - } - - [Test] - public async Task WriteCheckpointRead() - { - var session = store.NewSession(new MyFunctions()); - var bContext = session.BasicContext; - - var key = new byte[] { 0 }; - var obj = new SortedSetObject(); - obj.Add([15], 10); - - bContext.Upsert(key, obj); - - session.Dispose(); - - await store.TakeHybridLogCheckpointAsync(CheckpointType.FoldOver).ConfigureAwait(false); - - store.Dispose(); - CreateStore(); - - store.Recover(); - - session = store.NewSession(new MyFunctions()); - bContext = session.BasicContext; - - IGarnetObject output = null; - var status = bContext.Read(ref key, ref output); - - session.Dispose(); - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.IsTrue(obj.Equals((SortedSetObject)output)); - } - - [Test] - public async Task CopyUpdate() - { - var session = store.NewSession(new MyFunctions()); - var bContext = session.BasicContext; - - var key = new byte[] { 0 }; - IGarnetObject obj = new SortedSetObject(); - ((SortedSetObject)obj).Add([15], 10); - - bContext.Upsert(key, obj); - - store.Log.Flush(true); - - bContext.RMW(ref key, ref obj); - - session.Dispose(); - - await store.TakeHybridLogCheckpointAsync(CheckpointType.FoldOver).ConfigureAwait(false); - - store.Dispose(); - CreateStore(); - - store.Recover(); - - session = store.NewSession(new MyFunctions()); - bContext = session.BasicContext; - - IGarnetObject output = null; - var status = bContext.Read(ref key, ref output); - - session.Dispose(); - - ClassicAssert.IsTrue(status.Found); - ClassicAssert.IsTrue(((SortedSetObject)obj).Equals((SortedSetObject)output)); - } - - private class MyFunctions : SessionFunctionsBase - { - public MyFunctions() - { } - - public override bool SingleReader(ref byte[] key, ref IGarnetObject input, ref IGarnetObject value, ref IGarnetObject dst, ref ReadInfo updateInfo) - { - dst = value; - return true; - } - - public override bool ConcurrentReader(ref byte[] key, ref IGarnetObject input, ref IGarnetObject value, ref IGarnetObject dst, ref ReadInfo updateInfo, ref RecordInfo recordInfo) - { - dst = value; - return true; - } - - public override bool CopyUpdater(ref byte[] key, ref IGarnetObject input, ref IGarnetObject oldValue, ref IGarnetObject newValue, ref IGarnetObject output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) - { - oldValue.CopyUpdate(ref oldValue, ref newValue, false); - return true; - } - } - - private void CreateStore() - { - logDevice ??= Devices.CreateLogDevice(TestUtils.MethodTestDir + "/hlog.log"); - objectLogDevice ??= Devices.CreateLogDevice(TestUtils.MethodTestDir + "/hlog.obj.log"); - - var kvSettings = new KVSettings - { - IndexSize = 1L << 13, - LogDevice = logDevice, - ObjectLogDevice = objectLogDevice, - CheckpointDir = TestUtils.MethodTestDir - }; - - store = new(kvSettings - , StoreFunctions.Create(new ByteArrayKeyComparer(), () => new Tsavorite.core.ByteArrayBinaryObjectSerializer(), () => new MyGarnetObjectSerializer()) - , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - } - } - - /// - /// Serializer for IGarnetObject - /// - sealed class MyGarnetObjectSerializer : BinaryObjectSerializer - { - /// - public override void Deserialize(out IGarnetObject obj) - { - var type = (GarnetObjectType)reader.ReadByte(); - obj = type switch - { - GarnetObjectType.SortedSet => new SortedSetObject(reader), - GarnetObjectType.List => new ListObject(reader), - GarnetObjectType.Hash => new HashObject(reader), - GarnetObjectType.Set => new SetObject(reader), - _ => null, - }; - } - - /// - public override void Serialize(ref IGarnetObject obj) - { - if (obj == null) - writer.Write((byte)GarnetObjectType.Null); - else - obj.Serialize(writer); - } - } -} \ No newline at end of file diff --git a/test/Garnet.test/RespConfigTests.cs b/test/Garnet.test/RespConfigTests.cs deleted file mode 100644 index 05e6927ca9d..00000000000 --- a/test/Garnet.test/RespConfigTests.cs +++ /dev/null @@ -1,789 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -using System; -using System.Linq; -using System.Threading; -using Allure.NUnit; -using Garnet.common; -using Garnet.server; -using NUnit.Framework; -using NUnit.Framework.Legacy; -using StackExchange.Redis; -using Tsavorite.core; - -namespace Garnet.test -{ - using ObjectStoreAllocator = GenericAllocator>>; - using ObjectStoreFunctions = StoreFunctions>; - - /// - /// Test dynamically changing server configuration using CONFIG SET command. - /// - [AllureNUnit] - [TestFixture(false)] - [TestFixture(true)] - public class RespConfigTests : AllureTestBase - { - GarnetServer server; - private string memorySize = "17g"; - private string indexSize = "64m"; - private string objectStoreLogMemorySize = "17m"; - private string objectStoreHeapMemorySize = "32m"; - private string objectStoreIndexSize = "8m"; - private bool useReviv; - - public RespConfigTests(bool useReviv) - { - this.useReviv = useReviv; - } - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, - memorySize: memorySize, - indexSize: indexSize, - objectStoreLogMemorySize: objectStoreLogMemorySize, - objectStoreIndexSize: objectStoreIndexSize, - objectStoreHeapMemorySize: objectStoreHeapMemorySize, - useReviv: useReviv); - server.Start(); - } - - [TearDown] - public void TearDown() - { - server.Dispose(); - TestUtils.OnTearDown(); - } - - /// - /// This test verifies that dynamically changing the memory size configuration using CONFIG SET memory / obj-log-memory - /// incurs the expected changes in Garnet server metrics, as well as verifies error handling for incorrect inputs. - /// - /// Store type (Main / Object) - /// Memory size smaller than the initial size - /// Memory size larger than the initial size (within buffer bounds) - /// Memory size larger than the buffer size - /// Malformed memory size string - [Test] - [TestCase(StoreType.Main, "16g", "32g", "64g", "g4")] - [TestCase(StoreType.Main, "9gB", "28GB", "33G", "2gBB")] - [TestCase(StoreType.Object, "16m", "32m", "64m", "3bm")] - [TestCase(StoreType.Object, "5MB", "30M", "128mb", "44d")] - public void ConfigSetMemorySizeTest(StoreType storeType, string smallerSize, string largerSize, string largerThanBufferSize, string malformedSize) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - var option = storeType == StoreType.Main ? "memory" : "obj-log-memory"; - var metricType = storeType == StoreType.Main ? InfoMetricsType.STORE : InfoMetricsType.OBJECTSTORE; - var initMemorySize = storeType == StoreType.Main ? memorySize : objectStoreLogMemorySize; - - var currMemorySize = ServerOptions.ParseSize(initMemorySize, out _); - var bufferSize = ServerOptions.NextPowerOf2(currMemorySize); - var pageSize = storeType == StoreType.Main ? 32L * 1024 * 1024 : 4 * 1024; // default page size - - // Check initial MinEPC before any changes - var metrics = server.Metrics.GetInfoMetrics(metricType); - var miMinEPC = metrics.FirstOrDefault(mi => mi.Name == "Log.MinEmptyPageCount"); - ClassicAssert.IsNotNull(miMinEPC); - ClassicAssert.IsTrue(long.TryParse(miMinEPC.Value, out var minEmptyPageCount)); - var expectedMinEPC = (int)((bufferSize - currMemorySize) / pageSize); - ClassicAssert.AreEqual(expectedMinEPC, minEmptyPageCount); - - // Try to set memory size to the same value as current - var result = db.Execute("CONFIG", "SET", option, initMemorySize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // MinEPC should remain unchanged - metrics = server.Metrics.GetInfoMetrics(metricType); - miMinEPC = metrics.FirstOrDefault(mi => mi.Name == "Log.MinEmptyPageCount"); - ClassicAssert.IsNotNull(miMinEPC); - ClassicAssert.IsTrue(long.TryParse(miMinEPC.Value, out minEmptyPageCount)); - ClassicAssert.AreEqual(expectedMinEPC, minEmptyPageCount); - - // Try to set memory size to a smaller value than current - result = db.Execute("CONFIG", "SET", option, smallerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Check that MinEPC has changed accordingly - currMemorySize = ServerOptions.ParseSize(smallerSize, out _); - metrics = server.Metrics.GetInfoMetrics(metricType); - miMinEPC = metrics.FirstOrDefault(mi => mi.Name == "Log.MinEmptyPageCount"); - ClassicAssert.IsNotNull(miMinEPC); - ClassicAssert.IsTrue(long.TryParse(miMinEPC.Value, out minEmptyPageCount)); - expectedMinEPC = (int)((bufferSize - currMemorySize) / pageSize); - ClassicAssert.AreEqual(expectedMinEPC, minEmptyPageCount); - - // Try to set memory size to a larger value than current - result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Check that MinEPC has changed accordingly - currMemorySize = ServerOptions.ParseSize(largerSize, out _); - metrics = server.Metrics.GetInfoMetrics(metricType); - miMinEPC = metrics.FirstOrDefault(mi => mi.Name == "Log.MinEmptyPageCount"); - ClassicAssert.IsNotNull(miMinEPC); - ClassicAssert.IsTrue(long.TryParse(miMinEPC.Value, out minEmptyPageCount)); - expectedMinEPC = (int)((bufferSize - currMemorySize) / pageSize); - ClassicAssert.AreEqual(expectedMinEPC, minEmptyPageCount); - - // Try to set memory size larger than the buffer size - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, largerThanBufferSize), - string.Format(CmdStrings.GenericErrMemorySizeGreaterThanBuffer, option)); - - // MinEPC should remain unchanged - metrics = server.Metrics.GetInfoMetrics(metricType); - miMinEPC = metrics.FirstOrDefault(mi => mi.Name == "Log.MinEmptyPageCount"); - ClassicAssert.IsNotNull(miMinEPC); - ClassicAssert.IsTrue(long.TryParse(miMinEPC.Value, out minEmptyPageCount)); - ClassicAssert.AreEqual(expectedMinEPC, minEmptyPageCount); - - // Try to set memory size with a malformed size input - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, malformedSize), - string.Format(CmdStrings.GenericErrIncorrectSizeFormat, option)); - } - - /// - /// This test verifies that dynamically changing the index size configuration using CONFIG SET index / obj-index - /// incurs the expected changes in Garnet server metrics, as well as verifies error handling for incorrect inputs. - /// - /// Store type (Main / Object) - /// Index size smaller than the initial size - /// Index size larger than the initial size - /// Illegal index size (not a power of 2) - /// Malformed index size string - [Test] - [TestCase(StoreType.Main, "32m", "128m", "63m", "8d")] - [TestCase(StoreType.Main, "16mB", "256MB", "23m", "g8")] - [TestCase(StoreType.Object, "2m", "32m", "28m", "m9")] - [TestCase(StoreType.Object, "4Mb", "16mB", "129MB", "0.3gb")] - public void ConfigSetIndexSizeTest(StoreType storeType, string smallerSize, string largerSize, string illegalSize, string malformedSize) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - var metricType = storeType == StoreType.Main ? InfoMetricsType.STORE : InfoMetricsType.OBJECTSTORE; - var option = storeType == StoreType.Main ? "index" : "obj-index"; - var initIndexSize = storeType == StoreType.Main ? indexSize : objectStoreIndexSize; - - // Check initial index size before any changes - var currIndexSize = ServerOptions.ParseSize(initIndexSize, out _); - var metrics = server.Metrics.GetInfoMetrics(metricType); - var miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexSize"); - ClassicAssert.IsNotNull(miIndexSize); - ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out var actualIndexSize)); - var expectedIndexSize = currIndexSize / 64; - ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); - - // Try to set index size to the same value as current - var result = db.Execute("CONFIG", "SET", option, initIndexSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Index size should remain unchanged - metrics = server.Metrics.GetInfoMetrics(metricType); - miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexSize"); - ClassicAssert.IsNotNull(miIndexSize); - ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out actualIndexSize)); - ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); - - // Try to set index size to a larger value than current - result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Check that index size has changed accordingly - currIndexSize = ServerOptions.ParseSize(largerSize, out _); - metrics = server.Metrics.GetInfoMetrics(metricType); - miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexSize"); - ClassicAssert.IsNotNull(miIndexSize); - ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out actualIndexSize)); - expectedIndexSize = currIndexSize / 64; - ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); - - // Try to set index size to a smaller value than current - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, smallerSize), - string.Format(CmdStrings.GenericErrIndexSizeSmallerThanCurrent, option)); - - // Try to set index size to a value that is not a power of two - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, illegalSize), - string.Format(CmdStrings.GenericErrIndexSizePowerOfTwo, option)); - - // Try to set index size with a malformed size input - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, malformedSize), - string.Format(CmdStrings.GenericErrIncorrectSizeFormat, option)); - } - - /// - /// This test verifies that dynamically changing the object store heap size configuration using CONFIG SET object_store_heap_memory_target_size - /// incurs the expected changes in Garnet server metrics, as well as verifies error handling for incorrect inputs. - /// - /// Heap size smaller than the initial size - /// Heap size larger than the initial size - /// Malformed heap size string - [Test] - [TestCase("10m", "128m", "1.5mb")] - [TestCase("16m", "65m", "g6")] - public void ConfigObjHeapSizeTest(string smallerSize, string largerSize, string malformedSize) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var option = "obj-heap-memory"; - var currObjHeapSize = ServerOptions.ParseSize(objectStoreHeapMemorySize, out _); - - // Check initial heap size before any changes - var metrics = server.Metrics.GetInfoMetrics(InfoMetricsType.MEMORY); - var miObjHeapTargetSize = metrics.FirstOrDefault(mi => mi.Name == "object_store_heap_memory_target_size"); - ClassicAssert.IsNotNull(miObjHeapTargetSize); - ClassicAssert.IsTrue(long.TryParse(miObjHeapTargetSize.Value, out var objHeapTargetSize)); - ClassicAssert.AreEqual(currObjHeapSize, objHeapTargetSize); - - // Try to set heap size to the same value as current - var result = db.Execute("CONFIG", "SET", option, objectStoreHeapMemorySize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Heap size should remain unchanged - metrics = server.Metrics.GetInfoMetrics(InfoMetricsType.MEMORY); - miObjHeapTargetSize = metrics.FirstOrDefault(mi => mi.Name == "object_store_heap_memory_target_size"); - ClassicAssert.IsNotNull(miObjHeapTargetSize); - ClassicAssert.IsTrue(long.TryParse(miObjHeapTargetSize.Value, out objHeapTargetSize)); - ClassicAssert.AreEqual(currObjHeapSize, objHeapTargetSize); - - // Try to set heap size to a smaller value than current - result = db.Execute("CONFIG", "SET", option, smallerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Check that heap size has changed accordingly - currObjHeapSize = ServerOptions.ParseSize(smallerSize, out _); - metrics = server.Metrics.GetInfoMetrics(InfoMetricsType.MEMORY); - miObjHeapTargetSize = metrics.FirstOrDefault(mi => mi.Name == "object_store_heap_memory_target_size"); - ClassicAssert.IsNotNull(miObjHeapTargetSize); - ClassicAssert.IsTrue(long.TryParse(miObjHeapTargetSize.Value, out objHeapTargetSize)); - ClassicAssert.AreEqual(currObjHeapSize, objHeapTargetSize); - - // Try to set heap size to a larger value than current - result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Check that heap size has changed accordingly - currObjHeapSize = ServerOptions.ParseSize(largerSize, out _); - metrics = server.Metrics.GetInfoMetrics(InfoMetricsType.MEMORY); - miObjHeapTargetSize = metrics.FirstOrDefault(mi => mi.Name == "object_store_heap_memory_target_size"); - ClassicAssert.IsNotNull(miObjHeapTargetSize); - ClassicAssert.IsTrue(long.TryParse(miObjHeapTargetSize.Value, out objHeapTargetSize)); - ClassicAssert.AreEqual(currObjHeapSize, objHeapTargetSize); - - // Try to set heap size with a malformed size input - this should fail - Assert.Throws(() => db.Execute("CONFIG", "SET", option, malformedSize), - string.Format(CmdStrings.GenericErrIncorrectSizeFormat, option)); - } - } - - - - /// - /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET. - /// - [AllureNUnit] - [TestFixture(false)] - [TestFixture(true)] - public class RespConfigUtilizationTests : AllureTestBase - { - GarnetServer server; - private string memorySize = "3m"; - private string indexSize = "1m"; - private string objectStoreLogMemorySize = "2500"; - private string objectStoreHeapMemorySize = "1m"; - private string objectStoreIndexSize = "2048"; - private string pageSize = "1024"; - private bool useReviv; - - public RespConfigUtilizationTests(bool useReviv) - { - this.useReviv = useReviv; - } - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(null, - memorySize: memorySize, - indexSize: indexSize, - pageSize: pageSize, - objectStorePageSize: pageSize, - objectStoreLogMemorySize: objectStoreLogMemorySize, - objectStoreIndexSize: objectStoreIndexSize, - objectStoreHeapMemorySize: objectStoreHeapMemorySize, - useReviv: useReviv); - server.Start(); - } - - [TearDown] - public void TearDown() - { - server.Dispose(); - TestUtils.OnTearDown(); - } - - /// - /// This test verifies that dynamically changing the memory size configuration using CONFIG SET - /// incurs the expected shifts in the head and tail addresses of the store. - /// - /// Store Type (Main / Object) - /// Memory size smaller than the initial size - /// Memory size larger than the initial size (within buffer bounds) - [Test] - [TestCase(StoreType.Main, "1m", "4m")] - [TestCase(StoreType.Main, "1024k", "4000k")] - [TestCase(StoreType.Object, "1024", "4000")] - [TestCase(StoreType.Object, "1024", "4096")] - public void ConfigSetMemorySizeUtilizationTest(StoreType storeType, string smallerSize, string largerSize) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); - var db = redis.GetDatabase(0); - var option = storeType == StoreType.Main ? "memory" : "obj-log-memory"; - var initMemorySize = storeType == StoreType.Main ? memorySize : objectStoreLogMemorySize; - var currMemorySize = TestUtils.GetEffectiveMemorySize(initMemorySize, pageSize, out var parsedPageSize); - - var garnetServer = redis.GetServer(TestUtils.EndPoint); - var info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - ClassicAssert.AreEqual(storeType == StoreType.Main ? 64 : 24, info.TailAddress); - - var i = 0; - var val = new RedisValue(new string('x', storeType == StoreType.Main ? 512 - 32 : 1)); - - // Insert records until head address moves - var prevHead = info.HeadAddress; - var prevTail = info.TailAddress; - while (info.HeadAddress == prevHead) - { - var key = $"key{i++:00000}"; - if (storeType == StoreType.Main) - _ = db.StringSet(key, val); - else - _ = db.ListRightPush(key, [val]); - - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - } - - // Verify that records were inserted up to the configured memory size limit - Assert.That(prevTail, Is.LessThanOrEqualTo(currMemorySize)); - Assert.That(currMemorySize - prevTail, Is.LessThanOrEqualTo(parsedPageSize)); - - // Try to set memory size to a smaller value than current - var result = db.Execute("CONFIG", "SET", option, smallerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Verify that head address moved forward - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - Assert.That(info.HeadAddress, Is.GreaterThan(prevHead)); - - currMemorySize = TestUtils.GetEffectiveMemorySize(smallerSize, pageSize, out _); - - // Insert records until head address moves - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - while (info.HeadAddress == prevHead) - { - var key = $"key{i++:00000}"; - if (storeType == StoreType.Main) - _ = db.StringSet(key, val); - else - _ = db.ListRightPush(key, [val]); - - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - } - - // Verify that records were inserted up to the configured memory size limit - Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(currMemorySize)); - Assert.That(currMemorySize - (prevTail - prevHead), Is.LessThanOrEqualTo(parsedPageSize)); - - // Try to set memory size to a larger value than current - result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - currMemorySize = TestUtils.GetEffectiveMemorySize(largerSize, pageSize, out _); - - // Continue to insert records until new memory capacity is reached - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - while (info.HeadAddress == prevHead) - { - var key = $"key{i++:00000}"; - if (storeType == StoreType.Main) - _ = db.StringSet(key, val); - else - _ = db.ListRightPush(key, [val]); - - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - } - - // Verify that memory is fully utilized - Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(currMemorySize)); - Assert.That(currMemorySize - (prevTail - prevHead), Is.LessThanOrEqualTo(parsedPageSize)); - } - - /// - /// This test verifies recovery behavior after dynamically changing the memory size configuration using CONFIG SET. - /// The test fills the store to a larger capacity than the initial memory size, then verifies that recovering with the - /// smaller initial memory size retains the last inserted keys in the expected initial capacity. - /// - /// Store Type (Main / Object) - /// Memory size larger than the initial size (within buffer bounds) - [Test] - [TestCase(StoreType.Main, "4m")] - [TestCase(StoreType.Object, "4096")] - public void ConfigSetMemorySizeRecoveryTest(StoreType storeType, string largerSize) - { - var option = storeType == StoreType.Main ? "memory" : "obj-log-memory"; - var initMemorySize = storeType == StoreType.Main ? memorySize : objectStoreLogMemorySize; - - var currMemorySize = TestUtils.GetEffectiveMemorySize(initMemorySize, pageSize, out var parsedPageSize); - - int lastIdxSecondRound; - int keysInsertedFirstRound; - - using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) - { - var db = redis.GetDatabase(0); - var garnetServer = redis.GetServer(TestUtils.EndPoint); - var info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - ClassicAssert.AreEqual(storeType == StoreType.Main ? 64 : 24, info.TailAddress); - - var i = 0; - var val = new RedisValue(new string('x', storeType == StoreType.Main ? 512 - 32 : 1)); - - // Insert records until head address moves - var prevHead = info.HeadAddress; - var prevTail = info.TailAddress; - while (info.HeadAddress == prevHead) - { - var key = $"key{i++:00000}"; - if (storeType == StoreType.Main) - _ = db.StringSet(key, val); - else - _ = db.ListRightPush(key, [val]); - - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - } - - var lastIdxFirstRound = i - 1; - - // Verify that records were inserted up to the configured memory size limit - Assert.That(prevTail, Is.LessThanOrEqualTo(currMemorySize)); - Assert.That(currMemorySize - prevTail, Is.LessThanOrEqualTo(parsedPageSize)); - - // Find the first key index that still exists in the server - ClassicAssert.IsTrue(db.KeyExists($"key{lastIdxFirstRound:00000}")); - var c = lastIdxFirstRound; - while (c > 0) - { - if (!db.KeyExists($"key{--c:00000}")) break; - } - - // Record the number of keys inserted in the first round - keysInsertedFirstRound = lastIdxFirstRound + 1 - c; - - // Try to set memory size to a larger value than current - var result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - currMemorySize = TestUtils.GetEffectiveMemorySize(largerSize, pageSize, out _); - - // Continue to insert records until new memory capacity is reached - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - while (info.HeadAddress == prevHead) - { - var key = $"key{i++:00000}"; - if (storeType == StoreType.Main) - _ = db.StringSet(key, val); - else - _ = db.ListRightPush(key, [val]); - - prevHead = info.HeadAddress; - prevTail = info.TailAddress; - info = TestUtils.GetStoreAddressInfo(garnetServer, isObjectStore: storeType == StoreType.Object); - } - - lastIdxSecondRound = i - 1; - - // Verify that memory is fully utilized - Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(currMemorySize)); - Assert.That(currMemorySize - (prevTail - prevHead), Is.LessThanOrEqualTo(parsedPageSize)); - - garnetServer.Save(SaveType.BackgroundSave); - while (garnetServer.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); - } - - // Restart server with initial memory size and recover data - server.Dispose(false); - server = TestUtils.CreateGarnetServer(null, - memorySize: memorySize, - indexSize: indexSize, - pageSize: pageSize, - objectStorePageSize: pageSize, - objectStoreLogMemorySize: objectStoreLogMemorySize, - objectStoreIndexSize: objectStoreIndexSize, - objectStoreHeapMemorySize: objectStoreHeapMemorySize, - useReviv: useReviv, - tryRecover: true); - server.Start(); - - using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) - { - var db = redis.GetDatabase(0); - - // Find the smallest key index that still exists in the server - var c = lastIdxSecondRound; - while (c > 0) - { - if (!db.KeyExists($"key{--c:00000}")) - break; - } - - // Verify that the number of existing keys matches the count of inserted keys in the first round of insertions - ClassicAssert.AreEqual(keysInsertedFirstRound, lastIdxSecondRound + 1 - c); - - // Verify that all previous keys are not present in the database - while (c > 0) - { - ClassicAssert.IsFalse(db.KeyExists($"key{--c:00000}")); - } - } - } - } - - /// - /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET. - /// - [AllureNUnit] - [TestFixture(false)] - [TestFixture(true)] - public class RespConfigIndexUtilizationTests : AllureTestBase - { - GarnetServer server; - private string memorySize = "3m"; - private string indexSize = "512"; - private string objectStoreLogMemorySize = "16384"; - private string objectStoreHeapMemorySize = "16384"; - private string pageSize = "1024"; - private bool useReviv; - - public RespConfigIndexUtilizationTests(bool useReviv) - { - this.useReviv = useReviv; - } - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(null, - memorySize: memorySize, - indexSize: indexSize, - pageSize: pageSize, - objectStorePageSize: pageSize, - objectStoreLogMemorySize: objectStoreLogMemorySize, - objectStoreIndexSize: indexSize, - objectStoreHeapMemorySize: objectStoreHeapMemorySize, - useReviv: useReviv); - server.Start(); - } - - [TearDown] - public void TearDown() - { - server.Dispose(); - TestUtils.OnTearDown(); - } - - /// - /// This test verifies that dynamically changing the index size configuration using CONFIG SET - /// incurs the expected shifts in the overflow buckets of the store, and that no data is lost in the process. - /// - /// Store type (Main / Object) - /// Larger index size than configured - /// Larger index size than previous - [Test] - [TestCase(StoreType.Main, "1024", "4096")] - [TestCase(StoreType.Object, "1024", "4096")] - public void ConfigSetIndexSizeUtilizationTest(StoreType storeType, string largerSize1, string largerSize2) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); - var db = redis.GetDatabase(0); - var option = storeType == StoreType.Main ? "index" : "obj-index"; - var parsedIndexSize = ServerOptions.ParseSize(indexSize, out _); - - var currIndexSize = storeType == StoreType.Main - ? server.Provider.StoreWrapper.store.IndexSize - : server.Provider.StoreWrapper.objectStore.IndexSize; - - // Verify initial index size and overflow bucket allocations are zero - ClassicAssert.AreEqual(parsedIndexSize / 64, currIndexSize); - ClassicAssert.AreEqual(0, GetOverflowBucketAllocations()); - - // Generate data with random keys (so that hashtable overflows) - var val = new RedisValue("x"); - var keys = new string[500]; - for (var i = 0; i < keys.Length; i++) - keys[i] = TestUtils.GetRandomString(8); - - // Insert first batch of data - for (var i = 0; i < 250; i++) - { - if (storeType == StoreType.Main) - _ = db.StringSet(keys[i], val); - else - _ = db.ListRightPush(keys[i], [val]); - } - - // Verify that overflow bucket allocations are non-zero after initial insertions - var currOverflowBucketAllocations = GetOverflowBucketAllocations(); - ClassicAssert.Greater(currOverflowBucketAllocations, 0); - var prevOverflowBucketAllocations = currOverflowBucketAllocations; - - // Try to set index size to a larger value than current - var result = db.Execute("CONFIG", "SET", option, largerSize1); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Verify that overflow bucket allocations have decreased - currOverflowBucketAllocations = GetOverflowBucketAllocations(); - ClassicAssert.Less(currOverflowBucketAllocations, prevOverflowBucketAllocations); - - // Insert second batch of data - for (var i = 250; i < 500; i++) - { - if (storeType == StoreType.Main) - _ = db.StringSet(keys[i], val); - else - _ = db.ListRightPush(keys[i], [val]); - } - - prevOverflowBucketAllocations = GetOverflowBucketAllocations(); - - // Try to set index size to a larger value than current - result = db.Execute("CONFIG", "SET", option, largerSize2); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Verify that overflow bucket allocations have decreased again - currOverflowBucketAllocations = GetOverflowBucketAllocations(); - ClassicAssert.Less(currOverflowBucketAllocations, prevOverflowBucketAllocations); - - // Verify that all keys still exist in the database - foreach (var key in keys) - { - ClassicAssert.IsTrue(db.KeyExists(key)); - } - - long GetOverflowBucketAllocations() => - storeType == StoreType.Main - ? server.Provider.StoreWrapper.store.OverflowBucketAllocations - : server.Provider.StoreWrapper.objectStore.OverflowBucketAllocations; - } - } - - /// - /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET. - /// - [AllureNUnit] - [TestFixture(false)] - [TestFixture(true)] - public class RespConfigHeapUtilizationTests : AllureTestBase - { - GarnetServer server; - private string memorySize = "3m"; - private string indexSize = "512"; - private string objectStoreLogMemorySize = "8192"; - private string objectStoreHeapMemorySize = "4096"; - private string pageSize = "1024"; - private bool useReviv; - - public RespConfigHeapUtilizationTests(bool useReviv) - { - this.useReviv = useReviv; - } - - [SetUp] - public void Setup() - { - TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(null, - memorySize: memorySize, - indexSize: indexSize, - pageSize: pageSize, - objectStorePageSize: pageSize, - objectStoreLogMemorySize: objectStoreLogMemorySize, - objectStoreIndexSize: indexSize, - objectStoreHeapMemorySize: objectStoreHeapMemorySize, - useReviv: useReviv); - server.Start(); - } - - [TearDown] - public void TearDown() - { - server.Dispose(); - TestUtils.OnTearDown(); - } - - /// - /// This test verifies that dynamically changing the object store heap size configuration using CONFIG SET - /// incurs a reduction in the empty page count of the object store. - /// - /// Heap size larger than configured size - [Test] - [TestCase("8192")] - public void ConfigSetHeapSizeUtilizationTest(string largerSize) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); - var db = redis.GetDatabase(0); - var option = "obj-heap-memory"; - - // Verify that initial empty page count is zero - var objectStore = server.Provider.StoreWrapper.objectStore; - ClassicAssert.AreEqual(0, objectStore.Log.EmptyPageCount); - - // Add objects to store to fill up heap - var values = new RedisValue[16]; - for (var i = 0; i < values.Length; i++) - values[i] = "x"; - - for (var i = 0; i < 8; i++) - { - var key = $"key{i++:00000}"; - _ = db.ListRightPush(key, values); - } - - // Wait for log size tracker - var sizeTrackerDelay = - TimeSpan.FromSeconds( - LogSizeTracker.ResizeTaskDelaySeconds + 2); - Thread.Sleep(sizeTrackerDelay); - - // Verify that empty page count has increased - ClassicAssert.Greater(objectStore.Log.EmptyPageCount, 0); - var prevEpc = objectStore.Log.EmptyPageCount; - - // Try to set heap size to a larger value than current - var result = db.Execute("CONFIG", "SET", option, largerSize); - ClassicAssert.AreEqual("OK", result.ToString()); - - // Wait for log size tracker - Thread.Sleep(sizeTrackerDelay); - - // Verify that empty page count has decreased - ClassicAssert.Less(objectStore.Log.EmptyPageCount, prevEpc); - } - } -} \ No newline at end of file diff --git a/test/cluster/Directory.Build.props b/test/cluster/Directory.Build.props new file mode 100644 index 00000000000..8df574cf7f0 --- /dev/null +++ b/test/cluster/Directory.Build.props @@ -0,0 +1,6 @@ + + + + $(MSBuildThisFileDirectory)garnet-cluster.runsettings + + diff --git a/test/Garnet.test.cluster/ClusterMigrateTLSTests.cs b/test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTLSTests.cs similarity index 96% rename from test/Garnet.test.cluster/ClusterMigrateTLSTests.cs rename to test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTLSTests.cs index 3d5056b9cec..e1ca2d28fe0 100644 --- a/test/Garnet.test.cluster/ClusterMigrateTLSTests.cs +++ b/test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTLSTests.cs @@ -3,14 +3,12 @@ using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture, NonParallelizable] - public class ClusterTLSMT : AllureTestBase + public class ClusterTLSMT : TestBase { ClusterMigrateTests tests; diff --git a/test/Garnet.test.cluster/ClusterMigrateTests.cs b/test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTests.cs similarity index 94% rename from test/Garnet.test.cluster/ClusterMigrateTests.cs rename to test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTests.cs index e5d06f51094..38f28091663 100644 --- a/test/Garnet.test.cluster/ClusterMigrateTests.cs +++ b/test/cluster/Garnet.test.cluster.migrate/ClusterMigrateTests.cs @@ -9,7 +9,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; #if DEBUG using Garnet.server; @@ -21,9 +20,8 @@ namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture(false), NonParallelizable] - public class ClusterMigrateTests(bool UseTLS) : AllureTestBase + public class ClusterMigrateTests(bool UseTLS) : TestBase { const int testTimeout = 100000; @@ -167,71 +165,6 @@ private int CreateSingleSlotData( return slot; } - private void CreateMultiSlotData( - int slotCount, - int keyLen, - int valueLen, - int keyTagEnd, - int keyCount, - out Dictionary> data, - HashSet restrictedToSlots = null) - { - var db = context.clusterTestUtils.GetMultiplexer().GetDatabase(0); - Dictionary slotsTokey = []; - data = []; - var key = new byte[keyLen]; - var value = new byte[valueLen]; - - ClassicAssert.IsTrue(slotCount < keyCount); - for (var i = 0; i < slotCount; i++) - { - ushort slot; - byte[] newKey; - do - { - restrictedSlot: - newKey = RandomBytes(key); - newKey[0] = (byte)'{'; - newKey[keyTagEnd] = (byte)'}'; - slot = ClusterTestUtils.HashSlot(newKey); - - if (restrictedToSlots != null && !restrictedToSlots.Contains(slot)) - goto restrictedSlot; - - } while (slotsTokey.ContainsKey(slot)); - slotsTokey.Add(slot, newKey); - data[slot] = new(new ByteArrayComparer()); - } - - int j = 0; - List slots = [.. slotsTokey.Keys]; - for (int i = 0; i < keyCount; i++) - { - key = slotsTokey[slots[j]]; - var newKey = new byte[key.Length]; - var newValue = new byte[value.Length]; - - Array.Copy(key, 0, newKey, 0, key.Length); - Array.Copy(value, 0, newValue, 0, value.Length); - RandomBytes(ref newKey, keyTagEnd + 1); - RandomBytes(ref newValue); - - var slot = ClusterTestUtils.HashSlot(newKey); - ClassicAssert.AreEqual(slot, slots[j]); - ClassicAssert.IsTrue(slotsTokey.ContainsKey((ushort)slot)); - - if (!data[slot].ContainsKey(newKey)) - data[slot].Add(newKey, newValue); - else - data[slot][newKey] = newValue; - - ClassicAssert.IsTrue(db.StringSet(newKey, newValue)); - var _v = (byte[])db.StringGet(newKey); - ClassicAssert.AreEqual(newValue, _v); - j = j + 1 < slots.Count ? j + 1 : 0; - } - } - [Test, Order(1)] [Category("CLUSTER")] public void ClusterSimpleInitialize() @@ -268,15 +201,15 @@ public void ClusterSimpleSlotInfo() context.logger.LogDebug("2. Creating slot data {keyCount} done", keyCount); var sourceIndex = context.clusterTestUtils.GetSourceNodeIndexFromSlot((ushort)slot, context.logger); - var expectedKeyCount = context.clusterTestUtils.CountKeysInSlot(slot, context.logger); - ClassicAssert.AreEqual(keyCount, expectedKeyCount); + var actualKeyCount = context.clusterTestUtils.CountKeysInSlot(slot, context.logger); + ClassicAssert.AreEqual(keyCount, actualKeyCount); _ = context.clusterTestUtils.CountKeysInSlot(-1, context.logger); _ = context.clusterTestUtils.CountKeysInSlot(ushort.MaxValue, context.logger); - var result = context.clusterTestUtils.GetKeysInSlot(sourceIndex, slot, expectedKeyCount, context.logger); + var result = context.clusterTestUtils.GetKeysInSlot(sourceIndex, slot, actualKeyCount, context.logger); ClassicAssert.AreEqual(keyCount, result.Count); - _ = context.clusterTestUtils.GetKeysInSlot(-1, expectedKeyCount); - _ = context.clusterTestUtils.GetKeysInSlot(ushort.MaxValue, expectedKeyCount); + _ = context.clusterTestUtils.GetKeysInSlot(-1, actualKeyCount); + _ = context.clusterTestUtils.GetKeysInSlot(ushort.MaxValue, actualKeyCount); context.logger.LogDebug("3. ClusterSimpleSlotInfoTest done"); } @@ -619,7 +552,7 @@ public void ClusterSimpleMigrateSlots() } context.logger.LogDebug("5. Checking keys done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); context.logger.LogDebug("6. Checking configuration update starting"); // Check if configuration has updated by var otherPorts = context.clusterTestUtils.GetEndPoints().Select(x => ((IPEndPoint)x).Port).Where(x => x != sourcePort || x != targetPort); @@ -643,7 +576,7 @@ public void ClusterSimpleMigrateSlots() } context.logger.LogDebug("7. Checking configuration update done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); context.logger.LogDebug("8. ClusterSimpleMigrateSlotsTest done"); } @@ -689,7 +622,7 @@ public void ClusterSimpleMigrateSlotsExpiry() ClassicAssert.AreEqual(keyExpiryCount / 2, keyCountRet); context.logger.LogDebug("8. Checking migrating keys done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); context.logger.LogDebug("9. ClusterSimpleMigrateSlotsExpiryTest done"); } @@ -881,7 +814,7 @@ public void ClusterSimpleMigrateSlotsWithObjects() context.logger.LogDebug("6. Checking migrated keys done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); (resp, members) = DoZRANGE(targetNodeIndex, key, out _Address, out _Port, out _Slot); ClassicAssert.AreEqual(memberPair.Select(x => Encoding.ASCII.GetString(x.Item2)).ToList(), members, $"MESSAGE: {resp} {count}"); context.logger.LogDebug("7. ClusterSimpleMigrateSlotsWithObjectsTest done"); @@ -933,7 +866,7 @@ public void ClusterSimpleMigrateKeys() context.logger.LogDebug("4. Set slot {_slot} to MIGRATING state on node {port}", _workingSlot, context.clusterTestUtils.GetEndPoint(sourceNodeIndex).Port); var countKeys = context.clusterTestUtils.CountKeysInSlot(sourceNodeIndex, _workingSlot, context.logger); - ClassicAssert.AreEqual(countKeys, keyCount); + ClassicAssert.AreEqual(keyCount, countKeys); context.logger.LogDebug("5. CountKeysInSlot {countKeys}", countKeys); var keysInSlot = context.clusterTestUtils.GetKeysInSlot(sourceNodeIndex, _workingSlot, countKeys, context.logger); @@ -979,17 +912,17 @@ public void ClusterSimpleMigrateKeys() { resp = context.clusterTestUtils.GetKey(otherNodeIndex, _key, out slot, out endpoint, out responseState, logger: context.logger); } - ClassicAssert.AreEqual(resp, "MOVED"); - ClassicAssert.AreEqual(_workingSlot, slot); - ClassicAssert.AreEqual(context.clusterTestUtils.GetEndPoint(targetNodeIndex), endpoint); + Assert.That(resp, Is.EqualTo("MOVED")); + Assert.That(slot, Is.EqualTo(_workingSlot)); + Assert.That(endpoint, Is.EqualTo(context.clusterTestUtils.GetEndPoint(targetNodeIndex))); resp = context.clusterTestUtils.GetKey(targetNodeIndex, _key, out _, out _, out responseState, logger: context.logger); - ClassicAssert.AreEqual(responseState, ResponseState.OK); - ClassicAssert.AreEqual(resp, _key); + Assert.That(responseState, Is.EqualTo(ResponseState.OK)); + Assert.That(resp, Is.EqualTo(_key)); } context.logger.LogDebug("14. Checking migrate keys done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); context.logger.LogDebug("15. ClusterSimpleMigrateKeysTest done"); } @@ -1060,7 +993,7 @@ public void ClusterSimpleMigrateKeysWithObjects() context.logger.LogDebug("6. Set slot {_slot} to MIGRATING state on node {port}", _slot, context.clusterTestUtils.GetEndPoint(sourceNodeIndex).Port); var countKeys = context.clusterTestUtils.CountKeysInSlot(sourceNodeIndex, _slot, context.logger); - ClassicAssert.AreEqual(countKeys, keyCount); + ClassicAssert.AreEqual(keyCount, countKeys); context.logger.LogDebug("7. CountKeysInSlot {countKeys}", countKeys); var keysInSlot = context.clusterTestUtils.GetKeysInSlot(sourceNodeIndex, _slot, countKeys, context.logger); @@ -1100,7 +1033,7 @@ public void ClusterSimpleMigrateKeysWithObjects() { var resp = DoZCOUNT(targetNodeIndex, key, out var count, out _Address, out _Port, out _Slot, logger: context.logger); ClassicAssert.AreEqual(resp, "OK"); - ClassicAssert.AreEqual(data[_key].Count, count); + ClassicAssert.AreEqual(data[_key].Count, count, $"key: {Encoding.UTF8.GetString(_key)}"); List members; (resp, members) = DoZRANGE(targetNodeIndex, _key, out _Address, out _Port, out _Slot, context.logger); @@ -1109,7 +1042,7 @@ public void ClusterSimpleMigrateKeysWithObjects() context.logger.LogDebug("2. Loading object keys data done"); } context.logger.LogDebug("15. Checking migrate keys done"); - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); context.logger.LogDebug("16. ClusterSimpleMigrateKeysWithObjectsTest done"); } @@ -1346,7 +1279,7 @@ public void ClusterSimpleMigrateSlotsRanges(List migrateRange) _ = Thread.Yield(); } - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); } [Test, Order(14)] @@ -1399,7 +1332,7 @@ public void ClusterSimpleMigrateWithAuth(List migrateRange) _ = Thread.Yield(); } - context.clusterTestUtils.WaitForMigrationCleanup(context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(logger: context.logger); } [Test, Order(15)] @@ -2337,21 +2270,22 @@ public void ClusterMigrateSetSlotRangeResilience() { context.logger?.LogDebug("0. ClusterMigrateSetSlotRangeResilience started"); var shards = 2; + var sourceNodeIndex = 0; + var targetNodeIndex = 1; context.CreateInstances(shards, useTLS: UseTLS); context.CreateConnection(useTLS: UseTLS); // Setup: node 0 owns all slots, node 1 owns none _ = context.clusterTestUtils.AddDelSlotsRange(0, [(0, 16383)], addslot: true, logger: context.logger); - context.clusterTestUtils.SetConfigEpoch(0, 1, logger: context.logger); - context.clusterTestUtils.SetConfigEpoch(1, 2, logger: context.logger); - context.clusterTestUtils.Meet(0, 1, logger: context.logger); - context.clusterTestUtils.WaitUntilNodeIsKnown(1, 0, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(sourceNodeIndex, 1, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(targetNodeIndex, 2, logger: context.logger); + context.clusterTestUtils.Meet(sourceNodeIndex, targetNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(targetNodeIndex, sourceNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(sourceNodeIndex, targetNodeIndex, logger: context.logger); // Create data in a single slot using the standard helper var keyCount = 50; var slot = CreateSingleSlotData(keyLen: 16, valueLen: 16, keyTagEnd: 6, keyCount, out var data); - var sourceNodeIndex = 0; - var targetNodeIndex = 1; context.logger?.LogDebug("1. Verifying data insertion into slot {slot}", slot); var actualKeyCount = context.clusterTestUtils.CountKeysInSlot(sourceNodeIndex, slot, logger: context.logger); @@ -2395,5 +2329,79 @@ public void ClusterMigrateSetSlotRangeResilience() context.logger?.LogDebug("6. ClusterMigrateSetSlotRangeResilience completed"); } + + /// + /// Regression test for the lookup-based DeleteSlotKeys conversion (Task 4 of the + /// "tempKv elimination" change): verifies that CLUSTER DELKEYSINSLOT removes both + /// raw-string keys and collection-object keys from the targeted slot via a single + /// IterateLookup pass (no parallel tempKv), AND that keys in a different slot are + /// not collateral-deleted (the slot-set filter inside the scan callback works). + /// + [Test, Order(28)] + [Category("CLUSTER")] + public void ClusterDelKeysInSlotRemovesStringAndObjectKeys() + { + context.CreateInstances(defaultShards, useTLS: UseTLS); + context.CreateConnection(useTLS: UseTLS); + _ = context.clusterTestUtils.SimpleSetupCluster(logger: context.logger); + + // Pick two distinct slots owned by node 0: + // delSlot - target of CLUSTER DELKEYSINSLOT (keys here should be deleted) + // keepSlot - control (keys here must survive) + var ownedSlots = context.clusterTestUtils.GetOwnedSlotsFromNode(0, context.logger); + ClassicAssert.IsTrue(ownedSlots.Count >= 2, "Need at least two slots on node 0 for this test"); + var delSlot = ownedSlots[0]; + var keepSlot = ownedSlots[1]; + ClassicAssert.AreNotEqual(delSlot, keepSlot, "delSlot and keepSlot must differ"); + + // Populate one raw-string key and one collection-object key in EACH slot. + var delStringKey = new byte[16]; + var delObjectKey = new byte[16]; + var keepStringKey = new byte[16]; + var keepObjectKey = new byte[16]; + context.clusterTestUtils.RandomBytesRestrictedToSlot(ref delStringKey, delSlot); + context.clusterTestUtils.RandomBytesRestrictedToSlot(ref delObjectKey, delSlot); + context.clusterTestUtils.RandomBytesRestrictedToSlot(ref keepStringKey, keepSlot); + context.clusterTestUtils.RandomBytesRestrictedToSlot(ref keepObjectKey, keepSlot); + + var server = context.clusterTestUtils.GetServer(0); + _ = server.Execute("SET", delStringKey, "raw-del"); + _ = server.Execute("SADD", delObjectKey, "del-m1", "del-m2", "del-m3"); + _ = server.Execute("SET", keepStringKey, "raw-keep"); + _ = server.Execute("SADD", keepObjectKey, "keep-m1", "keep-m2", "keep-m3"); + + // Both slots should each contain their two keys before deletion. + ClassicAssert.AreEqual(2, context.clusterTestUtils.CountKeysInSlot(0, delSlot, context.logger), + "delSlot should contain both string and object keys before DELKEYSINSLOT"); + ClassicAssert.AreEqual(2, context.clusterTestUtils.CountKeysInSlot(0, keepSlot, context.logger), + "keepSlot should contain both string and object keys before DELKEYSINSLOT"); + + // Run CLUSTER DELKEYSINSLOT on delSlot — exercises StorageSession.DeleteSlotKeys via + // the unified IterateLookup path. Verifies that the unified scan covers both record + // types AND that the slot-set filter inside DeleteSlotKeysScan.Reader correctly + // skips records belonging to other slots. + var resp = (string)server.Execute("CLUSTER", "DELKEYSINSLOT", delSlot.ToString()); + ClassicAssert.AreEqual("OK", resp); + + // delSlot should be empty; keepSlot must be untouched. + ClassicAssert.AreEqual(0, context.clusterTestUtils.CountKeysInSlot(0, delSlot, context.logger), + "Both string and object keys in delSlot should be deleted by DELKEYSINSLOT"); + ClassicAssert.AreEqual(2, context.clusterTestUtils.CountKeysInSlot(0, keepSlot, context.logger), + "Keys in keepSlot must NOT be collateral-deleted by DELKEYSINSLOT on delSlot"); + + // Direct GET / EXISTS confirm the delSlot keys are gone. + var delStringRes = server.Execute("GET", delStringKey); + ClassicAssert.IsTrue(delStringRes.IsNull, "delSlot string key should no longer exist after DELKEYSINSLOT"); + var delObjectRes = (long)server.Execute("EXISTS", delObjectKey); + ClassicAssert.AreEqual(0, delObjectRes, "delSlot object key should no longer exist after DELKEYSINSLOT"); + + // Direct GET / EXISTS confirm the keepSlot keys still exist with their original payloads. + var keepStringRes = (string)server.Execute("GET", keepStringKey); + ClassicAssert.AreEqual("raw-keep", keepStringRes, "keepSlot string key value should be unchanged"); + var keepObjectExists = (long)server.Execute("EXISTS", keepObjectKey); + ClassicAssert.AreEqual(1, keepObjectExists, "keepSlot object key should still exist after DELKEYSINSLOT"); + var keepObjectCard = (long)server.Execute("SCARD", keepObjectKey); + ClassicAssert.AreEqual(3, keepObjectCard, "keepSlot object key should still contain all 3 members"); + } } } \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.migrate/Garnet.test.cluster.migrate.csproj b/test/cluster/Garnet.test.cluster.migrate/Garnet.test.cluster.migrate.csproj new file mode 100644 index 00000000000..0db6c2b85b3 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.migrate/Garnet.test.cluster.migrate.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/BaseCommand.cs similarity index 100% rename from test/Garnet.test.cluster/RedirectTests/BaseCommand.cs rename to test/cluster/Garnet.test.cluster.migrate/RedirectTests/BaseCommand.cs diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/ClusterSlotVerificationTests.cs similarity index 93% rename from test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs rename to test/cluster/Garnet.test.cluster.migrate/RedirectTests/ClusterSlotVerificationTests.cs index 05e106bcc6e..4b16070f8a0 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/ClusterSlotVerificationTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using Allure.NUnit; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -22,11 +21,9 @@ public sealed class BaseCommandComparer : IEqualityComparer public unsafe int GetHashCode([DisallowNull] BaseCommand obj) => obj.Command.GetHashCode(); } - - [AllureNUnit] [TestFixture] [NonParallelizable] - public class ClusterSlotVerificationTests : AllureTestBase + public class ClusterSlotVerificationTests : TestBase { static readonly HashSet TestCommands = new(BaseCommandComparer.Instance) { @@ -155,6 +152,10 @@ public class ClusterSlotVerificationTests : AllureTestBase new HCOLLECT(), new CLUSTERGETPROC(), new CLUSTERSETPROC(), + new CLUSTERTESTRAWCMD(), + new CLUSTERTESTOBJCMD(), + new CLUSTERTESTRAWREADCMD(), + new CLUSTERTESTOBJREADCMD(), new WATCH(), new WATCHMS(), new WATCHOS(), @@ -233,6 +234,38 @@ public void OneTimeSetUp() () => new TestClusterReadWriteCustomTxn(), new RespCommandsInfo { Arity = TestClusterReadWriteCustomTxn.Arity }); + // Register write and read variants of CustomRawStringCmd and CustomObjCmd so the + // readOnly=true and readOnly=false branches of CanServeSlot are exercised by the + // standard CLUSTERDOWN/OK/MOVED/ASK tests + foreach (var node in context.nodes) + { + _ = node.Register.NewCommand( + "CLUSTERTESTRAWCMD", + CommandType.ReadModifyWrite, + new TestClusterRawStringCmd(), + new RespCommandsInfo { Arity = 3 }); + + _ = node.Register.NewCommand( + "CLUSTERTESTOBJCMD", + CommandType.ReadModifyWrite, + new TestClusterObjFactory(), + new TestClusterObjSet(), + new RespCommandsInfo { Arity = 3 }); + + _ = node.Register.NewCommand( + "CLUSTERTESTRAWREADCMD", + CommandType.Read, + new TestClusterRawStringReadCmd(), + new RespCommandsInfo { Arity = 2 }); + + _ = node.Register.NewCommand( + "CLUSTERTESTOBJREADCMD", + CommandType.Read, + new TestClusterObjFactory(), + new TestClusterObjGet(), + new RespCommandsInfo { Arity = 2 }); + } + context.CreateConnection(); // Assign all slots to node 0 diff --git a/test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterCustomCommands.cs b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterCustomCommands.cs new file mode 100644 index 00000000000..bc88bed83b9 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterCustomCommands.cs @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using Garnet.common; +using Garnet.server; +using Tsavorite.core; + +namespace Garnet.test.cluster +{ + // Write-only custom raw-string command: overwrites the value with the first input arg + sealed class TestClusterRawStringCmd : CustomRawStringFunctions + { + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + => throw new InvalidOperationException(); + + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) + => true; + + public override int GetInitialLength(ref StringInput input) + => GetFirstArg(ref input).Length; + + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + { + GetFirstArg(ref input).CopyTo(value); + return true; + } + + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + { + var newVal = GetFirstArg(ref input); + if (newVal.Length > value.Length) + return false; // fall back to CopyUpdater + + newVal.CopyTo(value); + valueLength = newVal.Length; + return true; + } + + public override bool NeedCopyUpdate(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, ref RespMemoryWriter writer) + => true; + + public override int GetLength(ReadOnlySpan value, ref StringInput input) + => GetFirstArg(ref input).Length; + + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + { + GetFirstArg(ref input).CopyTo(newValue); + return true; + } + } + + // Read-only custom raw-string command for the readOnly=true slot-verification branch + sealed class TestClusterRawStringReadCmd : CustomRawStringFunctions + { + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + { + writer.WriteBulkString(value); + return true; + } + + // The dispatcher only invokes write methods for ReadModifyWrite commands, so throws + // here surface accidental write-path invocations as test failures instead of silent state corruption + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) + => throw new InvalidOperationException(); + + public override int GetInitialLength(ref StringInput input) + => throw new InvalidOperationException(); + + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + => throw new InvalidOperationException(); + + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + => throw new InvalidOperationException(); + + public override int GetLength(ReadOnlySpan value, ref StringInput input) + => throw new InvalidOperationException(); + + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + => throw new InvalidOperationException(); + } + + sealed class TestClusterObjFactory : CustomObjectFactory + { + public override CustomObjectBase Create(byte type) + => new TestClusterObj(type); + + public override CustomObjectBase Deserialize(byte type, BinaryReader reader) + => new TestClusterObj(type); + } + + // Empty stub used only to satisfy the CustomObjectBase contract; tests assert dispatch routing only + sealed class TestClusterObj : CustomObjectBase + { + public TestClusterObj(byte type) : base(type) { } + + public TestClusterObj(TestClusterObj obj) : base(obj) { } + + public override CustomObjectBase CloneObject() => new TestClusterObj(this); + + public override void SerializeObject(BinaryWriter writer) { } + + public override void Dispose() { } + + public override unsafe void Scan(long start, out List items, out long cursor, int count = 10, byte* pattern = null, int patternLength = 0, bool isNoValue = false) + { + items = []; + cursor = 0; + } + } + + sealed class TestClusterObjSet : CustomObjectFunctions + { + public override bool NeedInitialUpdate(scoped ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) + => true; + + // No-op; returning true emits the default +OK reply + public override bool Updater(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) + => true; + } + + // Read-only custom object command for the readOnly=true slot-verification branch + sealed class TestClusterObjGet : CustomObjectFunctions + { + public override bool Reader(ReadOnlySpan key, ref ObjectInput input, IGarnetObject value, ref RespMemoryWriter writer, ref ReadInfo readInfo) + { + writer.WriteNull(); + return true; + } + } + + internal class CLUSTERTESTRAWCMD : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(CLUSTERTESTRAWCMD); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "value"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + internal class CLUSTERTESTOBJCMD : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(CLUSTERTESTOBJCMD); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], "value"]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + // Read-mode custom raw-string command: only the key argument is needed + internal class CLUSTERTESTRAWREADCMD : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(CLUSTERTESTRAWREADCMD); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0]]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } + + // Read-mode custom object command: only the key argument is needed + internal class CLUSTERTESTOBJREADCMD : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(CLUSTERTESTOBJREADCMD); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0]]; + } + + public override string[] GetCrossSlotRequest() => throw new NotImplementedException(); + + public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/test/Garnet.test.cluster/RedirectTests/TestClusterProc.cs b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterProc.cs similarity index 89% rename from test/Garnet.test.cluster/RedirectTests/TestClusterProc.cs rename to test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterProc.cs index 9d793d0f952..3115b2e2252 100644 --- a/test/Garnet.test.cluster/RedirectTests/TestClusterProc.cs +++ b/test/cluster/Garnet.test.cluster.migrate/RedirectTests/TestClusterProc.cs @@ -30,9 +30,9 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var getB = GetNextArg(ref procInput, ref offset); var getC = GetNextArg(ref procInput, ref offset); - AddKey(getA, LockType.Shared, isObject: false); - AddKey(getB, LockType.Shared, isObject: false); - AddKey(getC, LockType.Shared, isObject: false); + AddKey(getA, LockType.Shared, StoreType.Main); + AddKey(getB, LockType.Shared, StoreType.Main); + AddKey(getC, LockType.Shared, StoreType.Main); return true; } @@ -44,11 +44,11 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var getB = GetNextArg(ref procInput, ref offset); var getC = GetNextArg(ref procInput, ref offset); - var status = api.GET(getA, out _); + var status = api.GET(getA, out PinnedSpanByte _); ClassicAssert.AreEqual(GarnetStatus.NOTFOUND, status); - _ = api.GET(getB, out _); + _ = api.GET(getB, out PinnedSpanByte _); ClassicAssert.AreEqual(GarnetStatus.NOTFOUND, status); - _ = api.GET(getC, out _); + _ = api.GET(getC, out PinnedSpanByte _); ClassicAssert.AreEqual(GarnetStatus.NOTFOUND, status); WriteSimpleString(ref output, "SUCCESS"); } @@ -95,7 +95,6 @@ sealed class TestClusterReadWriteCustomTxn : CustomTransactionProcedure /// CLUSTERSETPROC key1 key2 key3 /// /// - /// /// public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { @@ -104,9 +103,9 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var setB = GetNextArg(ref procInput, ref offset); var setC = GetNextArg(ref procInput, ref offset); - AddKey(getA, LockType.Shared, isObject: false); - AddKey(setB, LockType.Exclusive, isObject: false); - AddKey(setC, LockType.Exclusive, isObject: false); + AddKey(getA, LockType.Shared, StoreType.Main); + AddKey(setB, LockType.Exclusive, StoreType.Main); + AddKey(setC, LockType.Exclusive, StoreType.Main); return true; } @@ -118,7 +117,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var setB = GetNextArg(ref procInput, ref offset); var setC = GetNextArg(ref procInput, ref offset); - _ = api.GET(getA, out _); + _ = api.GET(getA, out PinnedSpanByte _); var status = api.SET(setB, setB); ClassicAssert.AreEqual(GarnetStatus.OK, status); status = api.SET(setC, setC); diff --git a/test/cluster/Garnet.test.cluster.migrate/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.migrate/TestProjectSetup.cs new file mode 100644 index 00000000000..fc523f48339 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.migrate/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterMigrate; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.multilog.diskless/Garnet.test.cluster.multilog.diskless.csproj b/test/cluster/Garnet.test.cluster.multilog.diskless/Garnet.test.cluster.multilog.diskless.csproj new file mode 100644 index 00000000000..1dd9a0f4fa4 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.multilog.diskless/Garnet.test.cluster.multilog.diskless.csproj @@ -0,0 +1,48 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + false + + diff --git a/test/cluster/Garnet.test.cluster.multilog.diskless/MultiLogTests/ClusterReplicationDisklessSyncShardedLog.cs b/test/cluster/Garnet.test.cluster.multilog.diskless/MultiLogTests/ClusterReplicationDisklessSyncShardedLog.cs new file mode 100644 index 00000000000..2272c057540 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.multilog.diskless/MultiLogTests/ClusterReplicationDisklessSyncShardedLog.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using NUnit.Framework; + +namespace Garnet.test.cluster.MultiLogTests +{ + [TestFixture] + [NonParallelizable] + public class ClusterReplicationDisklessSyncShardedLog : ClusterReplicationDisklessSyncTests + { + const int TestSublogCount = 2; + + public Dictionary enabledTests = new() + { + {"ClusterEmptyReplicaDisklessSync", true}, + {"ClusterAofReplayDisklessSync", true}, + {"ClusterDBVersionAlignmentDisklessSync", true}, + {"ClusterDisklessSyncParallelAttach", true}, + {"ClusterDisklessSyncFailover", true}, + {"ClusterDisklessSyncResetSyncManagerCts", true}, + }; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var methods = typeof(ClusterReplicationShardedLog).GetMethods().Where(static mtd => mtd.GetCustomAttribute() != null); + foreach (var method in methods) + enabledTests.TryAdd(method.Name, true); + } + + [SetUp] + public override void Setup() + { + var testName = TestContext.CurrentContext.Test.MethodName; + if (!enabledTests.TryGetValue(testName, out var isEnabled) || !isEnabled) + { + Assert.Ignore($"Skipping {testName} for {nameof(ClusterReplicationShardedLog)}"); + } + asyncReplay = false; + sublogCount = TestSublogCount; + base.Setup(); + } + + [TearDown] + public override void TearDown() + { + var testName = TestContext.CurrentContext.Test.MethodName; + if (!enabledTests.TryGetValue(testName, out var isEnabled) || !isEnabled) + { + Assert.Ignore($"Skipping {testName} for {nameof(ClusterReplicationShardedLog)}"); + } + base.TearDown(); + } + } +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.multilog/Garnet.test.cluster.multilog.csproj b/test/cluster/Garnet.test.cluster.multilog/Garnet.test.cluster.multilog.csproj new file mode 100644 index 00000000000..e29bd3169f8 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.multilog/Garnet.test.cluster.multilog.csproj @@ -0,0 +1,48 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + false + + diff --git a/test/cluster/Garnet.test.cluster.multilog/MultiLogTests/ClusterReplicationShardedLog.cs b/test/cluster/Garnet.test.cluster.multilog/MultiLogTests/ClusterReplicationShardedLog.cs new file mode 100644 index 00000000000..e5728fb54bd --- /dev/null +++ b/test/cluster/Garnet.test.cluster.multilog/MultiLogTests/ClusterReplicationShardedLog.cs @@ -0,0 +1,466 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; + +namespace Garnet.test.cluster.MultiLogTests +{ + [TestFixture] + [NonParallelizable] + public class ClusterReplicationShardedLog : ClusterReplicationBaseTests + { + const int TestSublogCount = 2; + const int TestReplayTaskCount = 3; + + public Dictionary enabledTests = new() + { + {"ClusterSRTest", true}, + {"ClusterSRNoCheckpointRestartSecondary", true}, + {"ClusterSRPrimaryCheckpoint", true}, + {"ClusterCheckpointRetrieveDisableStorageTier", true}, + {"ClusterSRAddReplicaAfterPrimaryCheckpoint", true}, + {"ClusterSRPrimaryRestart", true}, + {"ClusterSRRedirectWrites", false}, // Does not test AOF + {"ClusterSRReplicaOfTest", true}, + {"ClusterReplicationSimpleFailover", true}, + {"ClusterFailoverAttachReplicas", true}, + {"ClusterReplicationCheckpointCleanupTest", true}, + {"ClusterMainMemoryReplicationAttachReplicas", true}, + {"ClusterDivergentReplicasTest", true}, + {"ClusterDivergentCheckpointTest", true}, + {"ClusterDivergentReplicasMMTest", true}, + {"ClusterDivergentCheckpointMMTest", true}, + {"ClusterDivergentCheckpointMMFastCommitTest", true}, + {"ClusterReplicationCheckpointAlignmentTest", true}, + {"ClusterReplicationLua", true}, + {"ClusterReplicationStoredProc", false}, // Duplicate test in this class + {"ClusterReplicationManualCheckpointing", true}, + {"ReplicaSyncTaskFaultsRecoverAsync", true}, + {"ClusterReplicationMultiRestartRecover", false}, + {"ClusterReplicationDivergentHistoryWithoutCheckpoint", true}, + {"ClusterReplicationSimpleTransactionTest", false} // Duplicate test in this class + }; + + [OneTimeSetUp] + public void OneTimeSetUp() + { + var methods = typeof(ClusterReplicationShardedLog).GetMethods().Where(static mtd => mtd.GetCustomAttribute() != null); + foreach (var method in methods) + enabledTests.TryAdd(method.Name, true); + + monitorTests.Add("ClusterReplicationShardedLogTxnTest", LogLevel.Warning); + } + + [SetUp] + public override void Setup() + { + var testName = TestContext.CurrentContext.Test.MethodName; + if (!enabledTests.TryGetValue(testName, out var isEnabled) || !isEnabled) + { + Assert.Ignore($"Skipping {testName} for {nameof(ClusterReplicationShardedLog)}"); + } + asyncReplay = false; + sublogCount = TestSublogCount; + + base.Setup(); + } + + [TearDown] + public override void TearDown() + { + var testName = TestContext.CurrentContext.Test.MethodName; + if (!enabledTests.TryGetValue(testName, out var isEnabled) || !isEnabled) + { + Assert.Ignore($"Skipping {testName} for {nameof(ClusterReplicationShardedLog)}"); + } + base.TearDown(); + } + + [Test, Order(1)] + [Category("REPLICATION")] + public void ClusterReplicationShardedLogTxnTest([Values] bool storedProcedure, [Values] bool testParallelReplay) + { + var replica_count = 1;// Per primary + var primary_count = 1; + var nodes_count = primary_count + (primary_count * replica_count); + var primaryNodeIndex = 0; + var replicaNodeIndex = 1; + + context.CreateInstances(nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: testParallelReplay ? 1 : TestSublogCount, + replayTaskCount: testParallelReplay ? TestReplayTaskCount : 1); + context.CreateConnection(useTLS: useTLS); + + var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); + var replicaServer = context.clusterTestUtils.GetServer(replicaNodeIndex); + + // Register custom procedure + if (storedProcedure) + { + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); + _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); + } + + // Setup cluster + var resp = context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + ClassicAssert.AreEqual("OK", resp); + + context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(replicaNodeIndex, replicaNodeIndex + 1, logger: context.logger); + context.clusterTestUtils.Meet(primaryNodeIndex, replicaNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(primaryNodeIndex, replicaNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(replicaNodeIndex, primaryNodeIndex, logger: context.logger); + + // Attach replica + resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryNodeIndex, logger: context.logger); + ClassicAssert.AreEqual("OK", resp); + context.clusterTestUtils.WaitForReplicaRecovery(replicaNodeIndex, logger: context.logger); + + string[] keys = ["{_}a", "{_}b", "{_}c", "{_}x", "{_}y", "{_}z"]; + string[] values = ["10", "15", "20", "25", "30", "35"]; + + if (storedProcedure) + ClusterTestContext.ExecuteStoredProcBulkIncrement(primaryServer, keys, values); + else + context.ExecuteTxnBulkIncrement(keys, values); + + // Check keys at primary + for (var i = 0; i < keys.Length; i++) + { + resp = context.clusterTestUtils.GetKey(primaryNodeIndex, Encoding.ASCII.GetBytes(keys[i]), out _, out _, out _); + ClassicAssert.AreEqual(values[i], resp, "At primary"); + } + context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); + + // Check keys at replica + for (var i = 0; i < keys.Length; i++) + { + resp = context.clusterTestUtils.GetKey(replicaNodeIndex, Encoding.ASCII.GetBytes(keys[i]), out _, out _, out _); + ClassicAssert.AreEqual(values[i], resp, "At replica"); + } + + if (storedProcedure) + { + var result = ClusterTestContext.ExecuteBulkReadStoredProc(replicaServer, keys); + ClassicAssert.AreEqual(values, result); + } + else + { + var result = context.ExecuteTxnBulkRead(replicaServer, keys); + ClassicAssert.AreEqual(values, result); + } + + var primaryPInfo = context.clusterTestUtils.GetPersistenceInfo(primaryNodeIndex, context.logger); + var replicaPInfo = context.clusterTestUtils.GetPersistenceInfo(primaryNodeIndex, context.logger); + ClassicAssert.AreEqual(primaryPInfo.TailAddress, replicaPInfo.TailAddress); + var primaryReplOffset = context.clusterTestUtils.GetReplicationOffset(0); + var replicaReplOffset = context.clusterTestUtils.GetReplicationOffset(1); + } + + [Test, Order(2)] + [Category("REPLICATION")] + public async Task ClusterReplicationShardedLogRecoverAsync() + { + var primary_count = 1; + var primaryNodeIndex = 0; + + context.CreateInstances( + primary_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: TestSublogCount); + context.CreateConnection(useTLS: useTLS); + + _ = context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger); + + var keyLength = 16; + var kvpairCount = keyCount; + context.kvPairs = []; + + //Populate Primary + context.PopulatePrimary(ref context.kvPairs, keyLength, kvpairCount, 0); + + var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); + var expectedKeys = (string[])primaryServer.Execute("KEYS", ["*"]); + await context.nodes[primaryNodeIndex].Store.CommitAOFAsync(default).ConfigureAwait(false); + + // Shutdown node + context.nodes[primaryNodeIndex].Dispose(false); + + // Restart secondary + context.nodes[primaryNodeIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(primaryNodeIndex), + tryRecover: true, + enableAOF: true, + useTLS: useTLS, + cleanClusterConfig: false, + asyncReplay: asyncReplay, + sublogCount: TestSublogCount); + context.nodes[primaryNodeIndex].Start(); + context.CreateConnection(useTLS: useTLS); + + primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); + var keys = (string[])primaryServer.Execute("KEYS", ["*"]); + Array.Sort(keys); + Array.Sort(expectedKeys); + ClassicAssert.AreEqual(expectedKeys.Length, keys.Length); + ClassicAssert.AreEqual(expectedKeys, keys); + } + + [Test, Order(3)] + [Category("REPLICATION")] + public void ClusterReplicationSimpleMultiReplay() + { + var replica_count = 1;// Per primary + var primary_count = 1; + var nodes_count = primary_count + (primary_count * replica_count); + var primaryNodeIndex = 0; + var replicaNodeIndex = 1; + + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: 1, + replayTaskCount: TestReplayTaskCount); + context.CreateConnection(useTLS: useTLS); + + // Setup cluster + context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(replicaNodeIndex, replicaNodeIndex + 1, logger: context.logger); + context.clusterTestUtils.Meet(primaryNodeIndex, replicaNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(primaryNodeIndex, replicaNodeIndex, logger: context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(replicaNodeIndex, primaryNodeIndex, logger: context.logger); + + // Attach replica + var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryNodeIndex, logger: context.logger); + ClassicAssert.AreEqual("OK", resp); + context.clusterTestUtils.WaitForReplicaRecovery(replicaNodeIndex, logger: context.logger); + + var keyLength = 16; + var kvpairCount = 2; + context.kvPairs = []; + + // Populate Primary + context.SimplePopulateDB(disableObjects: true, keyLength, kvpairCount, primaryNodeIndex); + + // Wait for replica to sync + context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex); + + // Validate database + context.SimpleValidateDB(disableObjects: true, replicaNodeIndex); + } + + [Test, Order(4)] + [Category("REPLICATION")] + public void ClusterParallelReplicationUpgrade([Values] bool upgradeFromSingleLog, [Values] bool disableObjects) + { + var nodes_count = 3; + var singleLogNodeIndex = 0; + var multiLogNodeIndex = 1; + var replicaIndex = 2; + var timeout = 60; + var sourceIndex = upgradeFromSingleLog ? singleLogNodeIndex : multiLogNodeIndex; + var targetIndex = upgradeFromSingleLog ? multiLogNodeIndex : singleLogNodeIndex; + + context.nodes = new GarnetServer[nodes_count]; + context.endpoints = TestUtils.GetShardEndPoints(nodes_count, IPAddress.Loopback, ClusterTestContext.Port); + + context.clusterTestUtils = new ClusterTestUtils( + context.endpoints, + context: context, + textWriter: context.logTextWriter, + UseTLS: false, + authUsername: null, + authPassword: null, + certificates: null); + + // Create nodes with single log + context.nodes[singleLogNodeIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(singleLogNodeIndex), + disableObjects: disableObjects, + enableAOF: true, + timeout: timeout, + sublogCount: 1); + context.nodes[singleLogNodeIndex].Start(); + + // Create nodes with multi-log + context.nodes[multiLogNodeIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(multiLogNodeIndex), + disableObjects: disableObjects, + enableAOF: true, + timeout: timeout, + sublogCount: TestSublogCount); + context.nodes[multiLogNodeIndex].Start(); + + // Create replica with single or multi-log configuration + context.nodes[replicaIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(replicaIndex), + disableObjects: disableObjects, + enableAOF: true, + timeout: timeout, + sublogCount: upgradeFromSingleLog ? TestSublogCount : 1); + context.nodes[replicaIndex].Start(); + + // Create connection + context.CreateConnection(useTLS: useTLS); + + // Assign slot to source node + ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddSlotsRange(sourceIndex, [(0, 16383)], context.logger)); + + // Set config epoch + for (var i = 0; i < nodes_count; i++) + context.clusterTestUtils.SetConfigEpoch(i, i + 1, context.logger); + + // Introduce nodes + for (var i = 1; i < nodes_count; i++) + context.clusterTestUtils.Meet(0, i, context.logger); + + // Wait for gossip to propagate + for (var i = 0; i < nodes_count; i++) + { + for (var j = 0; j < nodes_count; j++) + { + if (i == j) continue; + context.clusterTestUtils.WaitUntilNodeIsKnown(i, j, context.logger); + context.clusterTestUtils.WaitUntilNodeIsKnown(j, i, context.logger); + } + } + + // Add replica + ClassicAssert.AreEqual("OK", context.clusterTestUtils.ClusterReplicate(replicaIndex, targetIndex, logger: context.logger)); + context.clusterTestUtils.WaitForReplicaRecovery(replicaIndex, logger: context.logger); + + var keyLength = 16; + var kvpairCount = keyCount; + context.kvPairs = []; + context.kvPairsObj = []; + + // Populate node + context.SimplePopulateDB(disableObjects: disableObjects, keyLength, kvpairCount, sourceIndex); + + // Migrate slots + var sourcePort = context.clusterTestUtils.GetEndPoint(sourceIndex); + var targetPort = context.clusterTestUtils.GetEndPoint(targetIndex); + context.clusterTestUtils.MigrateSlots(sourcePort, targetPort, [0, 16383], range: true, logger: context.logger); + context.clusterTestUtils.WaitForMigrationCleanup(sourceIndex, logger: context.logger); + context.clusterTestUtils.WaitForSlotOwnership(replicaIndex, context.clusterTestUtils.GetNodeIdFromNode(targetIndex, context.logger), [0, 16383], context.logger); + + // Validate migrated keys + context.SimpleValidateDB(disableObjects, targetIndex); + context.SimpleValidateDB(disableObjects, replicaIndex); + } + + [Test, Order(5)] + [Category("REPLICATION")] + public async Task ClusterAofUpgradeSLtoSLMRRecoverAsync([Values] bool useStoredProcedure) + { + var primary_count = 1; + var primaryNodeIndex = 0; + + // Phase 1: Start in single-log single-replay mode (SL) and write data + context.CreateInstances( + primary_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: 1, + replayTaskCount: 1); + context.CreateConnection(useTLS: useTLS); + + _ = context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger); + + var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); + + // Register stored proc if needed + if (useStoredProcedure) + { + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); + } + + // Write standalone ops + string[] keys = ["{_}a", "{_}b", "{_}c", "{_}x", "{_}y", "{_}z"]; + string[] values = ["10", "15", "20", "25", "30", "35"]; + + if (useStoredProcedure) + { + ClusterTestContext.ExecuteStoredProcBulkIncrement(primaryServer, keys, values); + } + else + { + context.ExecuteTxnBulkIncrement(keys, values); + } + + // Verify at primary before upgrade + for (var i = 0; i < keys.Length; i++) + { + var resp = context.clusterTestUtils.GetKey(primaryNodeIndex, Encoding.ASCII.GetBytes(keys[i]), out _, out _, out _); + ClassicAssert.AreEqual(values[i], resp, $"At primary before upgrade, key {keys[i]}"); + } + + // Commit AOF (no checkpoint) and stop + await context.nodes[primaryNodeIndex].Store.CommitAOFAsync(default).ConfigureAwait(false); + context.nodes[primaryNodeIndex].Dispose(false); + + // Phase 2: Restart with multi-replay config (SLMR) — AOF replay uses multiple tasks + context.nodes[primaryNodeIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(primaryNodeIndex), + tryRecover: true, + enableAOF: true, + useTLS: useTLS, + cleanClusterConfig: false, + asyncReplay: asyncReplay, + sublogCount: 1, + replayTaskCount: TestReplayTaskCount); + + if (useStoredProcedure) + { + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); + } + + context.nodes[primaryNodeIndex].Start(); + context.CreateConnection(useTLS: useTLS); + + // Verify SL-era data survived the multi-replay recovery + for (var i = 0; i < keys.Length; i++) + { + var resp = context.clusterTestUtils.GetKey(primaryNodeIndex, Encoding.ASCII.GetBytes(keys[i]), out _, out _, out _); + ClassicAssert.AreEqual(values[i], resp, $"At primary after SLMR upgrade, key {keys[i]}"); + } + + if (useStoredProcedure) + { + primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); + var result = ClusterTestContext.ExecuteBulkReadStoredProc(primaryServer, keys); + ClassicAssert.AreEqual(values, result); + } + } + } +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.multilog/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.multilog/TestProjectSetup.cs new file mode 100644 index 00000000000..94d11d7123c --- /dev/null +++ b/test/cluster/Garnet.test.cluster.multilog/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterMultiLog; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.replication.asyncreplay/Garnet.test.cluster.replication.asyncreplay.csproj b/test/cluster/Garnet.test.cluster.replication.asyncreplay/Garnet.test.cluster.replication.asyncreplay.csproj new file mode 100644 index 00000000000..e6407b8ac15 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.asyncreplay/Garnet.test.cluster.replication.asyncreplay.csproj @@ -0,0 +1,47 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationAsyncReplay.cs b/test/cluster/Garnet.test.cluster.replication.asyncreplay/ReplicationTests/ClusterReplicationAsyncReplay.cs similarity index 91% rename from test/Garnet.test.cluster/ReplicationTests/ClusterReplicationAsyncReplay.cs rename to test/cluster/Garnet.test.cluster.replication.asyncreplay/ReplicationTests/ClusterReplicationAsyncReplay.cs index cab5017862c..33dbdfd59d7 100644 --- a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationAsyncReplay.cs +++ b/test/cluster/Garnet.test.cluster.replication.asyncreplay/ReplicationTests/ClusterReplicationAsyncReplay.cs @@ -5,8 +5,8 @@ namespace Garnet.test.cluster { + [TestFixture] [NonParallelizable] - [Ignore("Skip to reduce CI duration.")] public class ClusterReplicationAsyncReplay : ClusterReplicationBaseTests { [SetUp] diff --git a/test/cluster/Garnet.test.cluster.replication.asyncreplay/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.replication.asyncreplay/TestProjectSetup.cs new file mode 100644 index 00000000000..1b27f36fc38 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.asyncreplay/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterReplicationAsync; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.replication.disklesssync/Garnet.test.cluster.replication.disklesssync.csproj b/test/cluster/Garnet.test.cluster.replication.disklesssync/Garnet.test.cluster.replication.disklesssync.csproj new file mode 100644 index 00000000000..91955b1d0f2 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.disklesssync/Garnet.test.cluster.replication.disklesssync.csproj @@ -0,0 +1,50 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationDisklessSyncTests.cs b/test/cluster/Garnet.test.cluster.replication.disklesssync/ReplicationTests/ClusterReplicationDisklessSyncTests.cs similarity index 91% rename from test/Garnet.test.cluster/ReplicationTests/ClusterReplicationDisklessSyncTests.cs rename to test/cluster/Garnet.test.cluster.replication.disklesssync/ReplicationTests/ClusterReplicationDisklessSyncTests.cs index f3a2e73b7b6..e4cfd7e374e 100644 --- a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationDisklessSyncTests.cs +++ b/test/cluster/Garnet.test.cluster.replication.disklesssync/ReplicationTests/ClusterReplicationDisklessSyncTests.cs @@ -1,9 +1,8 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; -using Allure.NUnit; #if DEBUG using Garnet.common; #endif @@ -13,31 +12,23 @@ namespace Garnet.test.cluster { - /// - /// TODO: Testing scenarios - /// 1. Empty replica attach sync - /// a. Primary empty - /// b. Primary non-empty - /// 2. Replica same history and version different AOF - /// 3. Replica same history and different version and AOF - /// 4. Replica different history, version and AOF - /// - [AllureNUnit] [TestFixture] [NonParallelizable] - public class ClusterReplicationDisklessSyncTests : AllureTestBase + public class ClusterReplicationDisklessSyncTests : TestBase { ClusterTestContext context; readonly int keyCount = 256; protected bool useTLS = false; protected bool asyncReplay = false; + protected int sublogCount = 1; int timeout = (int)TimeSpan.FromSeconds(15).TotalSeconds; int testTimeout = (int)TimeSpan.FromSeconds(120).TotalSeconds; public Dictionary monitorTests = new(){ - { "ClusterDisklessSyncFailover", LogLevel.Trace } + { "ClusterDisklessSyncFailover", LogLevel.Trace }, + { "ClusterDisklessSyncResetSyncManagerCts", LogLevel.Trace } }; [SetUp] @@ -115,7 +106,7 @@ public void ClusterEmptyReplicaDisklessSync([Values] bool disableObjects, [Value var nodes_count = 2; var primaryIndex = 0; var replicaIndex = 1; - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup primary and introduce it to future replica @@ -157,7 +148,7 @@ public void ClusterAofReplayDisklessSync([Values] bool disableObjects, [Values] var nodes_count = 2; var primaryIndex = 0; var replicaIndex = 1; - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, replicaDisklessSyncFullSyncAofThreshold: forceFullSync ? "1k" : string.Empty); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, replicaDisklessSyncFullSyncAofThreshold: forceFullSync ? "1k" : string.Empty, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup primary and introduce it to future replica @@ -215,7 +206,7 @@ public void ClusterDBVersionAlignmentDisklessSync([Values] bool disableObjects, var primaryIndex = 0; var replicaOneIndex = 1; var replicaTwoIndex = 2; - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup primary and introduce it to future replica @@ -244,12 +235,11 @@ public void ClusterDBVersionAlignmentDisklessSync([Values] bool disableObjects, Validate(primaryIndex, replicaOneIndex, disableObjects); // Validate db version - var primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, isMainStore: true, logger: context.logger); - var replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, isMainStore: true, logger: context.logger); + var primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, logger: context.logger); + var replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, logger: context.logger); - // With unified store, versions increase per scan (main and object) - // so expected versions depend on whether objects are disabled or not - var expectedVersion1 = disableObjects ? 2 : 3; + // Versions increase per scan and we have only a single store which takes a single scan regardless of 'disableObjects' setting + var expectedVersion1 = 2; ClassicAssert.AreEqual(expectedVersion1, primaryVersion); ClassicAssert.AreEqual(primaryVersion, replicaOneVersion); @@ -269,13 +259,12 @@ public void ClusterDBVersionAlignmentDisklessSync([Values] bool disableObjects, Validate(primaryIndex, replicaTwoIndex, disableObjects); // Validate db version - primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, isMainStore: true, logger: context.logger); - replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, isMainStore: true, logger: context.logger); - var replicaTwoVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaTwoIndex, isMainStore: true, logger: context.logger); + primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, logger: context.logger); + replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, logger: context.logger); + var replicaTwoVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaTwoIndex, logger: context.logger); - // With unified store, versions increase per scan (main and object) - // so expected versions depend on whether objects are disabled or not - var expectedVersion2 = disableObjects ? 3 : 5; + // With unified store we have only a single scan so a single increment regardless of 'disableObjects' setting + var expectedVersion2 = 3; ClassicAssert.AreEqual(expectedVersion2, primaryVersion); ClassicAssert.AreEqual(primaryVersion, replicaTwoVersion); ClassicAssert.AreEqual(expectedVersion1, replicaOneVersion); @@ -286,13 +275,12 @@ public void ClusterDBVersionAlignmentDisklessSync([Values] bool disableObjects, // Validate second replica data Validate(primaryIndex, replicaOneIndex, disableObjects); - primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, isMainStore: true, logger: context.logger); - replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, isMainStore: true, logger: context.logger); - replicaTwoVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaTwoIndex, isMainStore: true, logger: context.logger); + primaryVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, logger: context.logger); + replicaOneVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaOneIndex, logger: context.logger); + replicaTwoVersion = context.clusterTestUtils.GetStoreCurrentVersion(replicaTwoIndex, logger: context.logger); - // With unified store, versions increase per scan (main and object) - // so expected versions depend on whether objects are disabled or not - var expectedVersion3 = disableObjects ? 4 : 7; + // With unified store we have only a single scan so a single increment regardless of 'disableObjects' setting + var expectedVersion3 = 4; ClassicAssert.AreEqual(expectedVersion3, primaryVersion); ClassicAssert.AreEqual(primaryVersion, replicaOneVersion); ClassicAssert.AreEqual(primaryVersion, replicaTwoVersion); @@ -307,7 +295,7 @@ public void ClusterDisklessSyncParallelAttach([Values] bool disableObjects, [Val var replicaOneIndex = 1; var replicaTwoIndex = 2; var replicaThreeIndex = 3; - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup primary and introduce it to future replica @@ -350,7 +338,7 @@ public void ClusterDisklessSyncFailover([Values] bool disableObjects, [Values] b int[] nOffsets = [primary, replicaOne, replicaTwo]; - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup primary and introduce it to future replica @@ -410,7 +398,7 @@ public void ClusterDisklessSyncResetSyncManagerCts() var nodes_count = 2; var primaryIndex = 0; var replicaOneIndex = 1; - context.CreateInstances(nodes_count, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout); + context.CreateInstances(nodes_count, enableAOF: true, useTLS: useTLS, enableDisklessSync: true, timeout: timeout, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.AddDelSlotsRange(primaryIndex, [(0, 16383)], addslot: true, logger: context.logger); @@ -433,6 +421,9 @@ public void ClusterDisklessSyncResetSyncManagerCts() var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex: replicaOneIndex, primaryNodeIndex: primaryIndex, logger: context.logger); ClassicAssert.AreEqual("OK", resp); + + // Ensure that replicas have connected before completing the test + context.clusterTestUtils.WaitForReplicasConnected(primaryIndex, 1, logger: context.logger); } #endif } diff --git a/test/cluster/Garnet.test.cluster.replication.disklesssync/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.replication.disklesssync/TestProjectSetup.cs new file mode 100644 index 00000000000..a95b2652608 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.disklesssync/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterReplicationDiskless; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.replication.tls/Garnet.test.cluster.replication.tls.csproj b/test/cluster/Garnet.test.cluster.replication.tls/Garnet.test.cluster.replication.tls.csproj new file mode 100644 index 00000000000..e6407b8ac15 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.tls/Garnet.test.cluster.replication.tls.csproj @@ -0,0 +1,47 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationTLS.cs b/test/cluster/Garnet.test.cluster.replication.tls/ReplicationTests/ClusterReplicationTLS.cs similarity index 96% rename from test/Garnet.test.cluster/ReplicationTests/ClusterReplicationTLS.cs rename to test/cluster/Garnet.test.cluster.replication.tls/ReplicationTests/ClusterReplicationTLS.cs index 76e0b11e459..be9f09ff1fb 100644 --- a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationTLS.cs +++ b/test/cluster/Garnet.test.cluster.replication.tls/ReplicationTests/ClusterReplicationTLS.cs @@ -5,6 +5,7 @@ namespace Garnet.test.cluster { + [TestFixture] [NonParallelizable] public class ClusterReplicationTLS : ClusterReplicationBaseTests { diff --git a/test/cluster/Garnet.test.cluster.replication.tls/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.replication.tls/TestProjectSetup.cs new file mode 100644 index 00000000000..65e1638cf05 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication.tls/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterReplicationTls; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.replication/Garnet.test.cluster.replication.csproj b/test/cluster/Garnet.test.cluster.replication/Garnet.test.cluster.replication.csproj new file mode 100644 index 00000000000..e0ad6575627 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication/Garnet.test.cluster.replication.csproj @@ -0,0 +1,52 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + + + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationBaseTests.cs b/test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterReplicationBaseTests.cs similarity index 82% rename from test/Garnet.test.cluster/ReplicationTests/ClusterReplicationBaseTests.cs rename to test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterReplicationBaseTests.cs index 68b4947050a..4994ed821c8 100644 --- a/test/Garnet.test.cluster/ReplicationTests/ClusterReplicationBaseTests.cs +++ b/test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterReplicationBaseTests.cs @@ -3,27 +3,25 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; using NUnit.Framework; +using NUnit.Framework.Internal; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture] [NonParallelizable] - public class ClusterReplicationBaseTests : AllureTestBase + public class ClusterReplicationBaseTests : TestBase { public (Action, string)[] GetUnitTests() { @@ -38,22 +36,6 @@ public class ClusterReplicationBaseTests : AllureTestBase new(() => AsyncUtils.BlockingWait(ClusterSRPrimaryCheckpointAsync(false, true)), "ClusterSRPrimaryCheckpointAsync(false, true)"), new(() => AsyncUtils.BlockingWait(ClusterSRPrimaryCheckpointAsync(true, false)), "ClusterSRPrimaryCheckpointAsync(true, false)"), new(() => AsyncUtils.BlockingWait(ClusterSRPrimaryCheckpointAsync(true, true)), "ClusterSRPrimaryCheckpointAsync(true, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, false, false, false), "ClusterSRPrimaryCheckpointRetrieve(false, false, false, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, false, false, true), "ClusterSRPrimaryCheckpointRetrieve(false, false, false, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, true, false, false), "ClusterSRPrimaryCheckpointRetrieve(false, true, false, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, true, false, true), "ClusterSRPrimaryCheckpointRetrieve(false, true, false, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, false, false, false), "ClusterSRPrimaryCheckpointRetrieve(true, false, false, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, false, false, true), "ClusterSRPrimaryCheckpointRetrieve(true, false, false, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, true, false, false), "ClusterSRPrimaryCheckpointRetrieve(true, true, false, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, true, false, true), "ClusterSRPrimaryCheckpointRetrieve(true, true, false, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, false, true, false), "ClusterSRPrimaryCheckpointRetrieve(false, false, true, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, false, true, true), "ClusterSRPrimaryCheckpointRetrieve(false, false, true, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, true, true, false), "ClusterSRPrimaryCheckpointRetrieve(false, true, true, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(false, true, true, true), "ClusterSRPrimaryCheckpointRetrieve(false, true, true, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, false, true, false), "ClusterSRPrimaryCheckpointRetrieve(true, false, true, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, false, true, true), "ClusterSRPrimaryCheckpointRetrieve(true, false, true, true)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, true, true, false), "ClusterSRPrimaryCheckpointRetrieve(true, true, true, false)"), - new(() => ClusterSRPrimaryCheckpointRetrieve(true, true, true, true), "ClusterSRPrimaryCheckpointRetrieve(true, true, true, true)"), new(() => ClusterSRAddReplicaAfterPrimaryCheckpoint(false, false, false), "ClusterSRAddReplicaAfterPrimaryCheckpoint(false, false, false)"), new(() => ClusterSRAddReplicaAfterPrimaryCheckpoint(false, false, true), "ClusterSRAddReplicaAfterPrimaryCheckpoint(false, false, true)"), new(() => ClusterSRAddReplicaAfterPrimaryCheckpoint(false, true, false), "ClusterSRAddReplicaAfterPrimaryCheckpoint(false, true, false)"), @@ -67,31 +49,30 @@ public class ClusterReplicationBaseTests : AllureTestBase new(() => ClusterSRPrimaryRestart(true, false), "ClusterSRPrimaryRestart(true, false)"), new(() => ClusterSRPrimaryRestart(true, true), "ClusterSRPrimaryRestart(true, true)"), new(ClusterSRRedirectWrites, "ClusterSRRedirectWrites()"), - new(() => ClusterReplicationCheckpointCleanupTest(false, false, false), "ClusterReplicationCheckpointCleanupTest(false, false, false)"), - new(() => ClusterReplicationCheckpointCleanupTest(false, false, true), "ClusterReplicationCheckpointCleanupTest(false, false, true)"), - new(() => ClusterReplicationCheckpointCleanupTest(false, true, false), "ClusterReplicationCheckpointCleanupTest(false, true, false)"), - new(() => ClusterReplicationCheckpointCleanupTest(false, true, true), "ClusterReplicationCheckpointCleanupTest(false, true, true)"), - new(() => ClusterReplicationCheckpointCleanupTest(true, false, false), "ClusterReplicationCheckpointCleanupTest(true, false, false)"), - new(() => ClusterReplicationCheckpointCleanupTest(true, false, true), "ClusterReplicationCheckpointCleanupTest(true, false, true)"), - new(() => ClusterReplicationCheckpointCleanupTest(true, true, false), "ClusterReplicationCheckpointCleanupTest(true, true, false)"), - new(() => ClusterReplicationCheckpointCleanupTest(true, true, true), "ClusterReplicationCheckpointCleanupTest(true, true, true)") + new(() => ClusterReplicationCheckpointCleanupTest(false, false), "ClusterReplicationCheckpointCleanupTest(false, false)"), + new(() => ClusterReplicationCheckpointCleanupTest(false, true), "ClusterReplicationCheckpointCleanupTest(false, true)"), + new(() => ClusterReplicationCheckpointCleanupTest(true, false), "ClusterReplicationCheckpointCleanupTest(true, false)"), + new(() => ClusterReplicationCheckpointCleanupTest(true, true), "ClusterReplicationCheckpointCleanupTest(true, true)") ]; return testList.ToArray(); } - ClusterTestContext context; + protected ClusterTestContext context; public TextWriter LogTextWriter { get; set; } protected bool useTLS = false; protected bool asyncReplay = false; readonly int timeout = 60; - readonly int keyCount = 256; + protected int keyCount = 256; + protected int sublogCount = 1; public Dictionary monitorTests = new() { {"ClusterReplicationSimpleFailover", LogLevel.Warning}, - {"ClusterReplicationMultiRestartRecover", LogLevel.Trace}, + {"ClusterReplicationMultiRestartRecover", LogLevel.Error}, + {"ClusterFailoverAttachReplicas", LogLevel.Error}, + {"ClusterReplicationSimpleTransactionTest", LogLevel.Trace}, {"ClusterReplicationStoredProc", LogLevel.Error} }; @@ -119,7 +100,7 @@ public void ClusterSRTest([Values] bool disableObjects) var primaryIndex = 0; var replicaIndex = 1; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -153,13 +134,17 @@ public void ClusterSRTest([Values] bool disableObjects) [Category("REPLICATION")] public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Values] bool disableObjects) { + // Disable excessive logging; Leave for future debugging + //if (useTLS) + // context.EnableGarnetLoggingEvents([GarnetTestLoggingEventType.LogPrimaryStreamType, GarnetTestLoggingEventType.LogRunAofSyncTask]); + var replica_count = 1;// Per primary var primary_count = 1; var primaryIndex = 0; var replicaIndex = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount, threadPoolMinIOCompletionThreads: 512); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -184,32 +169,33 @@ public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Val context.SimplePopulateDB(disableObjects, keyLength, kvpairCount, primaryIndex, performRMW: performRMW, addCount: addCount); // Wait for replication offsets to synchronize - context.clusterTestUtils.WaitForReplicaAofSync(0, 1); + context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex, replicaIndex); // Validate database context.SimpleValidateDB(disableObjects, replicaIndex); // Shutdown secondary - context.nodes[1].Dispose(false); - - Thread.Sleep(TimeSpan.FromSeconds(2)); + context.nodes[replicaIndex].Dispose(false); + context.clusterTestUtils.WaitForAofSyncDriverDipose(primaryIndex); // New insert context.SimplePopulateDB(disableObjects, keyLength, kvpairCount, primaryIndex, performRMW: performRMW, addCount: addCount); // Restart secondary - context.nodes[1] = context.CreateInstance( - context.clusterTestUtils.GetEndPoint(1), + context.nodes[replicaIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(replicaIndex), disableObjects: disableObjects, tryRecover: true, enableAOF: true, timeout: timeout, useTLS: useTLS, - cleanClusterConfig: false); - context.nodes[1].Start(); + cleanClusterConfig: false, + sublogCount: sublogCount, + threadPoolMinIOCompletionThreads: 512); + context.nodes[replicaIndex].Start(); context.CreateConnection(useTLS: useTLS); // Validate synchronization was success - context.clusterTestUtils.WaitForReplicaAofSync(0, 1); + context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex, replicaIndex); // Validate database context.SimpleValidateDB(disableObjects, replicaIndex); } @@ -218,22 +204,26 @@ public void ClusterSRNoCheckpointRestartSecondary([Values] bool performRMW, [Val [Category("REPLICATION")] public async Task ClusterSRPrimaryCheckpointAsync([Values] bool performRMW, [Values] bool disableObjects) { + // Disable excessive logging; Leave for future debugging + //if (useTLS) + // context.EnableGarnetLoggingEvents([GarnetTestLoggingEventType.LogPrimaryStreamType, GarnetTestLoggingEventType.LogRunAofSyncTask]); + var replica_count = 1;// Per primary var primary_count = 1; var primaryIndex = 0; var replicaIndex = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount, threadPoolMinIOCompletionThreads: 512); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); - var cconfig = context.clusterTestUtils.ClusterNodes(0, context.logger); + var cconfig = context.clusterTestUtils.ClusterNodes(primaryIndex, context.logger); var myself = cconfig.Nodes.First(); var slotRangesStr = string.Join(",", myself.Slots.Select(x => $"({x.From}-{x.To})").ToList()); ClassicAssert.AreEqual(1, myself.Slots.Count, $"Setup failed slot ranges count greater than 1 {slotRangesStr}"); - shards = context.clusterTestUtils.ClusterShards(0, context.logger); + shards = context.clusterTestUtils.ClusterShards(primaryIndex, context.logger); ClassicAssert.AreEqual(1, shards.Count); ClassicAssert.AreEqual(1, shards[0].slotRanges.Count); ClassicAssert.AreEqual(0, shards[0].slotRanges[0].Item1); @@ -248,44 +238,47 @@ public async Task ClusterSRPrimaryCheckpointAsync([Values] bool performRMW, [Val // Populate Primary context.SimplePopulateDB(disableObjects, keyLength, kvpairCount, primaryIndex, performRMW: performRMW, addCount: addCount); - var primaryLastSaveTime = context.clusterTestUtils.LastSave(0, logger: context.logger); - var replicaLastSaveTime = context.clusterTestUtils.LastSave(1, logger: context.logger); + var primaryLastSaveTime = context.clusterTestUtils.LastSave(primaryIndex, logger: context.logger); + var replicaLastSaveTime = context.clusterTestUtils.LastSave(replicaIndex, logger: context.logger); context.clusterTestUtils.Checkpoint(0, logger: context.logger); // Populate Primary context.SimplePopulateDB(disableObjects, keyLength, kvpairCount, primaryIndex, performRMW: performRMW, addCount: addCount); context.SimpleValidateDB(disableObjects, replicaIndex); - context.clusterTestUtils.WaitForReplicaAofSync(0, 1, context.logger); - context.clusterTestUtils.WaitCheckpoint(0, primaryLastSaveTime, logger: context.logger); - context.clusterTestUtils.WaitCheckpoint(1, replicaLastSaveTime, logger: context.logger); + context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex, replicaIndex, context.logger); + context.clusterTestUtils.WaitCheckpoint(primaryIndex, primaryLastSaveTime, logger: context.logger); + context.clusterTestUtils.WaitCheckpoint(replicaIndex, replicaLastSaveTime, logger: context.logger); // Shutdown secondary - context.nodes[1].Dispose(false); - Thread.Sleep(TimeSpan.FromSeconds(2)); + context.nodes[replicaIndex].Dispose(false); + context.clusterTestUtils.WaitForAofSyncDriverDipose(primaryIndex); // New insert context.SimplePopulateDB(disableObjects, keyLength, kvpairCount, primaryIndex, performRMW: performRMW, addCount: addCount); // Restart secondary - context.nodes[1] = context.CreateInstance( - context.clusterTestUtils.GetEndPoint(1), + context.nodes[replicaIndex] = context.CreateInstance( + context.clusterTestUtils.GetEndPoint(replicaIndex), disableObjects: disableObjects, tryRecover: true, enableAOF: true, timeout: timeout, useTLS: useTLS, cleanClusterConfig: false, - asyncReplay: asyncReplay); - context.nodes[1].Start(); + asyncReplay: asyncReplay, + sublogCount: sublogCount, + threadPoolMinIOCompletionThreads: 512); + context.nodes[replicaIndex].Start(); context.CreateConnection(useTLS: useTLS); - for (int i = 1; i < replica_count; i++) context.clusterTestUtils.WaitForReplicaRecovery(i, context.logger); + for (var i = 1; i < replica_count; i++) + context.clusterTestUtils.WaitForReplicaRecovery(i, context.logger); await context.clusterTestUtils.WaitForConnectedReplicaCountAsync(0, replica_count, context.logger).ConfigureAwait(false); // Validate synchronization was success - context.clusterTestUtils.WaitForReplicaAofSync(0, 1, context.logger); - context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, 1); + context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex, replicaIndex, context.logger); + context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, replicaIndex); context.SimpleValidateDB(disableObjects, replicaIndex); } @@ -293,25 +286,17 @@ public async Task ClusterSRPrimaryCheckpointAsync([Values] bool performRMW, [Val [Category("REPLICATION")] public void ClusterCheckpointRetrieveDisableStorageTier([Values] bool performRMW, [Values] bool disableObjects) { - ClusterSRPrimaryCheckpointRetrieve(performRMW, disableObjects, false, false, true, false); - } - - [Test, Order(5)] - [Category("REPLICATION")] - public void ClusterCheckpointRetrieveDelta([Values] bool performRMW) - { - ClusterSRPrimaryCheckpointRetrieve(performRMW, true, false, false, false, true); + ClusterSRPrimaryCheckpointRetrieve(performRMW, disableObjects, false, true); } [Test, Order(6)] [Category("REPLICATION")] - public void ClusterSRPrimaryCheckpointRetrieve([Values] bool performRMW, [Values] bool disableObjects, [Values] bool lowMemory, [Values] bool manySegments) - => ClusterSRPrimaryCheckpointRetrieve(performRMW: performRMW, disableObjects: disableObjects, lowMemory: lowMemory, manySegments: manySegments, false, false); + public void ClusterSRPrimaryCheckpointRetrieve([Values] bool performRMW, [Values] bool disableObjects, [Values] bool manySegments) + => ClusterSRPrimaryCheckpointRetrieve(performRMW: performRMW, disableObjects: disableObjects, manySegments: manySegments, false); - void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bool lowMemory, bool manySegments, bool disableStorageTier, bool incrementalSnapshots) + void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bool manySegments, bool disableStorageTier) { - // Test many segments on or off with lowMemory - manySegments = lowMemory && manySegments; + var lowMemory = manySegments; var primaryIndex = 0; var replicaIndex = 1; @@ -319,7 +304,16 @@ void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bo var primary_count = 1; var nodes_count = primary_count + primary_count * replica_count; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: disableObjects, lowMemory: lowMemory, segmentSize: manySegments ? "4k" : "1g", DisableStorageTier: disableStorageTier, EnableIncrementalSnapshots: incrementalSnapshots, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances( + nodes_count, + disableObjects: disableObjects, + lowMemory: lowMemory, + segmentSize: manySegments ? "4k" : "1g", + DisableStorageTier: disableStorageTier, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -342,15 +336,15 @@ void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bo context.logger?.LogTrace("Test disposing node 1"); context.nodes[1].Dispose(false); - Thread.Sleep(TimeSpan.FromSeconds(1)); + context.clusterTestUtils.WaitForAofSyncDriverDipose(primaryIndex); // Populate Primary if (disableObjects) { if (!performRMW) - context.PopulatePrimary(ref context.kvPairs, keyLength, kvpairCount, primaryIndex, null, incrementalSnapshots, primaryIndex); + context.PopulatePrimary(ref context.kvPairs, keyLength, kvpairCount, primaryIndex); else - context.PopulatePrimaryRMW(ref context.kvPairs, keyLength, kvpairCount, primaryIndex, addCount, null, incrementalSnapshots, primaryIndex); + context.PopulatePrimaryRMW(ref context.kvPairs, keyLength, kvpairCount, primaryIndex, addCount); } else { @@ -376,7 +370,8 @@ void ClusterSRPrimaryCheckpointRetrieve(bool performRMW, bool disableObjects, bo lowMemory: lowMemory, SegmentSize: manySegments ? "4k" : "1g", DisableStorageTier: disableStorageTier, - asyncReplay: asyncReplay); + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.nodes[replicaIndex].Start(); context.CreateConnection(useTLS: useTLS); @@ -395,7 +390,7 @@ public void ClusterSRAddReplicaAfterPrimaryCheckpoint([Values] bool performRMW, var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, tryRecover: true, disableObjects: disableObjects, lowMemory: lowMemory, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, tryRecover: true, disableObjects: disableObjects, lowMemory: lowMemory, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddDelSlotsRange(0, new List<(int, int)>() { (0, 16383) }, true, context.logger)); @@ -454,7 +449,7 @@ public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disa var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, tryRecover: true, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, tryRecover: true, disableObjects: disableObjects, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddDelSlotsRange(0, new List<(int, int)>() { (0, 16383) }, true, context.logger)); @@ -483,12 +478,7 @@ public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disa context.clusterTestUtils.Checkpoint(0, logger: context.logger); var storeCurrentAofAddress = context.clusterTestUtils.GetStoreCurrentAofAddress(0, logger: context.logger); - long objectStoreCurrentAofAddress = -1; - if (!disableObjects) - objectStoreCurrentAofAddress = context.clusterTestUtils.GetObjectStoreCurrentAofAddress(0, context.logger); - context.nodes[0].Dispose(false); - Thread.Sleep(TimeSpan.FromSeconds(1)); // Restart Primary context.nodes[0] = context.CreateInstance( @@ -499,18 +489,13 @@ public void ClusterSRPrimaryRestart([Values] bool performRMW, [Values] bool disa timeout: timeout, useTLS: useTLS, cleanClusterConfig: false, - asyncReplay: asyncReplay); + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.nodes[0].Start(); context.CreateConnection(useTLS: useTLS); var storeRecoveredAofAddress = context.clusterTestUtils.GetStoreRecoveredAofAddress(0, context.logger); - long objectStoreRecoveredAofAddress = -1; - if (!disableObjects) - objectStoreRecoveredAofAddress = context.clusterTestUtils.GetObjectStoreRecoveredAofAddress(0, logger: context.logger); - ClassicAssert.AreEqual(storeCurrentAofAddress, storeRecoveredAofAddress); - if (!disableObjects) - ClassicAssert.AreEqual(objectStoreCurrentAofAddress, objectStoreRecoveredAofAddress); } [Test, Order(9)] @@ -521,11 +506,10 @@ public void ClusterSRRedirectWrites() var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); - var cconfig = context.clusterTestUtils.ClusterNodes(0, context.logger); var myself = cconfig.Nodes.First(); var slotRangesStr = string.Join(",", myself.Slots.Select(x => $"({x.From}-{x.To})").ToList()); @@ -546,7 +530,7 @@ public void ClusterSRRedirectWrites() public void ClusterSRReplicaOfTest([Values] bool performRMW) { var nodes_count = 2; - context.CreateInstances(nodes_count, tryRecover: true, disableObjects: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, tryRecover: true, disableObjects: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddDelSlotsRange(0, [(0, 16383)], true, context.logger)); @@ -573,7 +557,8 @@ public void ClusterSRReplicaOfTest([Values] bool performRMW) else context.PopulatePrimaryRMW(ref context.kvPairs, keyLength, kvpairCount, 0, addCount); - context.clusterTestUtils.ReplicaOf(replicaNodeIndex: 1, primaryNodeIndex: 0, logger: context.logger); + var resp = context.clusterTestUtils.ReplicaOf(replicaNodeIndex: 1, primaryNodeIndex: 0, logger: context.logger); + ClassicAssert.AreEqual("OK", resp); context.clusterTestUtils.WaitForReplicaAofSync(0, 1); } @@ -585,7 +570,7 @@ public void ClusterReplicationSimpleFailover([Values] bool performRMW, [Values] var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -648,17 +633,19 @@ public void ClusterReplicationSimpleFailover([Values] bool performRMW, [Values] context.PopulatePrimary(ref context.kvPairs, keyLength, kvpairCount, replicaIndex, slotMap: slotMap); else context.PopulatePrimaryRMW(ref context.kvPairs, keyLength, kvpairCount, replicaIndex, addCount, slotMap: slotMap); + + context.clusterTestUtils.WaitForReplicaAofSync(replicaIndex, primaryIndex, context.logger); } [Test, Order(12)] [Category("REPLICATION")] - public void ClusterFailoverAttachReplicas([Values] bool performRMW, [Values] bool takePrimaryCheckpoint, [Values] bool takeNewPrimaryCheckpoint, [Values] bool enableIncrementalSnapshots) + public void ClusterFailoverAttachReplicas([Values] bool performRMW, [Values] bool takePrimaryCheckpoint, [Values] bool takeNewPrimaryCheckpoint) { var replica_count = 2; // Per primary var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: true, EnableIncrementalSnapshots: enableIncrementalSnapshots, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -733,19 +720,25 @@ public void ClusterFailoverAttachReplicas([Values] bool performRMW, [Values] boo [Test, Order(13)] [Category("REPLICATION")] - //[Repeat(20)] - public void ClusterReplicationCheckpointCleanupTest([Values] bool performRMW, [Values] bool disableObjects, [Values] bool enableIncrementalSnapshots) + public void ClusterReplicationCheckpointCleanupTest([Values] bool performRMW, [Values] bool disableObjects) { - if (TestContext.CurrentContext.CurrentRepeatCount > 0) - Debug.WriteLine($"*** Current test iteration: {TestContext.CurrentContext.CurrentRepeatCount + 1}, name = {TestContext.CurrentContext.Test.Name} ***"); - var primaryIndex = 0; var replicaIndex = 1; var replica_count = 1;//Per primary var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, tryRecover: true, disableObjects: disableObjects, lowMemory: true, segmentSize: "4k", EnableIncrementalSnapshots: enableIncrementalSnapshots, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, deviceType: Tsavorite.core.DeviceType.Native); + + context.CreateInstances(nodes_count, + tryRecover: true, + disableObjects: disableObjects, + lowMemory: true, + segmentSize: "4k", + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + deviceType: Tsavorite.core.DeviceType.Native, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); // Setup cluster @@ -756,7 +749,7 @@ public void ClusterReplicationCheckpointCleanupTest([Values] bool performRMW, [V var slotRangesStr = string.Join(",", myself.Slots.Select(x => $"({x.From}-{x.To})").ToList()); ClassicAssert.AreEqual(1, myself.Slots.Count, $"Setup failed slot ranges count greater than 1 {slotRangesStr}"); - var shards = context.clusterTestUtils.ClusterShards(0, context.logger); + var shards = context.clusterTestUtils.ClusterShards(primaryIndex, context.logger); ClassicAssert.AreEqual(2, shards.Count); ClassicAssert.AreEqual(1, shards[0].slotRanges.Count); ClassicAssert.AreEqual(0, shards[0].slotRanges[0].Item1); @@ -765,13 +758,11 @@ public void ClusterReplicationCheckpointCleanupTest([Values] bool performRMW, [V context.kvPairs = []; context.kvPairsObj = []; context.checkpointTask = Task.Run(() => context.PopulatePrimaryAndTakeCheckpointTask(performRMW, disableObjects, takeCheckpoint: true)); - var attachReplicaTask = Task.Run(() => context.AttachAndWaitForSyncAsync(primary_count, replica_count, disableObjects)); - - if (!context.checkpointTask.Wait(TimeSpan.FromSeconds(60))) - Assert.Fail("checkpointTask timeout"); + var attachReplicaTask = Task.Run(() => context.AttachAndWaitForSyncAsync(primaryIndex, primary_count, replica_count, disableObjects)); - if (!attachReplicaTask.Wait(TimeSpan.FromSeconds(60))) - Assert.Fail("attachReplicaTask timeout"); + var tasks = new Task[] { context.checkpointTask, attachReplicaTask }; + if (!Task.WhenAll(tasks).Wait(TimeSpan.FromSeconds(60))) + Assert.Fail($"Task timeout - checkpointTask: {context.checkpointTask.Status}, attachReplicaTask: {attachReplicaTask.Status}"); context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex: primaryIndex, secondaryIndex: replicaIndex, logger: context.logger); } @@ -784,7 +775,7 @@ public void ClusterMainMemoryReplicationAttachReplicas() var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: true, FastAofTruncate: true, OnDemandCheckpoint: true, CommitFrequencyMs: -1, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: true, FastAofTruncate: true, OnDemandCheckpoint: true, CommitFrequencyMs: -1, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); ClassicAssert.AreEqual("OK", context.clusterTestUtils.AddDelSlotsRange(0, new List<(int, int)>() { (0, 16383) }, true)); @@ -823,7 +814,7 @@ public void ClusterMainMemoryReplicationAttachReplicas() [Test, Order(16)] [Category("REPLICATION")] public void ClusterDivergentReplicasTest([Values] bool performRMW, [Values] bool disableObjects, [Values] bool ckptBeforeDivergence) - => ClusterDivergentReplicasTest(performRMW, disableObjects, ckptBeforeDivergence, false, false, fastCommit: false); + => ClusterDivergentReplicasTest(performRMW, disableObjects, ckptBeforeDivergence, false, false); [Test, Order(17)] [Category("REPLICATION")] @@ -833,8 +824,7 @@ public void ClusterDivergentCheckpointTest([Values] bool performRMW, [Values] bo disableObjects, ckptBeforeDivergence: true, multiCheckpointAfterDivergence: true, - mainMemoryReplication: false, - fastCommit: false); + mainMemoryReplication: false); [Test, Order(18)] [Category("REPLICATION")] @@ -844,8 +834,7 @@ public void ClusterDivergentReplicasMMTest([Values] bool performRMW, [Values] bo disableObjects, ckptBeforeDivergence, multiCheckpointAfterDivergence: false, - mainMemoryReplication: true, - fastCommit: false); + mainMemoryReplication: true); [Test, Order(19)] [Category("REPLICATION")] @@ -855,8 +844,7 @@ public void ClusterDivergentCheckpointMMTest([Values] bool performRMW, [Values] disableObjects, ckptBeforeDivergence: true, multiCheckpointAfterDivergence: true, - mainMemoryReplication: true, - fastCommit: false); + mainMemoryReplication: true); [Test, Order(20)] [Category("REPLICATION")] @@ -866,17 +854,25 @@ public void ClusterDivergentCheckpointMMFastCommitTest([Values] bool disableObje disableObjects: disableObjects, ckptBeforeDivergence: true, multiCheckpointAfterDivergence: true, - mainMemoryReplication: mainMemoryReplication, - fastCommit: true); + mainMemoryReplication: mainMemoryReplication); - void ClusterDivergentReplicasTest(bool performRMW, bool disableObjects, bool ckptBeforeDivergence, bool multiCheckpointAfterDivergence, bool mainMemoryReplication, bool fastCommit) + void ClusterDivergentReplicasTest(bool performRMW, bool disableObjects, bool ckptBeforeDivergence, bool multiCheckpointAfterDivergence, bool mainMemoryReplication) { var set = false; var replica_count = 2;// Per primary var primary_count = 1; var nodes_count = primary_count + (primary_count * replica_count); ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: disableObjects, FastAofTruncate: mainMemoryReplication, CommitFrequencyMs: mainMemoryReplication ? -1 : 0, OnDemandCheckpoint: mainMemoryReplication, FastCommit: fastCommit, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances( + nodes_count, + disableObjects: disableObjects, + FastAofTruncate: mainMemoryReplication, + CommitFrequencyMs: mainMemoryReplication ? -1 : 0, + OnDemandCheckpoint: mainMemoryReplication, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1005,7 +1001,16 @@ public async Task ClusterReplicationCheckpointAlignmentTestAsync([Values] bool p var primaryNodeIndex = 0; var replicaNodeIndex = 1; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, FastAofTruncate: true, CommitFrequencyMs: -1, OnDemandCheckpoint: true, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances( + nodes_count, + disableObjects: false, + FastAofTruncate: true, + CommitFrequencyMs: -1, + OnDemandCheckpoint: true, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1021,6 +1026,11 @@ public async Task ClusterReplicationCheckpointAlignmentTestAsync([Values] bool p context.PopulatePrimaryRMW(ref context.kvPairs, keyLength, kvpairCount, primaryNodeIndex, addCount); context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); + var primaryVersion = context.clusterTestUtils.GetInfo(primaryNodeIndex, "store", "CurrentVersion", logger: context.logger); + var replicaVersion = context.clusterTestUtils.GetInfo(replicaNodeIndex, "store", "CurrentVersion", logger: context.logger); + ClassicAssert.AreEqual("1", primaryVersion); + ClassicAssert.AreEqual(primaryVersion, replicaVersion); + for (var i = 0; i < 5; i++) { var primaryLastSaveTime = context.clusterTestUtils.LastSave(primaryNodeIndex, logger: context.logger); @@ -1029,12 +1039,12 @@ public async Task ClusterReplicationCheckpointAlignmentTestAsync([Values] bool p context.clusterTestUtils.WaitCheckpoint(primaryNodeIndex, primaryLastSaveTime, logger: context.logger); context.clusterTestUtils.WaitCheckpoint(replicaNodeIndex, replicaLastSaveTime, logger: context.logger); context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); - } - var primaryVersion = context.clusterTestUtils.GetInfo(primaryNodeIndex, "store", "CurrentVersion", logger: context.logger); - var replicaVersion = context.clusterTestUtils.GetInfo(replicaNodeIndex, "store", "CurrentVersion", logger: context.logger); - ClassicAssert.AreEqual("6", primaryVersion); - ClassicAssert.AreEqual(primaryVersion, replicaVersion); + primaryVersion = context.clusterTestUtils.GetInfo(primaryNodeIndex, "store", "CurrentVersion", logger: context.logger); + replicaVersion = context.clusterTestUtils.GetInfo(replicaNodeIndex, "store", "CurrentVersion", logger: context.logger); + ClassicAssert.AreEqual($"{i + 2}", primaryVersion); // +2 because version started at 1 and our loop index is zero-based + ClassicAssert.AreEqual(primaryVersion, replicaVersion); + } context.ValidateKVCollectionAgainstReplica(ref context.kvPairs, replicaNodeIndex); @@ -1055,7 +1065,8 @@ public async Task ClusterReplicationCheckpointAlignmentTestAsync([Values] bool p timeout: timeout, useTLS: useTLS, cleanClusterConfig: true, - asyncReplay: asyncReplay); + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.nodes[primaryNodeIndex].Start(); // Restart secondary and recover @@ -1070,7 +1081,8 @@ public async Task ClusterReplicationCheckpointAlignmentTestAsync([Values] bool p timeout: timeout, useTLS: useTLS, cleanClusterConfig: true, - asyncReplay: asyncReplay); + asyncReplay: asyncReplay, + sublogCount: sublogCount); context.nodes[replicaNodeIndex].Start(); context.CreateConnection(useTLS: useTLS); @@ -1116,7 +1128,15 @@ public void ClusterReplicationLua([Values] bool luaTransactionMode) var replicaNodeIndex = 1; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, enableLua: true, luaTransactionMode: luaTransactionMode); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + enableLua: true, + luaTransactionMode: luaTransactionMode, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1136,6 +1156,7 @@ public void ClusterReplicationLua([Values] bool luaTransactionMode) [Test, Order(24)] [Category("REPLICATION")] + [CancelAfter(30_000)] public void ClusterReplicationStoredProc([Values] bool enableDisklessSync, [Values] bool attachFirst, [Values] bool objectStore) { var replica_count = 1;// Per primary @@ -1147,7 +1168,14 @@ public void ClusterReplicationStoredProc([Values] bool enableDisklessSync, [Valu var expectedKeys = new[] { "X", "Y" }; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, enableDisklessSync: enableDisklessSync); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + enableDisklessSync: enableDisklessSync, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); @@ -1191,11 +1219,20 @@ public void ClusterReplicationStoredProc([Values] bool enableDisklessSync, [Valu // Validate primary keys var resp = primaryServer.Execute("KEYS", ["*"]); ClassicAssert.AreEqual(expectedKeys, (string[])resp); - context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); replicaServer = context.clusterTestUtils.GetServer(replicaNodeIndex); - resp = replicaServer.Execute("KEYS", ["*"]); - ClassicAssert.AreEqual(expectedKeys, (string[])resp); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + while (true) + { + resp = replicaServer.Execute("KEYS", ["*"]); + if (expectedKeys.Length == ((string[])resp).Length) + break; + ClusterTestUtils.BackOff(cts.Token); + } + + var actual = (string[])resp; + Array.Sort(actual); + ClassicAssert.AreEqual(expectedKeys, actual); void ExecuteRateLimit() { @@ -1214,20 +1251,27 @@ void ClusterReplicate() } } - [Test, Order(24)] + [Test, Order(25)] [Category("REPLICATION")] public async Task ClusterReplicationManualCheckpointingAsync() { // Use case here is, outside of the cluster, period COMMITAOFs are requested. // Done so in recovery scenarios, if a primary does NOT come back there's still confidence // a replica has recent data. - var replica_count = 1;// Per primary var primary_count = 1; var nodes_count = primary_count + primary_count * replica_count; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: true, tryRecover: false, FastAofTruncate: true, CommitFrequencyMs: -1); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: true, + tryRecover: false, + FastAofTruncate: true, + CommitFrequencyMs: -1, + sublogCount: sublogCount); context.CreateConnection(useTLS: true); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1279,7 +1323,7 @@ public async Task ClusterReplicationManualCheckpointingAsync() } } - [Test, Order(25)] + [Test, Order(26)] [Category("CLUSTER")] [CancelAfter(30_000)] [TestCase(ExceptionInjectionType.Divergent_AOF_Stream)] @@ -1300,7 +1344,16 @@ public async Task ReplicaSyncTaskFaultsRecoverAsync(ExceptionInjectionType fault var nodes_count = primary_count + primary_count * replica_count; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: true, tryRecover: false, FastAofTruncate: true, CommitFrequencyMs: -1, clusterReplicationReestablishmentTimeout: 1); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: true, + tryRecover: false, + FastAofTruncate: true, + CommitFrequencyMs: -1, + clusterReplicationReestablishmentTimeout: 1, + sublogCount: sublogCount); context.CreateConnection(useTLS: true); var (shards, _) = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1366,9 +1419,10 @@ public async Task ReplicaSyncTaskFaultsRecoverAsync(ExceptionInjectionType fault } } - [Test, Order(26)] + [Test, Order(27)] [Category("REPLICATION")] - public async Task ClusterReplicationMultiRestartRecover() + [CancelAfter(60_000)] + public async Task ClusterReplicationMultiRestartRecover(CancellationToken cancellationToken) { var replica_count = 1;// Per primary var primary_count = 1; @@ -1378,30 +1432,33 @@ public async Task ClusterReplicationMultiRestartRecover() ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, cleanClusterConfig: false); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + cleanClusterConfig: false, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); var replicaServer = context.clusterTestUtils.GetServer(replicaNodeIndex); - var keyCount = 1000; + var keyCount = 256; var taskCount = 4; var tasks = new List(); for (var i = 0; i < taskCount; i++) - tasks.Add(Task.Run(() => RunWorkload(i * keyCount, (i + 1) * keyCount))); - var restartRecover = 10; - tasks.Add(Task.Run(() => RestartRecover(restartRecover))); + tasks.Add(Task.Run(() => RunWorkload(i * keyCount, (i + 1) * keyCount), cancellationToken)); + var restartRecover = 4; + tasks.Add(Task.Run(async () => await RestartRecover(restartRecover), cancellationToken)); await Task.WhenAll(tasks).ConfigureAwait(false); - context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); + context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger, cancellationToken); // Validate that replica has the same keys as primary - var resp = primaryServer.Execute("KEYS", ["*"]); - var resp2 = replicaServer.Execute("KEYS", ["*"]); - ClassicAssert.AreEqual(resp.Length, resp2.Length); - for (var i = 0; i < resp.Length; i++) - ClassicAssert.AreEqual((string)resp[i], (string)resp2[i]); + ValidateKeys(); // Run write workload at primary void RunWorkload(int start, int count) @@ -1419,11 +1476,27 @@ void RunWorkload(int start, int count) } // Restart and recover replica multiple times - void RestartRecover(int iteration) + async Task RestartRecover(int iteration) { while (iteration-- > 0) { context.nodes[replicaNodeIndex].Dispose(false); + + var items = context.clusterTestUtils.GetReplicationInfo( + primaryNodeIndex, + [ReplicationInfoItem.CONNECTED_REPLICAS, ReplicationInfoItem.SYNC_DRIVER_COUNT], + context.logger); + while (!items[0].Item2.Equals("0") || !items[1].Item2.Equals("0")) + { + items = context.clusterTestUtils.GetReplicationInfo( + primaryNodeIndex, + [ReplicationInfoItem.CONNECTED_REPLICAS, ReplicationInfoItem.SYNC_DRIVER_COUNT], + context.logger); + if (cancellationToken.IsCancellationRequested) + Assert.Fail($"Failed waiting for primary aof sync cleanup ({iteration}: {items[0]};{items[1]})!"); + await Task.Yield(); + } + context.nodes[replicaNodeIndex] = context.CreateInstance( context.clusterTestUtils.GetEndPoint(replicaNodeIndex), disableObjects: false, @@ -1431,13 +1504,28 @@ void RestartRecover(int iteration) useTLS: useTLS, asyncReplay: asyncReplay, tryRecover: true, - cleanClusterConfig: false); + cleanClusterConfig: false, + sublogCount: sublogCount); context.nodes[replicaNodeIndex].Start(); + context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, cancellation: cancellationToken, logger: context.logger); + + await Task.Yield(); } } + + void ValidateKeys() + { + var resp = (string[])primaryServer.Execute("KEYS", ["*"]); + var resp2 = (string[])replicaServer.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(resp.Length, resp2.Length); + Array.Sort(resp); + Array.Sort(resp2); + for (var i = 0; i < resp.Length; i++) + ClassicAssert.AreEqual(resp[i], resp2[i]); + } } - [Test, Order(27)] + [Test, Order(28)] [Category("CLUSTER")] [Category("REPLICATION")] public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() @@ -1449,7 +1537,14 @@ public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() var replicaNodeIndex = 1; ClassicAssert.IsTrue(primary_count > 0); - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, cleanClusterConfig: false); + context.CreateInstances( + nodes_count, + disableObjects: false, + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + cleanClusterConfig: false, + sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); _ = context.clusterTestUtils.SimpleSetupCluster(primary_count, replica_count, logger: context.logger); @@ -1460,12 +1555,9 @@ public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() RunWorkload(primaryServer, offset, keyCount); context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); + // Validate that replica has the same keys as primary - var resp = primaryServer.Execute("KEYS", ["*"]); - var resp2 = replicaServer.Execute("KEYS", ["*"]); - ClassicAssert.AreEqual(resp.Length, resp2.Length); - for (var i = 0; i < resp.Length; i++) - ClassicAssert.AreEqual((string)resp[i], (string)resp2[i]); + ValidateKeys(); // Kill primary context.nodes[primaryNodeIndex].Dispose(false); @@ -1480,7 +1572,8 @@ public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() useTLS: useTLS, asyncReplay: asyncReplay, tryRecover: false, - cleanClusterConfig: false); + cleanClusterConfig: false, + sublogCount: sublogCount); context.nodes[primaryNodeIndex].Start(); await context.clusterTestUtils.ReconnectAsync([primaryNodeIndex]).ConfigureAwait(false); primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); @@ -1497,7 +1590,8 @@ public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() useTLS: useTLS, asyncReplay: asyncReplay, tryRecover: true, - cleanClusterConfig: false); + cleanClusterConfig: false, + sublogCount: sublogCount); context.nodes[replicaNodeIndex].Start(); await context.clusterTestUtils.ReconnectAsync([primaryNodeIndex, replicaNodeIndex]).ConfigureAwait(false); primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); @@ -1506,11 +1600,7 @@ public async Task ClusterReplicationDivergentHistoryWithoutCheckpointAsync() context.clusterTestUtils.WaitForReplicaAofSync(primaryNodeIndex, replicaNodeIndex, context.logger); // Validate that replica has the same keys as primary - resp = primaryServer.Execute("KEYS", ["*"]); - resp2 = replicaServer.Execute("KEYS", ["*"]); - ClassicAssert.AreEqual(resp.Length, resp2.Length); - for (var i = 0; i < resp.Length; i++) - ClassicAssert.AreEqual((string)resp[i], (string)resp2[i]); + ValidateKeys(); // Run write workload at primary void RunWorkload(IServer server, int start, int count) @@ -1525,9 +1615,20 @@ void RunWorkload(IServer server, int start, int count) ClassicAssert.AreEqual(key, (string)resp); } } + + void ValidateKeys() + { + var resp = (string[])primaryServer.Execute("KEYS", ["*"]); + var resp2 = (string[])replicaServer.Execute("KEYS", ["*"]); + ClassicAssert.AreEqual(resp.Length, resp2.Length); + Array.Sort(resp); + Array.Sort(resp2); + for (var i = 0; i < resp.Length; i++) + ClassicAssert.AreEqual(resp[i], resp2[i]); + } } - [Test, Order(28)] + [Test, Order(31)] [Category("REPLICATION")] public void ClusterReplicationSimpleTransactionTest([Values] bool storedProcedure) { @@ -1537,7 +1638,7 @@ public void ClusterReplicationSimpleTransactionTest([Values] bool storedProcedur var primaryNodeIndex = 0; var replicaNodeIndex = 1; - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay); + context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, useTLS: useTLS, asyncReplay: asyncReplay, sublogCount: sublogCount); context.CreateConnection(useTLS: useTLS); var primaryServer = context.clusterTestUtils.GetServer(primaryNodeIndex); @@ -1546,15 +1647,17 @@ public void ClusterReplicationSimpleTransactionTest([Values] bool storedProcedur // Register custom procedure if (storedProcedure) { - _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc("BULKINCRBY", () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); - _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc("BULKINCRBY", () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); + _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc(BulkIncrementBy.Name, () => new BulkIncrementBy(), BulkIncrementBy.CommandInfo); - _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc("BULKREAD", () => new BulkRead(), BulkRead.CommandInfo); - _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc("BULKREAD", () => new BulkRead(), BulkRead.CommandInfo); + _ = context.nodes[primaryNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); + _ = context.nodes[replicaNodeIndex].Register.NewTransactionProc(BulkRead.Name, () => new BulkRead(), BulkRead.CommandInfo); } // Setup cluster - context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + var resp = context.clusterTestUtils.AddDelSlotsRange(primaryNodeIndex, [(0, 16383)], addslot: true, logger: context.logger); + ClassicAssert.AreEqual("OK", resp); + context.clusterTestUtils.SetConfigEpoch(primaryNodeIndex, primaryNodeIndex + 1, logger: context.logger); context.clusterTestUtils.SetConfigEpoch(replicaNodeIndex, replicaNodeIndex + 1, logger: context.logger); context.clusterTestUtils.Meet(primaryNodeIndex, replicaNodeIndex, logger: context.logger); @@ -1562,7 +1665,7 @@ public void ClusterReplicationSimpleTransactionTest([Values] bool storedProcedur context.clusterTestUtils.WaitUntilNodeIsKnown(replicaNodeIndex, primaryNodeIndex, logger: context.logger); // Attach replica - var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryNodeIndex, logger: context.logger); + resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex, primaryNodeIndex, logger: context.logger); ClassicAssert.AreEqual("OK", resp); string[] keys = ["{_}a", "{_}b", "{_}c"]; @@ -1600,10 +1703,62 @@ public void ClusterReplicationSimpleTransactionTest([Values] bool storedProcedur result = ClusterTestContext.ExecuteBulkReadStoredProc(replicaServer, keys); else result = context.ExecuteTxnBulkRead(replicaServer, keys); + ClassicAssert.AreEqual(values, result); var primaryPInfo = context.clusterTestUtils.GetPersistenceInfo(primaryNodeIndex, context.logger); var replicaPInfo = context.clusterTestUtils.GetPersistenceInfo(replicaNodeIndex, context.logger); ClassicAssert.AreEqual(primaryPInfo.TailAddress, replicaPInfo.TailAddress); } + + [Test, Order(32)] + [Category("REPLICATION")] + public void ClusterReplicationHlogSegmentCleanupTest([Values] bool performRMW, [Values] bool disableObjects) + { + var primaryIndex = 0; + var replicaIndex = 1; + var replica_count = 1; + var primary_count = 1; + var nodes_count = primary_count + (primary_count * replica_count); + ClassicAssert.IsTrue(primary_count > 0); + + context.CreateInstances(nodes_count, + tryRecover: true, + disableObjects: disableObjects, + lowMemory: true, + segmentSize: "4k", + enableAOF: true, + useTLS: useTLS, + asyncReplay: asyncReplay, + deviceType: Tsavorite.core.DeviceType.Native, + sublogCount: sublogCount); + context.CreateConnection(useTLS: useTLS); + + context.SimplePrimaryReplicaSetup(); + + var cconfig = context.clusterTestUtils.ClusterNodes(primaryIndex, context.logger); + var myself = cconfig.Nodes.First(); + var slotRangesStr = string.Join(",", myself.Slots.Select(x => $"({x.From}-{x.To})").ToList()); + ClassicAssert.AreEqual(1, myself.Slots.Count, $"Setup failed slot ranges count greater than 1 {slotRangesStr}"); + + var shards = context.clusterTestUtils.ClusterShards(primaryIndex, context.logger); + ClassicAssert.AreEqual(2, shards.Count); + ClassicAssert.AreEqual(1, shards[0].slotRanges.Count); + ClassicAssert.AreEqual(0, shards[0].slotRanges[0].Item1); + ClassicAssert.AreEqual(16383, shards[0].slotRanges[0].Item2); + + context.kvPairs = []; + context.kvPairsObj = []; + + // Populate with more iterations to accumulate multiple hlog segments and trigger segment cleanup + context.checkpointTask = Task.Run(() => context.PopulatePrimaryAndTakeCheckpointTask(performRMW, disableObjects, takeCheckpoint: true, iter: 10)); + var attachReplicaTask = Task.Run(() => context.AttachAndWaitForSyncAsync(primaryIndex, primary_count, replica_count, disableObjects)); + + var tasks = new Task[] { context.checkpointTask, attachReplicaTask }; + if (!Task.WhenAll(tasks).Wait(TimeSpan.FromSeconds(timeout))) + Assert.Fail($"Task timeout - checkpointTask: {context.checkpointTask.Status}, attachReplicaTask: {attachReplicaTask.Status}"); + + context.clusterTestUtils.WaitForReplicaAofSync(primaryIndex: primaryIndex, secondaryIndex: replicaIndex, logger: context.logger); + context.SimpleValidateDB(disableObjects, replicaIndex); + } } } \ No newline at end of file diff --git a/test/Garnet.test.cluster/ReplicationTests/ClusterResetDuringReplicationTests.cs b/test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterResetDuringReplicationTests.cs similarity index 82% rename from test/Garnet.test.cluster/ReplicationTests/ClusterResetDuringReplicationTests.cs rename to test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterResetDuringReplicationTests.cs index f4bb636e3a3..abedea31d47 100644 --- a/test/Garnet.test.cluster/ReplicationTests/ClusterResetDuringReplicationTests.cs +++ b/test/cluster/Garnet.test.cluster.replication/ReplicationTests/ClusterResetDuringReplicationTests.cs @@ -5,22 +5,20 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; -namespace Garnet.test.cluster.ReplicationTests +namespace Garnet.test.cluster { /// /// These tests simulate scenarios where a replica gets stuck or is in replication attach and verify that /// CLUSTER RESET HARD can properly cancel ongoing operations and allow the replica to be reused. /// - [AllureNUnit] [TestFixture] [NonParallelizable] - public class ClusterResetDuringReplicationTests : AllureTestBase + public class ClusterResetDuringReplicationTests : TestBase { ClusterTestContext context; @@ -56,7 +54,7 @@ public async Task ClusterResetHardDuringDisklessReplicationAttach(CancellationTo var nodes_count = 2; // Create instances with diskless sync enabled - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, enableDisklessSync: true, timeout: createInstanceTimeout); + context.CreateInstances(nodes_count, enableAOF: true, enableDisklessSync: true, timeout: createInstanceTimeout); context.CreateConnection(); // Setup primary @@ -72,9 +70,11 @@ public async Task ClusterResetHardDuringDisklessReplicationAttach(CancellationTo { ExceptionInjectionHelper.EnableException(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync); + // Initiate replication. var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex: replicaIndex, primaryNodeIndex: primaryIndex, failEx: false, async: true, logger: context.logger); - await Task.Delay(1000, cancellationToken).ConfigureAwait(false); + // Wait for the primary to reach the desired code path (ReplicationManager.TryBeginDisklessSync). + await ExceptionInjectionHelper.WaitOnClearAsync(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync); // Verify that the replica is in a replicating state var replicationInfo = context.clusterTestUtils.GetReplicationInfo(replicaIndex, [ReplicationInfoItem.RECOVER_STATUS], logger: context.logger); @@ -84,6 +84,9 @@ public async Task ClusterResetHardDuringDisklessReplicationAttach(CancellationTo var resetResp = context.clusterTestUtils.ClusterReset(replicaIndex, soft: false, expiry: 60, logger: context.logger); ClassicAssert.AreEqual("OK", resetResp); + // Release waiting task at ReplicationManager.TryBeginDisklessSync + ExceptionInjectionHelper.EnableException(ExceptionInjectionType.Replication_InProgress_During_Diskless_Replica_Attach_Sync); + // Verify that the node is no longer in recovery state replicationInfo = context.clusterTestUtils.GetReplicationInfo(replicaIndex, [ReplicationInfoItem.RECOVER_STATUS], logger: context.logger); ClassicAssert.AreEqual("NoRecovery", replicationInfo[0].Item2); @@ -112,7 +115,7 @@ public async Task ClusterResetHardDuringDiskBasedReplicationAttach(CancellationT var nodes_count = 2; // (diskless sync is false) - context.CreateInstances(nodes_count, disableObjects: false, enableAOF: true, enableDisklessSync: false, timeout: createInstanceTimeout); + context.CreateInstances(nodes_count, enableAOF: true, enableDisklessSync: false, timeout: createInstanceTimeout); context.CreateConnection(); // Setup primary @@ -127,18 +130,23 @@ public async Task ClusterResetHardDuringDiskBasedReplicationAttach(CancellationT { ExceptionInjectionHelper.EnableException(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync); + // Initiate replication. var resp = context.clusterTestUtils.ClusterReplicate(replicaNodeIndex: replicaIndex, primaryNodeIndex: primaryIndex, failEx: false, async: true, logger: context.logger); - await Task.Delay(1000, cancellationToken).ConfigureAwait(false); + // Wait for the primary to reach the desired code path (ReplicationManager.TryBeginDiskbasedSync). + await ExceptionInjectionHelper.WaitOnClearAsync(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync); // Verify that the replica is in a replicating state var replicationInfo = context.clusterTestUtils.GetReplicationInfo(replicaIndex, [ReplicationInfoItem.RECOVER_STATUS], logger: context.logger); ClassicAssert.AreEqual("ClusterReplicate", replicationInfo[0].Item2); - // Issueing CLUSTER RESET HARD while replication is ongoing/stuck. + // Issuing CLUSTER RESET HARD while replication is ongoing/stuck. var resetResp = context.clusterTestUtils.ClusterReset(replicaIndex, soft: false, expiry: 60, logger: context.logger); ClassicAssert.AreEqual("OK", resetResp); + // Release waiting task at ReplicaSyncSession.SendCheckpoint + ExceptionInjectionHelper.EnableException(ExceptionInjectionType.Replication_InProgress_During_DiskBased_Replica_Attach_Sync); + // Verify that the node is no longer in recovery state replicationInfo = context.clusterTestUtils.GetReplicationInfo(replicaIndex, [ReplicationInfoItem.RECOVER_STATUS], logger: context.logger); ClassicAssert.AreEqual("NoRecovery", replicationInfo[0].Item2); diff --git a/test/cluster/Garnet.test.cluster.replication/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.replication/TestProjectSetup.cs new file mode 100644 index 00000000000..5b5b827cc80 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.replication/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterReplication; +} \ No newline at end of file diff --git a/test/cluster/Garnet.test.cluster.vectorsets/Garnet.test.cluster.vectorsets.csproj b/test/cluster/Garnet.test.cluster.vectorsets/Garnet.test.cluster.vectorsets.csproj new file mode 100644 index 00000000000..0db6c2b85b3 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.vectorsets/Garnet.test.cluster.vectorsets.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/cluster/Garnet.test.cluster.vectorsets/TestProjectSetup.cs b/test/cluster/Garnet.test.cluster.vectorsets/TestProjectSetup.cs new file mode 100644 index 00000000000..85255458ec9 --- /dev/null +++ b/test/cluster/Garnet.test.cluster.vectorsets/TestProjectSetup.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.test.cluster; +using NUnit.Framework; + +[SetUpFixture] +public class TestProjectSetup +{ + [OneTimeSetUp] + public void SetPort() => ClusterTestContext.Port = (int)ClusterPortAssignment.ClusterVectorSets; +} \ No newline at end of file diff --git a/test/Garnet.test.cluster/VectorSets/ClusterVectorSetTests.cs b/test/cluster/Garnet.test.cluster.vectorsets/VectorSets/ClusterVectorSetTests.cs similarity index 99% rename from test/Garnet.test.cluster/VectorSets/ClusterVectorSetTests.cs rename to test/cluster/Garnet.test.cluster.vectorsets/VectorSets/ClusterVectorSetTests.cs index 769b6fe792b..8468bafedeb 100644 --- a/test/Garnet.test.cluster/VectorSets/ClusterVectorSetTests.cs +++ b/test/cluster/Garnet.test.cluster.vectorsets/VectorSets/ClusterVectorSetTests.cs @@ -16,7 +16,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; @@ -27,9 +26,10 @@ namespace Garnet.test.cluster { [TestFixture, NonParallelizable] - [AllureNUnit] - public class ClusterVectorSetTests : AllureTestBase + public class ClusterVectorSetTests : TestBase { + protected int sublogCount = 1; + private sealed class StringAndByteArrayComparer : IEqualityComparer<(string Key, byte[] Elem)> { public static readonly StringAndByteArrayComparer Instance = new(); @@ -428,7 +428,7 @@ public async Task RepeatedCreateDeleteAsync(CancellationToken testCancellationTo context.clusterTestUtils.WaitForReplicaAofSync(PrimaryIndex, SecondaryIndex); var querySecondary = (byte[][])context.clusterTestUtils.Execute(secondary, "VSIM", ["foo", "XB8", bytes3]); - ClassicAssert.IsTrue(querySecondary.Length >= 1); + ClassicAssert.IsTrue(querySecondary.Length is (1 or 2)); for (var j = 0; j < querySecondary.Length; j++) { @@ -462,6 +462,7 @@ static void Incr(byte[] k) } [Test] + [CancelAfter(180_000)] public async Task MultipleReplicasWithVectorSetsAsync() { const int PrimaryIndex = 0; @@ -606,6 +607,7 @@ public async Task MultipleReplicasWithVectorSetsAsync() } [Test] + [CancelAfter(180_000)] public async Task MultipleReplicasWithVectorSetsAndDeletesAsync() { const int PrimaryIndex = 0; @@ -1050,7 +1052,7 @@ public async Task VectorSetMigrateByKeysAsync() } // Finish migration - context.clusterTestUtils.WaitForMigrationCleanup(NullLogger.Instance); + context.clusterTestUtils.WaitForMigrationCleanup(); // Validate vector sets coherent for (var i = 0; i < keys.Count; i++) @@ -1076,7 +1078,7 @@ public void VectorSetMigrateManyBySlot() const int Secondary0Index = 2; const int Secondary1Index = 3; - const int VectorSetsPerPrimary = 8; + const int VectorSetsPerPrimary = 4; context.CreateInstances(DefaultMultiPrimaryShards, useTLS: true, enableAOF: true, AofMemorySize: DefaultAOFMemorySize); context.CreateConnection(useTLS: true); @@ -1296,7 +1298,7 @@ public async Task MigrateVectorSetWhileModifyingAsync(CancellationToken testCanc const int Secondary0Index = 2; const int Secondary1Index = 3; - _ = await SimpleSetupClusterAsync(DefaultMultiPrimaryShards, primaryCount: DefaultMultiPrimaryShards / 2, replicaCount: 1, onDemandCheckpoint: true, enableIncrementalSnapshots: true); + _ = await SimpleSetupClusterAsync(DefaultMultiPrimaryShards, primaryCount: DefaultMultiPrimaryShards / 2, replicaCount: 1, onDemandCheckpoint: true); var primary0 = (IPEndPoint)context.endpoints[Primary0Index]; var primary1 = (IPEndPoint)context.endpoints[Primary1Index]; @@ -1987,7 +1989,7 @@ exc is RedisTimeoutException migrateCancel.Cancel(); var migrationTimes = await migrateTask.ConfigureAwait(false); - ClassicAssert.IsTrue(migrationTimes.Count >= 2, $"Should have moved back and forth, saw: {migrationTimes.Count}"); + ClassicAssert.IsTrue(migrationTimes.Count >= 2, $"Should have moved back and forth, saw {migrationTimes}"); writeCancel.Cancel(); await Task.WhenAll(writeTasks).ConfigureAwait(false); @@ -2100,9 +2102,9 @@ public async Task FailoverStopsVectorManagerReplicationTasksAsync() ClassicAssert.IsTrue(vsimRes.Length > 0); } - private Task<(List Shards, List Slots)> SimpleSetupClusterAsync(int shardCount, int primaryCount, int replicaCount, bool onDemandCheckpoint = false, bool enableIncrementalSnapshots = false, bool useTLS = true) + private Task<(List Shards, List Slots)> SimpleSetupClusterAsync(int shardCount, int primaryCount, int replicaCount, bool onDemandCheckpoint = false, bool useTLS = true) { - context.CreateInstances(shardCount, useTLS: useTLS, enableAOF: true, AofMemorySize: DefaultAOFMemorySize, OnDemandCheckpoint: onDemandCheckpoint, EnableIncrementalSnapshots: enableIncrementalSnapshots, threadPoolMinIOCompletionThreads: 512); + context.CreateInstances(shardCount, useTLS: useTLS, enableAOF: true, AofMemorySize: DefaultAOFMemorySize, OnDemandCheckpoint: onDemandCheckpoint, sublogCount: sublogCount, threadPoolMinIOCompletionThreads: 512); context.CreateConnection(useTLS: useTLS); return context.clusterTestUtils.SimpleSetupClusterAsync(primary_count: primaryCount, replica_count: replicaCount); } diff --git a/test/Garnet.test.cluster/ClientClusterConfig.cs b/test/cluster/Garnet.test.cluster/ClientClusterConfig.cs similarity index 100% rename from test/Garnet.test.cluster/ClientClusterConfig.cs rename to test/cluster/Garnet.test.cluster/ClientClusterConfig.cs diff --git a/test/Garnet.test.cluster/ClusterAadAuthTests.cs b/test/cluster/Garnet.test.cluster/ClusterAadAuthTests.cs similarity index 97% rename from test/Garnet.test.cluster/ClusterAadAuthTests.cs rename to test/cluster/Garnet.test.cluster/ClusterAadAuthTests.cs index 80a86f82eed..9988c32300a 100644 --- a/test/Garnet.test.cluster/ClusterAadAuthTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterAadAuthTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; using System.Security.Claims; -using Allure.NUnit; using Garnet.server.Auth.Settings; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Tokens; @@ -14,11 +13,9 @@ namespace Garnet.test.cluster { - - [AllureNUnit] [TestFixture] [NonParallelizable] - class ClusterAadAuthTests : AllureTestBase + class ClusterAadAuthTests : TestBase { ClusterTestContext context; diff --git a/test/Garnet.test.cluster/ClusterAuthCommsTests.cs b/test/cluster/Garnet.test.cluster/ClusterAuthCommsTests.cs similarity index 95% rename from test/Garnet.test.cluster/ClusterAuthCommsTests.cs rename to test/cluster/Garnet.test.cluster/ClusterAuthCommsTests.cs index 9ed5f44dd26..e6563f6413f 100644 --- a/test/Garnet.test.cluster/ClusterAuthCommsTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterAuthCommsTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Collections.Generic; using System.Linq; using System.Net; -using Allure.NUnit; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,9 +11,8 @@ namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture, NonParallelizable] - internal class ClusterAuthCommsTests : AllureTestBase + internal class ClusterAuthCommsTests : TestBase { ClusterTestContext context; @@ -126,7 +124,7 @@ public void ClusterReplicationAuth() context.CreateConnection(clientCreds: context.credManager.GetUserCredentials("admin")); // Assign slots - _ = context.clusterTestUtils.AddSlotsRange(0, new List<(int, int)> { (0, 16383) }, logger: context.logger); + _ = context.clusterTestUtils.AddSlotsRange(0, [(0, 16383)], logger: context.logger); // Retrieve credentials var cred = context.credManager.GetUserCredentials("admin"); @@ -141,13 +139,13 @@ public void ClusterReplicationAuth() } // Initiate meet now that we know the credentatial - for (int i = 1; i < shards; i++) + for (var i = 1; i < shards; i++) context.clusterTestUtils.Meet(0, i, logger: context.logger); // Wait for config convergence context.clusterTestUtils.WaitAll(logger: context.logger); - context.kvPairs = new(); + context.kvPairs = []; //Populate Primary context.PopulatePrimary(ref context.kvPairs, keyLength: 8, kvpairCount: 100, primaryIndex: 0); @@ -188,8 +186,8 @@ public void ClusterSimpleFailoverAuth() context.ClusterFailoverSpinWait(replicaNodeIndex: 1, logger: context.logger); // Reconfigure slotMap to reflect new primary - int[] slotMap = new int[16384]; - for (int i = 0; i < 16384; i++) + var slotMap = new int[16384]; + for (var i = 0; i < 16384; i++) slotMap[i] = 1; context.PopulatePrimary(ref context.kvPairs, keyLength: 8, kvpairCount: 100, primaryIndex: 1); diff --git a/test/Garnet.test.cluster/ClusterConfigTests.cs b/test/cluster/Garnet.test.cluster/ClusterConfigTests.cs similarity index 62% rename from test/Garnet.test.cluster/ClusterConfigTests.cs rename to test/cluster/Garnet.test.cluster/ClusterConfigTests.cs index e68f987fdac..b4cdeae9a44 100644 --- a/test/Garnet.test.cluster/ClusterConfigTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterConfigTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Collections.Generic; using System.Linq; using System.Net; using System.Text; -using Allure.NUnit; using Garnet.cluster; using Garnet.common; using Microsoft.Extensions.Logging; @@ -14,9 +13,8 @@ namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture, NonParallelizable] - internal class ClusterConfigTests : AllureTestBase + internal class ClusterConfigTests : TestBase { ClusterTestContext context; @@ -42,7 +40,7 @@ public void ClusterConfigInitializesUnassignedWorkerTest() var config = new ClusterConfig().InitializeLocalWorker( Generator.CreateHexId(), "127.0.0.1", - 7001, + ClusterTestContext.Port + 1, configEpoch: 0, Garnet.cluster.NodeRole.PRIMARY, null, @@ -95,13 +93,13 @@ public void ClusterForgetAfterNodeRestartTest() nodesResult = context.clusterTestUtils.ClusterNodes(0); Assert.That(nodesResult.Nodes.Count == nbInstances, "No node should've been removed from the cluster after an invalid id was passed."); Assert.That(nodesResult.Nodes.ElementAt(0).IsMyself); - Assert.That(nodesResult.Nodes.ElementAt(0).EndPoint.ToIPEndPoint().Port == 7000, "Expected the node to be replying to be the one with port 7000."); + Assert.That(nodesResult.Nodes.ElementAt(0).EndPoint.ToIPEndPoint().Port == ClusterTestContext.Port, $"Expected the node to be replying to be the one with ClusterTestContext.Port {ClusterTestContext.Port} pt 1."); context.clusterTestUtils.ClusterForget(0, nodesResult.Nodes.Last().NodeId, 0); nodesResult = context.clusterTestUtils.ClusterNodes(0); Assert.That(nodesResult.Nodes.Count == nbInstances - 1, "A node should've been removed from the cluster."); Assert.That(nodesResult.Nodes.ElementAt(0).IsMyself); - Assert.That(nodesResult.Nodes.ElementAt(0).EndPoint.ToIPEndPoint().Port == 7000, "Expected the node to be replying to be the one with port 7000."); + Assert.That(nodesResult.Nodes.ElementAt(0).EndPoint.ToIPEndPoint().Port == ClusterTestContext.Port, $"Expected the node to be replying to be the one with ClusterTestContext.Port {ClusterTestContext.Port} pt 2."); } [Test, Order(2)] @@ -138,17 +136,17 @@ public void ClusterAnnounceRecoverTest() public void ClusterAnyIPAnnounce() { context.nodes = new GarnetServer[1]; - context.nodes[0] = context.CreateInstance(new IPEndPoint(IPAddress.Any, 7000)); + context.nodes[0] = context.CreateInstance(new IPEndPoint(IPAddress.Any, ClusterTestContext.Port)); context.nodes[0].Start(); - context.endpoints = TestUtils.GetShardEndPoints(1, IPAddress.Loopback, 7000); + context.endpoints = TestUtils.GetShardEndPoints(1, IPAddress.Loopback, ClusterTestContext.Port); context.CreateConnection(); var config = context.clusterTestUtils.ClusterNodes(0, logger: context.logger); var origin = config.Origin; var endpoint = origin.ToIPEndPoint(); - ClassicAssert.AreEqual(7000, endpoint.Port); + ClassicAssert.AreEqual(ClusterTestContext.Port, endpoint.Port); using var client = TestUtils.GetGarnetClient(config.Origin); client.Connect(); @@ -157,5 +155,88 @@ public void ClusterAnyIPAnnounce() resp = client.QuitAsync().GetAwaiter().GetResult(); ClassicAssert.AreEqual("OK", resp); } + + [Test, Order(4)] + [Category("CLUSTER-CONFIG"), CancelAfter(1000)] + public void ClusterConfigVersionRoundTripTest() + { + var config = new ClusterConfig().InitializeLocalWorker( + Generator.CreateHexId(), + "127.0.0.1", + ClusterTestContext.Port + 1, + configEpoch: 1, + Garnet.cluster.NodeRole.PRIMARY, + null, + ""); + + var configBytes = config.ToByteArray(); + + // Verify version byte at start of payload + Assert.That(ClusterConfig.TryPeekVersion(configBytes, out var version), Is.True); + Assert.That(version, Is.EqualTo(ClusterConfig.ClusterConfigVersion)); + + // Round-trip should succeed + var restored = ClusterConfig.FromByteArray(configBytes); + Assert.That(restored.LocalNodeId, Is.EqualTo(config.LocalNodeId)); + } + + [Test, Order(5)] + [Category("CLUSTER-CONFIG"), CancelAfter(1000)] + public void ClusterConfigVersionMismatchThrowsTest() + { + var config = new ClusterConfig().InitializeLocalWorker( + Generator.CreateHexId(), + "127.0.0.1", + ClusterTestContext.Port + 1, + configEpoch: 1, + Garnet.cluster.NodeRole.PRIMARY, + null, + ""); + + var configBytes = config.ToByteArray(); + + // Corrupt the version byte (at index 0) + configBytes[0] = (byte)(ClusterConfig.ClusterConfigVersion + 1); + + // Deserialization should throw + Assert.Throws(() => ClusterConfig.FromByteArray(configBytes)); + } + + [Test, Order(6)] + [Category("CLUSTER-CONFIG"), CancelAfter(1000)] + public void ClusterConfigTryPeekVersionEmptyDataTest() + { + Assert.That(ClusterConfig.TryPeekVersion([], out _), Is.False); + } + + [Test, Order(7)] + [Category("CLUSTER-CONFIG"), CancelAfter(1000)] + public void ReplicationHistoryVersionRoundTripTest() + { + var history = new ReplicationHistory(1); + var bytes = history.ToByteArray(); + + // Verify version byte at start of payload + Assert.That(bytes[0], Is.EqualTo(ReplicationHistory.ReplicationHistoryVersion)); + + // Round-trip should succeed and preserve fields + var restored = ReplicationHistory.FromByteArray(bytes); + Assert.That(restored.PrimaryReplId, Is.EqualTo(history.PrimaryReplId)); + Assert.That(restored.PrimaryReplId2, Is.EqualTo(history.PrimaryReplId2)); + } + + [Test, Order(9)] + [Category("CLUSTER-CONFIG"), CancelAfter(1000)] + public void ReplicationHistoryVersionMismatchThrowsTest() + { + var history = new ReplicationHistory(1); + var bytes = history.ToByteArray(); + + // Corrupt the version byte (at index 0) + bytes[0] = (byte)(ReplicationHistory.ReplicationHistoryVersion + 1); + + // Deserialization should throw + Assert.Throws(() => ReplicationHistory.FromByteArray(bytes)); + } } } \ No newline at end of file diff --git a/test/Garnet.test.cluster/ClusterManagementTests.cs b/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs similarity index 93% rename from test/Garnet.test.cluster/ClusterManagementTests.cs rename to test/cluster/Garnet.test.cluster/ClusterManagementTests.cs index 23ad0ac1ea2..9e80d05a054 100644 --- a/test/Garnet.test.cluster/ClusterManagementTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterManagementTests.cs @@ -10,7 +10,6 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -25,10 +24,8 @@ public enum InstanceType Standalone, Cluster } - - [AllureNUnit] [TestFixture, NonParallelizable] - public class ClusterManagementTests : AllureTestBase + public class ClusterManagementTests : TestBase { ClusterTestContext context; readonly int defaultShards = 3; @@ -669,7 +666,7 @@ public void ClusterClientList() var numReplica = fullList.Split("\n").Count(static x => x.Contains(" flags=S ")); var numMaster = fullList.Split("\n").Count(static x => x.Contains(" flags=M ")); - ClassicAssert.AreEqual(1, numNormal, $"normalCheck: nodeIx={nodeIx}, normal={numNormal}, replica={numReplica}, master={numMaster}"); + ClassicAssert.IsTrue(2 >= numNormal, $"normalCheck: nodeIx={nodeIx}, normal={numNormal}, replica={numReplica}, master={numMaster}"); ClassicAssert.IsTrue(numReplica >= 1 && numReplica <= 2, $"replicaCheck: nodeIx={nodeIx}, normal={numNormal}, replica={numReplica}, master={numMaster}"); ClassicAssert.IsTrue(numMaster >= 1 && numMaster <= 2, $"masterCheck: nodeIx={nodeIx}, normal={numNormal}, replica={numReplica}, master={numMaster}"); @@ -849,29 +846,77 @@ public void ClusterSetSlotBadOptions() } [Test, Order(11)] - public void ClusterRoleCommand() + public void ClusterRoleCommand([Values] bool useMultiLog) { + var primaryIndex = 0; + var sublogCount = useMultiLog ? 4 : 1; var node_count = 3; var replica_count = node_count - 1; - context.CreateInstances(node_count, enableAOF: true); + context.CreateInstances(node_count, enableAOF: true, sublogCount: sublogCount); context.CreateConnection(); var (_, _) = context.clusterTestUtils.SimpleSetupCluster(1, replica_count, logger: context.logger); - var result = context.clusterTestUtils.GetServer(0).Execute("ROLE"); - ClassicAssert.AreEqual(3, result.Length); - ClassicAssert.AreEqual("master", result[0].ToString()); - ClassicAssert.True(int.TryParse(result[1].ToString(), out _)); - ClassicAssert.AreEqual(2, result[2].Length); - ClassicAssert.AreEqual("127.0.0.1", result[2][0][0].ToString()); - ClassicAssert.AreEqual("127.0.0.1", result[2][1][0].ToString()); - - result = context.clusterTestUtils.GetServer(1).Execute("ROLE"); - ClassicAssert.AreEqual(5, result.Length); - ClassicAssert.AreEqual("slave", result[0].ToString()); - ClassicAssert.AreEqual("127.0.0.1", result[1].ToString()); - ClassicAssert.True(int.TryParse(result[2].ToString(), out _)); - ClassicAssert.AreEqual("connected", result[3].ToString()); - ClassicAssert.True(int.TryParse(result[4].ToString(), out _)); + var primaryFormatLength = Enum.GetValues().Length - (useMultiLog ? 0 : 1); + var result = context.clusterTestUtils.GetServer(primaryIndex).Execute("ROLE"); + ClassicAssert.AreEqual(primaryFormatLength, result.Length); + // RoleType + ClassicAssert.AreEqual("master", result[(int)RoleCommandPrimaryFormat.RoleType].ToString()); + // Replication offset + ClassicAssert.True(int.TryParse(result[(int)RoleCommandPrimaryFormat.RoleReplicationOffset].ToString(), out var parsed)); + ClassicAssert.AreEqual(64, parsed); + + ClassicAssert.AreEqual(2, result[(int)RoleCommandPrimaryFormat.RoleReplicaInfo].Length); + if (useMultiLog) + { + // ReplicationOffsetVector (MultiLog support) + var aofAddress = AofAddress.FromString(result[(int)RoleCommandPrimaryFormat.RoleReplicationOffsetString].ToString()); + ClassicAssert.AreEqual(sublogCount, aofAddress.Length); + } + + for (var i = 0; i < replica_count; i++) + { + var primaryResultForReplica = result[(int)RoleCommandPrimaryFormat.RoleReplicaInfo][i]; + var replicaIndex = i + 1; + var roleReplicaFormatLength = Enum.GetValues().Length - (useMultiLog ? 0 : 1); + + // NOTE: Role command from primary perspective does not include role type or connection status + ClassicAssert.AreEqual(roleReplicaFormatLength - 2, primaryResultForReplica.Length); + // Address + ClassicAssert.AreEqual("127.0.0.1", primaryResultForReplica[(int)RoleCommandReplicaFormat.RoleAddress - 1].ToString(), "Failed to match replica address"); + // Port + ClassicAssert.True(int.TryParse(primaryResultForReplica[(int)RoleCommandReplicaFormat.RolePort - 1].ToString(), out parsed), "Failed to match replica port"); + ClassicAssert.AreEqual(ClusterTestContext.Port + replicaIndex, parsed); + // ReplicationOffset + ClassicAssert.True(int.TryParse(primaryResultForReplica[(int)RoleCommandReplicaFormat.RoleReplicationOffset - 2].ToString(), out parsed), "Failed to match replica replication offset"); + ClassicAssert.AreEqual(64, parsed); + if (useMultiLog) + { + // ReplicationOffsetVector (MultiLog support) + var replicaAofAddress = AofAddress.FromString(primaryResultForReplica[(int)RoleCommandReplicaFormat.RoleReplicationOffsetString - 2].ToString()); + ClassicAssert.AreEqual(sublogCount, replicaAofAddress.Length); + } + + var replicaResult = context.clusterTestUtils.GetServer(i + 1).Execute("ROLE"); + ClassicAssert.AreEqual(roleReplicaFormatLength, replicaResult.Length); + // RoleType + ClassicAssert.AreEqual("slave", replicaResult[(int)RoleCommandReplicaFormat.RoleType].ToString()); + // Replica Address + ClassicAssert.AreEqual("127.0.0.1", replicaResult[(int)RoleCommandReplicaFormat.RoleAddress].ToString()); + // Replica Port + ClassicAssert.True(int.TryParse(replicaResult[(int)RoleCommandReplicaFormat.RolePort].ToString(), out parsed)); + ClassicAssert.AreEqual(ClusterTestContext.Port, parsed); + // Connection State + ClassicAssert.AreEqual("connected", replicaResult[(int)RoleCommandReplicaFormat.RoleState].ToString()); + // ReplicationOffset + ClassicAssert.True(int.TryParse(replicaResult[(int)RoleCommandReplicaFormat.RoleReplicationOffset].ToString(), out parsed)); + ClassicAssert.AreEqual(64, parsed); + if (useMultiLog) + { + // ReplicationOffsetVector (MultiLog support) + var replicaAofAddress = AofAddress.FromString(replicaResult[(int)RoleCommandReplicaFormat.RoleReplicationOffsetString].ToString()); + ClassicAssert.AreEqual(sublogCount, replicaAofAddress.Length); + } + } } [Test, Order(12)] @@ -1210,7 +1255,6 @@ public async Task PrimaryUnavailableRecoveryAsync(ExceptionInjectionType faultTy metricsSamplingFrequency: 1, loggingFrequencySecs: 10, checkpointThrottleFlushDelayMs: 0, - FastCommit: true, FastAofTruncate: true, OnDemandCheckpoint: true, useTLS: true, diff --git a/test/Garnet.test.cluster/ClusterNegativeTests.cs b/test/cluster/Garnet.test.cluster/ClusterNegativeTests.cs similarity index 99% rename from test/Garnet.test.cluster/ClusterNegativeTests.cs rename to test/cluster/Garnet.test.cluster/ClusterNegativeTests.cs index 31dcbee3fce..a73e90213ab 100644 --- a/test/Garnet.test.cluster/ClusterNegativeTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterNegativeTests.cs @@ -8,7 +8,6 @@ using System.Net.Sockets; using System.Text; using System.Threading; -using Allure.NUnit; using Microsoft.Extensions.Logging; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -23,9 +22,8 @@ namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture, NonParallelizable] - public class ClusterNegativeTests : AllureTestBase + public class ClusterNegativeTests : TestBase { ClusterTestContext context; @@ -80,11 +78,11 @@ public void TearDown() [TestCase("mtasks", new int[] { 1, 2, 3, 4 })] [TestCase("replicas", new int[] { 0, 2, 3, 4 })] [TestCase("replicate", new int[] { 0, 3, 4 })] - [TestCase("AOFSYNC", new int[] { 0, 1, 3, 4 })] [TestCase("APPENDLOG", new int[] { 0, 1, 2, 3, 4, 6 })] [TestCase("INITIATE_REPLICA_SYNC", new int[] { 0, 1, 2, 3, 4, 6 })] [TestCase("SEND_CKPT_METADATA", new int[] { 0, 1, 2, 4, 5, 6 })] - [TestCase("SEND_CKPT_FILE_SEGMENT", new int[] { 0, 1, 2, 3, 4, 6 })] + [TestCase("SEND_CKPT_FILE_SEGMENT", new int[] { 0, 1, 2, 3, 5, 6 })] + [TestCase("SNAPSHOT_DATA", new int[] { 0, 1, 2, 3, 5, 6 })] [TestCase("BEGIN_REPLICA_RECOVER", new int[] { 0, 1, 2, 3, 4, 5, 6, 8, 9 })] [TestCase("FAILSTOPWRITES", new int[] { 0, 2, 3, 4 })] [TestCase("FAILREPLICATIONOFFSET", new int[] { 0, 2, 3, 4 })] @@ -94,7 +92,7 @@ public void ClusterCommandWrongParameters(string subcommand, params int[] invali using var socket = new Socket(SocketType.Stream, ProtocolType.Tcp); socket.NoDelay = true; - socket.Connect(IPAddress.Loopback, 7000); + socket.Connect(IPAddress.Loopback, ClusterTestContext.Port); var clusterCMD = $"$7\r\ncluster\r\n${subcommand.Length}\r\n{subcommand}\r\n"; var errorCmd = $"cluster|{subcommand.ToLowerInvariant()}"; @@ -129,7 +127,7 @@ public void ClusterAddSlotsPartialPackage(int chunkSize) context.CreateInstances(1); using var socket = new Socket(SocketType.Stream, ProtocolType.Tcp); socket.NoDelay = true; - socket.Connect(IPAddress.Loopback, 7000); + socket.Connect(IPAddress.Loopback, ClusterTestContext.Port); var slots = Enumerable.Range(0, 8192).ToList(); var packet = $"*{2 + slots.Count}\r\n$7\r\ncluster\r\n$8\r\naddslots\r\n"; @@ -343,7 +341,7 @@ public void ClusterReplicaAttachIntenseWrite(CancellationToken cancellationToken context.PopulatePrimary(ref context.kvPairs, keyLength, kvpairCount, primaryIndex, null); var primaryOffset2 = context.clusterTestUtils.GetReplicationOffset(primaryIndex, logger: context.logger); - ClassicAssert.Less(primaryOffset1, primaryOffset2); + ClassicAssert.Less(primaryOffset1[0], primaryOffset2[0]); // Take another checkpoin to truncate primaryLastSaveTime = context.clusterTestUtils.LastSave(primaryIndex, logger: context.logger); @@ -626,6 +624,7 @@ public void ClusterReplicateFails() var exc = Assert.Throws(() => replicaServer.Execute("CLUSTER", ["REPLICATE", Guid.NewGuid().ToString()], flags: CommandFlags.NoRedirect)); ClassicAssert.IsTrue(exc.Message.StartsWith("ERR I don't know about node ")); } + [Test, Order(14), CancelAfter(testTimeout)] [Category("REPLICATION")] public async Task ClusterFailoverSucceedsDuringEnsureReplicationAsync(CancellationToken cancellationToken) diff --git a/test/Garnet.test.cluster/ClusterRedirectTests.cs b/test/cluster/Garnet.test.cluster/ClusterRedirectTests.cs similarity index 99% rename from test/Garnet.test.cluster/ClusterRedirectTests.cs rename to test/cluster/Garnet.test.cluster/ClusterRedirectTests.cs index 3d93f50bc92..c5d82985aa6 100644 --- a/test/Garnet.test.cluster/ClusterRedirectTests.cs +++ b/test/cluster/Garnet.test.cluster/ClusterRedirectTests.cs @@ -1,11 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; using System.Text; -using Allure.NUnit; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -14,9 +13,8 @@ namespace Garnet.test.cluster { - [AllureNUnit] [TestFixture, NonParallelizable] - public unsafe class ClusterRedirectTests : AllureTestBase + public unsafe class ClusterRedirectTests : TestBase { ClusterTestContext context; diff --git a/test/Garnet.test.cluster/ClusterTestContext.cs b/test/cluster/Garnet.test.cluster/ClusterTestContext.cs similarity index 91% rename from test/Garnet.test.cluster/ClusterTestContext.cs rename to test/cluster/Garnet.test.cluster/ClusterTestContext.cs index 5eb42adbd66..dca9a21b40f 100644 --- a/test/Garnet.test.cluster/ClusterTestContext.cs +++ b/test/cluster/Garnet.test.cluster/ClusterTestContext.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Garnet.common; using Garnet.server; using Garnet.server.Auth.Settings; using Microsoft.Extensions.Logging; @@ -21,6 +22,21 @@ namespace Garnet.test.cluster { + /// + /// Unique base port for each cluster test sub-project, enabling parallel test runs without port conflicts. + /// + public enum ClusterPortAssignment + { + ClusterTest = 7000, + ClusterMigrate = 7100, + ClusterReplication = 7200, + ClusterReplicationTls = 7300, + ClusterReplicationAsync = 7400, + ClusterReplicationDiskless = 7500, + ClusterVectorSets = 7600, + ClusterMultiLog = 7700, + } + public class ClusterTestContext { public CredentialManager credManager; @@ -30,10 +46,11 @@ public class ClusterTestContext public EndPointCollection endpoints; public TextWriter logTextWriter = TestContext.Progress; public ILoggerFactory loggerFactory; + public NUnitLoggerProvider loggerProvider; public ILogger logger; public int defaultShards = 3; - public static int Port = 7000; + public static int Port = (int)ClusterPortAssignment.ClusterTest; // No OneTimeSetUp needed for "Garnet.test.cluster" to set this public Random r = new(); public ManualResetEventSlim waiter; @@ -44,6 +61,12 @@ public class ClusterTestContext public CancellationTokenSource cts; + public void EnableGarnetLoggingEvents(GarnetTestLoggingEventType[] events) + { + foreach (var e in events) + loggerProvider.GarnetTestLoggingEvents[(int)e] = true; + } + public void Setup(Dictionary monitorTests, int testTimeoutSeconds = 60) { // Pull timeout off [CancelAfter] if its specified, otherwise use default @@ -58,7 +81,7 @@ public void Setup(Dictionary monitorTests, int testTimeoutSeco var logLevel = LogLevel.Error; if (!string.IsNullOrEmpty(TestContext.CurrentContext.Test.MethodName) && monitorTests.TryGetValue(TestContext.CurrentContext.Test.MethodName, out var value)) logLevel = value; - loggerFactory = TestUtils.CreateLoggerFactoryInstance(logTextWriter, logLevel, scope: TestContext.CurrentContext.Test.FullName); + (loggerFactory, loggerProvider) = TestUtils.CreateLoggerFactoryInstance(logTextWriter, logLevel, scope: TestContext.CurrentContext.Test.FullName); logger = loggerFactory.CreateLogger(TestContext.CurrentContext.Test.FullName); logger.LogDebug("0. Setup >>>>>>>>>>>>"); r = new Random(674386); @@ -124,7 +147,6 @@ public void RestartNode(int nodeIndex) nodes[nodeIndex].Start(); } - public void TearDown() { // Capture test outcome before any teardown work to distinguish @@ -219,8 +241,6 @@ public void RegisterCustomTxn(string name, Func proc /// /// /// - /// - /// /// /// /// @@ -239,6 +259,7 @@ public void RegisterCustomTxn(string name, Func proc /// /// /// + /// /// /// /// @@ -252,6 +273,7 @@ public void RegisterCustomTxn(string name, Func proc /// /// /// + /// public void CreateInstances( int shards, bool enableCluster = true, @@ -269,8 +291,6 @@ public void CreateInstances( int CommitFrequencyMs = 0, bool useAofNullDevice = false, bool DisableStorageTier = false, - bool EnableIncrementalSnapshots = false, - bool FastCommit = true, int timeout = -1, bool useTLS = false, bool useAcl = false, @@ -298,6 +318,8 @@ public void CreateInstances( int checkpointThrottleFlushDelayMs = 0, bool clusterReplicaResumeWithData = false, int replicaSyncTimeout = 60, + int sublogCount = 1, + int replayTaskCount = 1, int expiredObjectCollectionFrequencySecs = 0, ClusterPreferredEndpointType clusterPreferredEndpointType = ClusterPreferredEndpointType.Ip, bool useClusterAnnounceHostname = false, @@ -305,8 +327,8 @@ public void CreateInstances( int threadPoolMinIOCompletionThreads = 0) { var ipAddress = IPAddress.Loopback; - TestUtils.EndPoint = new IPEndPoint(ipAddress, 7000); - endpoints = TestUtils.GetShardEndPoints(shards, useHostname ? IPAddress.Any : ipAddress, 7000); + TestUtils.EndPoint = new IPEndPoint(ipAddress, Port); + endpoints = TestUtils.GetShardEndPoints(shards, useHostname ? IPAddress.Any : ipAddress, Port); (nodes, nodeOptions) = TestUtils.CreateGarnetCluster( TestFolder, @@ -330,8 +352,6 @@ public void CreateInstances( useAofNullDevice: useAofNullDevice, DisableStorageTier: DisableStorageTier, OnDemandCheckpoint: OnDemandCheckpoint, - EnableIncrementalSnapshots: EnableIncrementalSnapshots, - FastCommit: FastCommit, useAcl: useAcl, aclFile: credManager.aclFilePath, authUsername: clusterCreds.user, @@ -357,6 +377,8 @@ public void CreateInstances( checkpointThrottleFlushDelayMs: checkpointThrottleFlushDelayMs, clusterReplicaResumeWithData: clusterReplicaResumeWithData, replicaSyncTimeout: replicaSyncTimeout, + sublogCount: sublogCount, + replayTaskCount: replayTaskCount, expiredObjectCollectionFrequencySecs: expiredObjectCollectionFrequencySecs, clusterPreferredEndpointType: clusterPreferredEndpointType, clusterAnnounceHostname: useClusterAnnounceHostname ? "localhost" : null, @@ -366,7 +388,7 @@ public void CreateInstances( foreach (var node in nodes) node.Start(); - endpoints = TestUtils.GetShardEndPoints(shards, ipAddress, 7000); + endpoints = TestUtils.GetShardEndPoints(shards, ipAddress, Port); } /// @@ -388,17 +410,17 @@ public void CreateInstances( /// /// /// - /// - /// /// /// /// /// /// + /// /// /// /// /// + /// /// public GarnetServer CreateInstance( EndPoint endpoint, @@ -417,23 +439,25 @@ public GarnetServer CreateInstance( string AofMemorySize = "64m", int CommitFrequencyMs = 0, bool DisableStorageTier = false, - bool EnableIncrementalSnapshots = false, - bool FastCommit = true, int timeout = -1, int gossipDelay = 5, bool useTLS = false, bool useAcl = false, bool asyncReplay = false, + int sublogCount = 1, + int replayTaskCount = 1, int vectorSetReplayTaskCount = 0, EndPoint clusterAnnounceEndpoint = null, X509CertificateCollection certificates = null, - ServerCredential clusterCreds = new ServerCredential()) + ServerCredential clusterCreds = new ServerCredential(), + int threadPoolMinIOCompletionThreads = 0) { var opts = TestUtils.GetGarnetServerOptions( TestFolder, TestFolder, endpoint, + logger: loggerFactory?.CreateLogger("GarnetServer"), enableCluster: enableCluster, disablePubSub: true, disableObjects: disableObjects, @@ -452,16 +476,17 @@ public GarnetServer CreateInstance( commitFrequencyMs: CommitFrequencyMs, disableStorageTier: DisableStorageTier, onDemandCheckpoint: OnDemandCheckpoint, - enableIncrementalSnapshots: EnableIncrementalSnapshots, - fastCommit: FastCommit, useAcl: useAcl, asyncReplay: asyncReplay, + sublogCount: sublogCount, + replayTaskCount: replayTaskCount, aclFile: credManager.aclFilePath, authUsername: clusterCreds.user, authPassword: clusterCreds.password, certificates: certificates, clusterAnnounceEndpoint: clusterAnnounceEndpoint, - vectorSetReplayTaskCount: vectorSetReplayTaskCount); + vectorSetReplayTaskCount: vectorSetReplayTaskCount, + threadPoolMinIOCompletionThreads: threadPoolMinIOCompletionThreads); return new GarnetServer(opts, loggerFactory); } @@ -473,17 +498,17 @@ public void DisposeCluster() { if (nodes != null) { - _ = Parallel.For(0, nodes.Length, i => + for (var i = 0; i < nodes.Length; i++) { if (nodes[i] != null) { - logger.LogDebug("\t a. Dispose node {testName}", TestContext.CurrentContext.Test.Name); + logger.LogDebug("\t a. Before dispose node {i}{testName}", i, TestContext.CurrentContext.Test.Name); var node = nodes[i]; nodes[i] = null; node.Dispose(true); - logger.LogDebug("\t b. Dispose node {testName}", TestContext.CurrentContext.Test.Name); + logger.LogDebug("\t b. After dispose node {i}{testName}", i, TestContext.CurrentContext.Test.Name); } - }); + } } } @@ -532,8 +557,6 @@ public void PopulatePrimary( int kvpairCount, int primaryIndex, int[] slotMap = null, - bool incrementalSnapshots = false, - int ckptNode = 0, int randomSeed = -1) { if (randomSeed != -1) clusterTestUtils.InitRandom(randomSeed); @@ -558,9 +581,6 @@ public void PopulatePrimary( ClassicAssert.AreEqual(value, int.Parse(retVal)); kvPairs.Add(key, int.Parse(retVal)); - - if (incrementalSnapshots && i == kvpairCount / 2) - clusterTestUtils.Checkpoint(ckptNode, logger: logger); } } @@ -582,16 +602,16 @@ public void SimplePopulateDB(bool disableObjects, int keyLength, int kvpairCount { //Populate Primary if (disableObjects) - { - PopulatePrimary(ref kvPairs, keyLength, kvpairCount, primaryIndex); - } - else { if (!performRMW) - PopulatePrimaryWithObjects(ref kvPairsObj, keyLength, kvpairCount, primaryIndex); + PopulatePrimary(ref kvPairs, keyLength, kvpairCount, primaryIndex); else PopulatePrimaryRMW(ref kvPairs, keyLength, kvpairCount, primaryIndex, addCount); } + else + { + PopulatePrimaryWithObjects(ref kvPairsObj, keyLength, kvpairCount, primaryIndex); + } } public void SimpleValidateDB(bool disableObjects, int replicaIndex) @@ -607,7 +627,7 @@ public void SimpleValidateDB(bool disableObjects, int replicaIndex) } } - public void PopulatePrimaryRMW(ref Dictionary kvPairs, int keyLength, int kvpairCount, int primaryIndex, int addCount, int[] slotMap = null, bool incrementalSnapshots = false, int ckptNode = 0, int randomSeed = -1) + public void PopulatePrimaryRMW(ref Dictionary kvPairs, int keyLength, int kvpairCount, int primaryIndex, int addCount, int[] slotMap = null, int randomSeed = -1) { if (randomSeed != -1) clusterTestUtils.InitRandom(randomSeed); for (int i = 0; i < kvpairCount; i++) @@ -627,9 +647,6 @@ public void PopulatePrimaryRMW(ref Dictionary kvPairs, int keyLengt value = clusterTestUtils.IncrBy(primaryIndex, key, randomSeed == -1 ? 1 : clusterTestUtils.r.Next(1, 100)); kvPairs.Add(key, value); - - if (incrementalSnapshots && i == kvpairCount / 2) - clusterTestUtils.Checkpoint(ckptNode, logger: logger); } } @@ -785,31 +802,31 @@ public void ClusterFailoverSpinWait(int replicaNodeIndex, ILogger logger) } } - public async Task AttachAndWaitForSyncAsync(int primary_count, int replica_count, bool disableObjects) + public async Task AttachAndWaitForSyncAsync(int primaryIndex, int replicaStartIndex, int replicaCount, bool disableObjects) { - var primaryId = clusterTestUtils.GetNodeIdFromNode(0, logger); + var primaryId = clusterTestUtils.GetNodeIdFromNode(primaryIndex, logger); // Wait until primary node is known so as not to fail replicate - for (var i = primary_count; i < primary_count + replica_count; i++) + for (var i = replicaStartIndex; i < replicaStartIndex + replicaCount; i++) clusterTestUtils.WaitUntilNodeIdIsKnown(i, primaryId, logger: logger); // Issue cluster replicate and bump epoch manually to capture config. - for (var i = primary_count; i < primary_count + replica_count; i++) + for (var i = replicaStartIndex; i < replicaStartIndex + replicaCount; i++) _ = clusterTestUtils.ClusterReplicate(i, primaryId, async: true, logger: logger); if (!checkpointTask.Wait(TimeSpan.FromSeconds(100))) Assert.Fail("Checkpoint task timeout"); // Wait for recovery and AofSync - for (var i = primary_count; i < replica_count; i++) + for (var i = replicaStartIndex; i < replicaStartIndex + replicaCount; i++) { clusterTestUtils.WaitForReplicaRecovery(i, logger); - clusterTestUtils.WaitForReplicaAofSync(0, i, logger); + clusterTestUtils.WaitForReplicaAofSync(primaryIndex, i, logger); } - await clusterTestUtils.WaitForConnectedReplicaCountAsync(0, replica_count, logger: logger).ConfigureAwait(false); + await clusterTestUtils.WaitForConnectedReplicaCountAsync(0, replicaCount, logger: logger).ConfigureAwait(false); // Validate data on replicas - for (var i = primary_count; i < replica_count; i++) + for (var i = replicaStartIndex; i < replicaStartIndex + replicaCount; i++) { if (disableObjects) ValidateKVCollectionAgainstReplica(ref kvPairs, i); diff --git a/test/Garnet.test.cluster/ClusterTestUtils.cs b/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs similarity index 94% rename from test/Garnet.test.cluster/ClusterTestUtils.cs rename to test/cluster/Garnet.test.cluster/ClusterTestUtils.cs index c7b299f5a80..547506c7e3e 100644 --- a/test/Garnet.test.cluster/ClusterTestUtils.cs +++ b/test/cluster/Garnet.test.cluster/ClusterTestUtils.cs @@ -15,6 +15,7 @@ using System.Threading.Tasks; using Garnet.client; using Garnet.common; +using Garnet.server; using Garnet.server.TLS; using GarnetClusterManagement; using Microsoft.Extensions.Logging; @@ -99,12 +100,11 @@ public enum ReplicationInfoItem : byte STORE_CURRENT_SAFE_AOF_ADDRESS, STORE_RECOVERED_SAFE_AOF_ADDRESS, - OBJECT_STORE_CURRENT_SAFE_AOF_ADDRESS, - OBJECT_STORE_RECOVERED_SAFE_AOF_ADDRESS, PRIMARY_SYNC_IN_PROGRESS, PRIMARY_FAILOVER_STATE, RECOVER_STATUS, LAST_FAILOVER_STATE, + SYNC_DRIVER_COUNT, MASTER_HOST, MASTER_PORT, @@ -119,14 +119,32 @@ public enum StoreInfoItem public struct PersistencInfo { - public long CommittedBeginAddress; - public long CommittedUntilAddress; - public long FlushedUntilAddress; - public long BeginAddress; - public long TailAddress; - public long SafeAofAddress; + public AofAddress CommittedBeginAddress; + public AofAddress CommittedUntilAddress; + public AofAddress FlushedUntilAddress; + public AofAddress BeginAddress; + public AofAddress TailAddress; + public AofAddress SafeAofAddress; }; + enum RoleCommandPrimaryFormat : byte + { + RoleType = 0, + RoleReplicationOffset = 1, + RoleReplicaInfo = 2, + RoleReplicationOffsetString = 3, + } + + enum RoleCommandReplicaFormat : byte + { + RoleType = 0, + RoleAddress = 1, + RolePort = 2, + RoleState = 3, + RoleReplicationOffset = 4, + RoleReplicationOffsetString = 5 + } + public static class EndpointExtensions { public static IPEndPoint ToIPEndPoint(this EndPoint endPoint) @@ -420,7 +438,7 @@ private async Task WaitForSyncAsync(ClientClusterConfig clusterConfig) var slots = new List(); // Assign slots to primaries - for (int i = 0; i < slotRanges.Length; i++) + for (var i = 0; i < slotRanges.Length; i++) { foreach (var slotRange in slotRanges[i]) { @@ -462,7 +480,6 @@ private async Task WaitForSyncAsync(ClientClusterConfig clusterConfig) await SetConfigEpochAsync(endpoints[i], i + 1, logger).ConfigureAwait(false); // Initiate meets - var sendMeetTo = endpoints[0]; for (var newNode = 1; newNode < endpoints.Length; newNode++) { @@ -834,7 +851,7 @@ public async Task GetNodeIdsAsync(List nodes = null, ILogger logg public async Task ReconnectAsync(List nodes = null, TextWriter textWriter = null, ILogger logger = null) { await CloseConnectionsAsync().ConfigureAwait(false); - EndPointCollection endPoints = endpoints; + var endPoints = endpoints; if (nodes != null) { endPoints = new EndPointCollection(); @@ -1671,6 +1688,10 @@ public async Task> GetOwnedSlotsFromNodeAsync(IPEndPoint endPoint, ILo { for (int i = (int)ClusterInfoTag.SLOT; i < nodeInfo.Length; i++) { + // Skip migration/import markers like [slot->-nodeId] and [slot-<-nodeId] + if (nodeInfo[i].StartsWith('[')) + continue; + var range = nodeInfo[i].Split('-'); if (!ushort.TryParse(range[0], out var slotStart)) Assert.Fail($"GetOwnedSlotsFromNode: {range[0]}"); @@ -2035,19 +2056,47 @@ public int MigrateTasks(IPEndPoint endPoint, ILogger logger) } } - public void WaitForMigrationCleanup(int nodeIndex, ILogger logger = null, CancellationToken cancellationToken = default) - => WaitForMigrationCleanup(endpoints[nodeIndex].ToIPEndPoint(), logger, cancellationToken); + public void WaitForMigrationCleanup(int nodeIndex, CancellationToken cancellationToken = default, ILogger logger = null) + => WaitForMigrationCleanup(endpoints[nodeIndex].ToIPEndPoint(), cancellationToken, logger); - public void WaitForMigrationCleanup(IPEndPoint endPoint, ILogger logger, CancellationToken cancellationToken = default) + public void WaitForMigrationCleanup(IPEndPoint endPoint, CancellationToken cancellationToken = default, ILogger logger = null) { var backoffToken = cancellationToken.CanBeCanceled ? cancellationToken : context.cts.Token; while (MigrateTasks(endPoint, logger) > 0) { BackOff(cancellationToken: backoffToken); } } - public void WaitForMigrationCleanup(ILogger logger) + public void WaitForMigrationCleanup(CancellationToken cancellationToken = default, ILogger logger = null) { foreach (var endPoint in endpoints) - WaitForMigrationCleanup((IPEndPoint)endPoint, logger); + WaitForMigrationCleanup((IPEndPoint)endPoint, cancellationToken, logger); + } + + public void WaitForSlotOwnership(int nodeIndex, string expectedOwnerId, List ranges, ILogger logger = null) + => WaitForSlotOwnership(endpoints[nodeIndex].ToIPEndPoint(), expectedOwnerId, ranges, logger); + + public void WaitForSlotOwnership(IPEndPoint endPoint, string expectedOwnerId, List ranges, ILogger logger = null) + { + ClassicAssert.IsTrue((ranges.Count & 1) == 0, "Ranges should come in pairs!"); + var server = redis.GetServer(endPoint); + + while (true) + { + retry: + BackOff(cancellationToken: context.cts.Token, msg: ""); + var config = server.ClusterNodes(); + for (var i = 0; i < ranges.Count; i += 2) + { + var from = ranges[i]; + var to = ranges[i + 1]; + for (var j = from; j <= to; j++) + { + var node = config.GetBySlot(j); + if (node == null || node.NodeId == null || !node.NodeId.Equals(expectedOwnerId)) + goto retry; + } + } + break; + } } public static void Asking(ref LightClientRequest sourceNode) @@ -2789,76 +2838,58 @@ public List Smembers(int nodeIndex, string key, ILogger logger = null) } } - public long GetStoreCurrentAofAddress(int nodeIndex, ILogger logger = null) + public AofAddress GetStoreCurrentAofAddress(int nodeIndex, ILogger logger = null) => GetStoreCurrentAofAddress((IPEndPoint)endpoints[nodeIndex], logger); - public long GetStoreCurrentAofAddress(IPEndPoint endPoint, ILogger logger = null) + public AofAddress GetStoreCurrentAofAddress(IPEndPoint endPoint, ILogger logger = null) { try { var storeCurrentSafeAofAddress = GetReplicationInfo(endPoint, [ReplicationInfoItem.STORE_CURRENT_SAFE_AOF_ADDRESS], logger)[0].Item2; - return long.Parse(storeCurrentSafeAofAddress); + return AofAddress.FromString(storeCurrentSafeAofAddress); } catch (Exception ex) { logger?.LogError(ex, "An error has occurred; GetStoreCurrentAofAddress"); Assert.Fail(ex.Message); - return 0; + return default; } } - public long GetStoreRecoveredAofAddress(int nodeIndex, ILogger logger = null) + public AofAddress GetStoreRecoveredAofAddress(int nodeIndex, ILogger logger = null) => GetStoreRecoveredAofAddress((IPEndPoint)endpoints[nodeIndex], logger); - public long GetStoreRecoveredAofAddress(IPEndPoint endPoint, ILogger logger = null) + public AofAddress GetStoreRecoveredAofAddress(IPEndPoint endPoint, ILogger logger = null) { try { var storeRecoveredSafeAofAddress = GetReplicationInfo(endPoint, [ReplicationInfoItem.STORE_RECOVERED_SAFE_AOF_ADDRESS], logger)[0].Item2; - return long.Parse(storeRecoveredSafeAofAddress); + return AofAddress.FromString(storeRecoveredSafeAofAddress); } catch (Exception ex) { logger?.LogError(ex, "An error has occured; GetStoreRecoveredAofAddress"); Assert.Fail(ex.Message); - return 0; + return default; } } - public long GetObjectStoreCurrentAofAddress(int nodeIndex, ILogger logger = null) - => GetObjectStoreCurrentAofAddress((IPEndPoint)endpoints[nodeIndex], logger); - - public long GetObjectStoreCurrentAofAddress(IPEndPoint endPoint, ILogger logger = null) + /// + /// Blocks execution until the specified number of replicas are connected to the given node. + /// + /// The zero-based index of the node to check for connected replicas. + /// The number of replicas that must be connected before the method returns. Must be non-negative. + /// An optional logger used to record diagnostic information during the wait operation. May be null. + public void WaitForReplicasConnected(int nodeIndex, int replicaCount, ILogger logger = null) { - try + // Ensure that replicas have connected before completing the test + var count = context.clusterTestUtils.GetConnectedReplicas(nodeIndex, logger: logger); + while (count != replicaCount) { - var objectStoreCurrentSafeAofAddress = GetReplicationInfo(endPoint, [ReplicationInfoItem.OBJECT_STORE_CURRENT_SAFE_AOF_ADDRESS], logger)[0].Item2; - return long.Parse(objectStoreCurrentSafeAofAddress); - } - catch (Exception ex) - { - logger?.LogError(ex, "An error has occured; GetObjectStoreCurrentAofAddress"); - Assert.Fail(ex.Message); - return 0; - } - } - - public long GetObjectStoreRecoveredAofAddress(int nodeIndex, ILogger logger = null) - => GetObjectStoreRecoveredAofAddress((IPEndPoint)endpoints[nodeIndex], logger); - - public long GetObjectStoreRecoveredAofAddress(IPEndPoint endPoint, ILogger logger = null) - { - try - { - var objectStoreRecoveredSafeAofAddress = GetReplicationInfo(endPoint, [ReplicationInfoItem.OBJECT_STORE_RECOVERED_SAFE_AOF_ADDRESS], logger)[0].Item2; - return long.Parse(objectStoreRecoveredSafeAofAddress); - } - catch (Exception ex) - { - logger?.LogError(ex, "An error has occurred; GetObjectStoreRecoveredAofAddress"); - Assert.Fail(ex.Message); - return 0; + BackOff(); + count = context.clusterTestUtils.GetConnectedReplicas(nodeIndex, logger: logger); } + ClassicAssert.AreEqual(replicaCount, count); } public long GetConnectedReplicas(int nodeIndex, ILogger logger = null) @@ -2917,21 +2948,21 @@ public async Task GetReplicationRoleAsync(IPEndPoint endPoint, ILogger l } } - public long GetReplicationOffset(int nodeIndex, ILogger logger = null) + public AofAddress GetReplicationOffset(int nodeIndex, ILogger logger = null) => GetReplicationOffset((IPEndPoint)endpoints[nodeIndex], logger); - public long GetReplicationOffset(IPEndPoint endPoint, ILogger logger = null) + public AofAddress GetReplicationOffset(IPEndPoint endPoint, ILogger logger = null) { try { var offset = GetReplicationInfo(endPoint, [ReplicationInfoItem.REPLICATION_OFFSET], logger)[0].Item2; - return long.Parse(offset); + return AofAddress.FromString(offset); } catch (Exception ex) { logger?.LogError(ex, "An error has occurred; GetReplicationOffset"); Assert.Fail(ex.Message); - return 0; + return AofAddress.Create(1, -1); } } @@ -3040,27 +3071,27 @@ PersistencInfo ProcessPersistenceInfo(string infoSection) { if (item.StartsWith("CommittedBeginAddress:")) { - pinfo.CommittedBeginAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.CommittedBeginAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } else if (item.StartsWith("CommittedUntilAddress:")) { - pinfo.CommittedUntilAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.CommittedUntilAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } else if (item.StartsWith("FlushedUntilAddress:")) { - pinfo.FlushedUntilAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.FlushedUntilAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } else if (item.StartsWith("BeginAddress:")) { - pinfo.BeginAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.BeginAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } else if (item.StartsWith("TailAddress:")) { - pinfo.TailAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.TailAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } else if (item.StartsWith("SafeAofAddress:")) { - pinfo.SafeAofAddress = long.Parse(item.Split(":")[1].Trim()); + pinfo.SafeAofAddress = AofAddress.FromString(item.Split(":")[1].Trim()); } } return pinfo; @@ -3086,6 +3117,10 @@ PersistencInfo ProcessPersistenceInfo(string infoSection) startsWith = "connected_slaves:"; if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); continue; + case ReplicationInfoItem.SYNC_DRIVER_COUNT: + startsWith = "sync_driver_count:"; + if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); + continue; case ReplicationInfoItem.PRIMARY_REPLID: startsWith = "master_replid:"; if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); @@ -3102,14 +3137,6 @@ PersistencInfo ProcessPersistenceInfo(string infoSection) startsWith = "store_recovered_safe_aof_address:"; if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); continue; - case ReplicationInfoItem.OBJECT_STORE_CURRENT_SAFE_AOF_ADDRESS: - startsWith = "object_store_current_safe_aof_address:"; - if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); - continue; - case ReplicationInfoItem.OBJECT_STORE_RECOVERED_SAFE_AOF_ADDRESS: - startsWith = "object_store_recovered_safe_aof_address:"; - if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); - continue; case ReplicationInfoItem.PRIMARY_SYNC_IN_PROGRESS: startsWith = "master_sync_in_progress:"; if (item.StartsWith(startsWith)) items.Add((ii, item.Split(startsWith)[1].Trim())); @@ -3160,7 +3187,7 @@ public int GetStoreCurrentVersion(int nodeIndex, bool isMainStore = true, ILogge try { var server = redis.GetServer(endPoint); - var result = server.InfoRawAsync(isMainStore ? "store" : "objectstore").Result; + var result = server.InfoRawAsync("store").Result; var data = result.Split('\n'); foreach (var line in data) { @@ -3208,17 +3235,35 @@ public string GetInfo(IPEndPoint endPoint, string section, string segment, ILogg } } + public void WaitForAofSyncDriverDipose(int primaryNodeIndex) + { + var items = context.clusterTestUtils.GetReplicationInfo( + primaryNodeIndex, + [ReplicationInfoItem.CONNECTED_REPLICAS, ReplicationInfoItem.SYNC_DRIVER_COUNT], + context.logger); + while (!items[0].Item2.Equals("0") || !items[1].Item2.Equals("0")) + { + items = context.clusterTestUtils.GetReplicationInfo( + primaryNodeIndex, + [ReplicationInfoItem.CONNECTED_REPLICAS, ReplicationInfoItem.SYNC_DRIVER_COUNT], + context.logger); + if (context.cts.Token.IsCancellationRequested) + Assert.Fail($"Failed waiting for primary aof sync cleanup ({items[0]};{items[1]})!"); + BackOff(cancellationToken: context.cts.Token); + } + } + public void WaitForReplicaAofSync(int primaryIndex, int secondaryIndex, ILogger logger = null, CancellationToken cancellation = default) { - long primaryReplicationOffset; - long secondaryReplicationOffset1; + AofAddress primaryReplicationOffset; + AofAddress secondaryReplicationOffset1; while (true) { cancellation.ThrowIfCancellationRequested(); primaryReplicationOffset = GetReplicationOffset(primaryIndex, logger); secondaryReplicationOffset1 = GetReplicationOffset(secondaryIndex, logger); - if (primaryReplicationOffset == secondaryReplicationOffset1) + if (primaryReplicationOffset.Equals(secondaryReplicationOffset1)) break; var primaryMainStoreVersion = context.clusterTestUtils.GetStoreCurrentVersion(primaryIndex, logger: logger); @@ -3311,26 +3356,32 @@ public void Checkpoint(int nodeIndex, ILogger logger = null) public void Checkpoint(IPEndPoint endPoint, ILogger logger = null) { + const int maxRetries = 10; var server = redis.GetServer(endPoint); - try + for (var attempt = 0; ; attempt++) { - var previousSaveTicks = (long)server.Execute("LASTSAVE"); + try + { #pragma warning disable CS0618 // Type or member is obsolete - server.Save(SaveType.ForegroundSave); + server.Save(SaveType.ForegroundSave); #pragma warning restore CS0618 // Type or member is obsolete + break; + } + catch (RedisServerException ex) when (ex.Message.Contains("checkpoint already in progress", StringComparison.OrdinalIgnoreCase)) + { + if (attempt >= maxRetries) + Assert.Fail($"Checkpoint still in progress after {maxRetries} retries"); - //// Spin wait for checkpoint to complete - //while (true) - //{ - // var lastSaveTicks = (long)server.Execute("LASTSAVE"); - // if (previousSaveTicks < lastSaveTicks) break; - // BackOff(TimeSpan.FromSeconds(1)); - //} - } - catch (Exception ex) - { - logger?.LogError(ex, "An error has occurred; StoreWrapper.Checkpoint"); - Assert.Fail(); + // Another checkpoint is in progress (e.g., on-demand checkpoint from replication). + // Retry after a short delay. + logger?.LogWarning(ex, "Checkpoint already in progress, retrying (attempt {attempt})", attempt); + BackOff(cancellationToken: context?.cts?.Token ?? CancellationToken.None); + } + catch (Exception ex) + { + logger?.LogError(ex, "An error has occurred; StoreWrapper.Checkpoint"); + Assert.Fail(ex.Message); + } } } diff --git a/test/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs b/test/cluster/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs similarity index 66% rename from test/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs rename to test/cluster/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs index 5f71e94ae99..d8b734d4100 100644 --- a/test/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs +++ b/test/cluster/Garnet.test.cluster/CustomProcs/ClusterDelRmw.cs @@ -9,20 +9,18 @@ namespace Garnet.test.cluster { - sealed class ClusterDelRmw : CustomTransactionProcedure + public sealed class ClusterDelRmw : CustomTransactionProcedure { public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, isObject: false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); return true; } public override unsafe void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) { var offset = 0; - var timeStamp = DateTime.Now.Ticks; - var unixTimeInMilliSecond = timeStamp / TimeSpan.TicksPerMillisecond; var key = GetNextArg(ref procInput, ref offset); var value = GetNextArg(ref procInput, ref offset); @@ -31,15 +29,16 @@ public override unsafe void Main(TGarnetApi api, ref CustomProcedure var status = api.DELETE(key); Debug.Assert(status == GarnetStatus.OK); - var parsed = ParseUtils.TryReadLong(ref value, out var valueToIncrement); + var parsed = ParseUtils.TryReadLong(value, out var valueToIncrement); Debug.Assert(parsed, "Value to increment must be a valid long integer."); - var input = new RawStringInput(RespCommand.INCRBY, 0, valueToIncrement); + var input = new StringInput(RespCommand.INCRBY, 0, valueToIncrement); Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length + 1]; - var outputArgSlice = ArgSlice.FromPinnedSpan(outputBuffer); + var stringOutput = StringOutput.FromPinnedSpan(outputBuffer); + // Increment key - status = api.Increment(key, ref input, ref outputArgSlice); - Debug.Assert(status == GarnetStatus.OK); + _ = api.Increment(key, ref input, ref stringOutput); + Debug.Assert(!stringOutput.HasError); WriteSimpleString(ref output, "OK"); } diff --git a/test/cluster/Garnet.test.cluster/Garnet.test.cluster.csproj b/test/cluster/Garnet.test.cluster/Garnet.test.cluster.csproj new file mode 100644 index 00000000000..3b2445756c6 --- /dev/null +++ b/test/cluster/Garnet.test.cluster/Garnet.test.cluster.csproj @@ -0,0 +1,68 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + PreserveNewest + + + + + false + + diff --git a/test/Garnet.test.cluster/JwtTokenHelpers.cs b/test/cluster/Garnet.test.cluster/JwtTokenHelpers.cs similarity index 100% rename from test/Garnet.test.cluster/JwtTokenHelpers.cs rename to test/cluster/Garnet.test.cluster/JwtTokenHelpers.cs diff --git a/test/Garnet.test.cluster/packages.config b/test/cluster/Garnet.test.cluster/packages.config similarity index 100% rename from test/Garnet.test.cluster/packages.config rename to test/cluster/Garnet.test.cluster/packages.config diff --git a/test/cluster/garnet-cluster.runsettings b/test/cluster/garnet-cluster.runsettings new file mode 100644 index 00000000000..c8703b53003 --- /dev/null +++ b/test/cluster/garnet-cluster.runsettings @@ -0,0 +1,7 @@ + + + + + 0 + + diff --git a/test/standalone/Directory.Build.props b/test/standalone/Directory.Build.props new file mode 100644 index 00000000000..be2ee3bdc8c --- /dev/null +++ b/test/standalone/Directory.Build.props @@ -0,0 +1,6 @@ + + + + $(MSBuildThisFileDirectory)garnet.runsettings + + diff --git a/test/standalone/Garnet.test.acl/Garnet.test.acl.csproj b/test/standalone/Garnet.test.acl/Garnet.test.acl.csproj new file mode 100644 index 00000000000..340c2ffce68 --- /dev/null +++ b/test/standalone/Garnet.test.acl/Garnet.test.acl.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/AclConfigurationFileTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/AclConfigurationFileTests.cs index ed4d8f05864..9ba5d990d95 100644 --- a/test/Garnet.test/Resp/ACL/AclConfigurationFileTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/AclConfigurationFileTests.cs @@ -5,7 +5,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server.ACL; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -16,7 +15,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for ACL Configuration file related operations. /// - [AllureNUnit] [TestFixture] class AclConfigurationFileTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/AclParserTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/AclParserTests.cs similarity index 98% rename from test/Garnet.test/Resp/ACL/AclParserTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/AclParserTests.cs index 5c6a1b0b435..83e4aaa214a 100644 --- a/test/Garnet.test/Resp/ACL/AclParserTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/AclParserTests.cs @@ -1,7 +1,6 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using Garnet.server.ACL; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -11,7 +10,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for the . /// - [AllureNUnit] [TestFixture] internal class AclParserTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/AclTest.cs b/test/standalone/Garnet.test.acl/Resp/ACL/AclTest.cs similarity index 95% rename from test/Garnet.test/Resp/ACL/AclTest.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/AclTest.cs index b5552062227..0e92a88ce8b 100644 --- a/test/Garnet.test/Resp/ACL/AclTest.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/AclTest.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; @@ -9,7 +9,7 @@ namespace Garnet.test.Resp.ACL /// /// Base class used for all RESP ACL tests /// - abstract class AclTest : AllureTestBase + abstract class AclTest : TestBase { /// /// Dummy password used by some of the tests. diff --git a/test/Garnet.test/Resp/ACL/BasicTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/BasicTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/BasicTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/BasicTests.cs index 1f63edd4339..4ff7a29b1b8 100644 --- a/test/Garnet.test/Resp/ACL/BasicTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/BasicTests.cs @@ -3,7 +3,6 @@ using System; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,7 +12,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for Resp ACL commands that don't have subcommands /// - [AllureNUnit] [TestFixture] internal class BasicTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/DeleteUserTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/DeleteUserTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/DeleteUserTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/DeleteUserTests.cs index 44b04c0e144..6dd88f24d7a 100644 --- a/test/Garnet.test/Resp/ACL/DeleteUserTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/DeleteUserTests.cs @@ -3,7 +3,6 @@ using System; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,7 +11,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for ACL DELUSER operations. /// - [AllureNUnit] [TestFixture] class DeleteUserTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/GetUserTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/GetUserTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/GetUserTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/GetUserTests.cs index bf5be7ad705..62197bb3408 100644 --- a/test/Garnet.test/Resp/ACL/GetUserTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/GetUserTests.cs @@ -5,7 +5,6 @@ using System.IO; using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -15,7 +14,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for ACL GETUSER command. /// - [AllureNUnit] [TestFixture] internal class GetUserTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/ParallelTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/ParallelTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/ParallelTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/ParallelTests.cs index cecbfc50976..b011a2c36e1 100644 --- a/test/Garnet.test/Resp/ACL/ParallelTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/ParallelTests.cs @@ -4,7 +4,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.client; using Garnet.server.ACL; using NUnit.Framework; @@ -15,7 +14,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests that operate in parallel on the ACL /// - [AllureNUnit] [TestFixture] internal class ParallelTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/RespCommandTests.cs similarity index 96% rename from test/Garnet.test/Resp/ACL/RespCommandTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/RespCommandTests.cs index d2af98a3e62..d591517a1dd 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/RespCommandTests.cs @@ -10,7 +10,6 @@ using System.Reflection; using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.client; using Garnet.server; using Garnet.server.ACL; @@ -19,9 +18,8 @@ namespace Garnet.test.Resp.ACL { - [AllureNUnit] [TestFixture] - public class RespCommandTests : AllureTestBase + public class RespCommandTests : TestBase { private const string DefaultPassword = nameof(RespCommandTests); private const string DefaultUser = "default"; @@ -37,7 +35,8 @@ public void Setup() TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, defaultPassword: DefaultPassword, useAcl: true, enableLua: true, - enableModuleCommand: Garnet.server.Auth.Settings.ConnectionProtectionOption.Yes); + enableModuleCommand: Garnet.server.Auth.Settings.ConnectionProtectionOption.Yes, + enableRangeIndexPreview: true); // Register custom commands so we can test ACL'ing them ClassicAssert.IsTrue(TestUtils.TryGetCustomCommandsInfo(out respCustomCommandsInfo)); @@ -99,7 +98,7 @@ public void AllCommandsCovered() // Exclude things like ACL, CLIENT, CLUSTER which are "commands" but only their sub commands can be run IEnumerable subCommands = allInfo.Where(static x => x.Value.SubCommands != null).SelectMany(static x => x.Value.SubCommands).Select(static x => x.Name); var x = advertisedCommands.Except(withOnlySubCommands).Union(subCommands); - IEnumerable deSubCommanded = advertisedCommands.Except(withOnlySubCommands).Union(subCommands).Select(static x => x.Replace("|", "").Replace("_", "").Replace("-", "")); + IEnumerable deSubCommanded = advertisedCommands.Except(withOnlySubCommands).Union(subCommands).Select(static x => x.Replace("|", "").Replace("_", "").Replace("-", "").Replace(".", "")); IEnumerable notCovered = deSubCommanded.Except(covered, StringComparer.OrdinalIgnoreCase).Except(notCoveredByACLs, StringComparer.OrdinalIgnoreCase); ClassicAssert.IsEmpty(notCovered, $"Commands in RespCommandsInfo not covered by ACL Tests:{Environment.NewLine}{string.Join(Environment.NewLine, notCovered.OrderBy(static x => x))}"); @@ -110,7 +109,7 @@ public void AllCommandsCovered() IEnumerable allValues = Enum.GetValues().Select(static x => x.NormalizeForACLs()).Distinct(); IEnumerable testableValues = allValues - .Except([RespCommand.NONE, RespCommand.INVALID, RespCommand.DELIFEXPIM]) + .Except([RespCommand.NONE, RespCommand.INVALID, RespCommand.DELIFEXPIM, RespCommand.RIPROMOTE, RespCommand.RIRESTORE]) .Where(cmd => !withOnlySubCommands.Contains(cmd.ToString().Replace("_", ""), StringComparer.OrdinalIgnoreCase)) .Where(cmd => !notCoveredByACLs.Contains(cmd.ToString().Replace("_", ""), StringComparer.OrdinalIgnoreCase)); IEnumerable notCovered = testableValues.Where(cmd => !covered.Contains(cmd.ToString().Replace("_", ""), StringComparer.OrdinalIgnoreCase)); @@ -999,35 +998,6 @@ static async Task DoClusterAddSlotsRangeMultiAsync(GarnetClient client) } } - [Test] - public async Task ClusterAofSyncACLsAsync() - { - // All cluster command "success" is a thrown exception, because clustering is disabled - - await CheckCommandsAsync( - "CLUSTER AOFSYNC", - [DoClusterAofSyncAsync] - ).ConfigureAwait(false); - - static async Task DoClusterAofSyncAsync(GarnetClient client) - { - try - { - await client.ExecuteForStringResultAsync("CLUSTER", ["AOFSYNC", "abc", "def"]).ConfigureAwait(false); - Assert.Fail("Shouldn't be reachable, cluster isn't enabled"); - } - catch (Exception e) - { - if (e.Message == "ERR This instance has cluster support disabled") - { - return; - } - - throw; - } - } - } - [Test] public async Task ClusterAppendLogACLsAsync() { @@ -1891,6 +1861,35 @@ static async Task DoClusterMTasksAsync(GarnetClient client) } } + [Test] + public async Task ClusterAdvanceTimeACLsAsync() + { + // All cluster command "success" is a thrown exception, because clustering is disabled + + await CheckCommandsAsync( + "CLUSTER ADVANCE_TIME", + [DoClusterAdvanceTimeAsync] + ); + + static async Task DoClusterAdvanceTimeAsync(GarnetClient client) + { + try + { + await client.ExecuteForStringResultAsync("CLUSTER", ["ADVANCE_TIME"]); + Assert.Fail("Shouldn't be reachable, cluster isn't enabled"); + } + catch (Exception e) + { + if (e.Message == "ERR This instance has cluster support disabled") + { + return; + } + + throw; + } + } + } + [Test] public async Task ClusterMyIdACLsAsync() { @@ -2170,6 +2169,35 @@ static async Task DoClusterSendCkptMetadataAsync(GarnetClient client) } } + [Test] + public async Task ClusterSnapshotDataACLsAsync() + { + // All cluster command "success" is a thrown exception, because clustering is disabled + + await CheckCommandsAsync( + "CLUSTER SNAPSHOT_DATA", + [DoClusterSnapshotDataAsync] + ).ConfigureAwait(false); + + static async Task DoClusterSnapshotDataAsync(GarnetClient client) + { + try + { + await client.ExecuteForStringResultAsync("CLUSTER", ["SNAPSHOT_DATA", "1", "2", "3", "4"]).ConfigureAwait(false); + Assert.Fail("Shouldn't be reachable, cluster isn't enabled"); + } + catch (Exception e) + { + if (e.Message == "ERR This instance has cluster support disabled") + { + return; + } + + throw; + } + } + } + [Test] public async Task ClusterSetConfigEpochACLsAsync() { @@ -2434,6 +2462,53 @@ static async Task DoClusterSlotStateAsync(GarnetClient client) } } + [Test] + public async Task ClusterMlogKeyTimeACLsAsync() + { + // All cluster command "success" is a thrown exception, because clustering is disabled + + await CheckCommandsAsync( + "CLUSTER MLOG_KEY_TIME", + [DoClusterMlogKeyTimeAsync, DoClusterMlogKeyTimeFrontierAsync] + ); + + static async Task DoClusterMlogKeyTimeAsync(GarnetClient client) + { + try + { + await client.ExecuteForStringResultAsync("CLUSTER", ["MLOG_KEY_TIME", "key"]); + Assert.Fail("Shouldn't be reachable, cluster isn't enabled"); + } + catch (Exception e) + { + if (e.Message == "ERR This instance has cluster support disabled") + { + return; + } + + throw; + } + } + + static async Task DoClusterMlogKeyTimeFrontierAsync(GarnetClient client) + { + try + { + await client.ExecuteForStringResultAsync("CLUSTER", ["MLOG_KEY_TIME", "key", "FRONTIER"]); + Assert.Fail("Shouldn't be reachable, cluster isn't enabled"); + } + catch (Exception e) + { + if (e.Message == "ERR This instance has cluster support disabled") + { + return; + } + + throw; + } + } + } + [Test] public async Task ClusterPublishACLsAsync() { @@ -5381,6 +5456,189 @@ static async Task DoRENAMENXAsync(GarnetClient client) } } + [Test] + public async Task RICreateACLsAsync() + { + int count = 0; + + await CheckCommandsAsync( + "RI.CREATE", + [DoRICreateAsync] + ).ConfigureAwait(false); + + async Task DoRICreateAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("RI.CREATE", [$"myindex-{count}", "MEMORY", "CACHESIZE", "65536"]).ConfigureAwait(false); + count++; + ClassicAssert.AreEqual("OK", val); + } + } + + [Test] + public async Task RIDelACLsAsync() + { + // Pre-create the index using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["ridel-acl-idx", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"]).ConfigureAwait(false); + + int count = 0; + + await CheckCommandsAsync( + "RI.DEL", + [DoRIDelAsync] + ).ConfigureAwait(false); + + async Task DoRIDelAsync(GarnetClient client) + { + // Insert a field as default user, then delete it as the test user + await setupClient.ExecuteForStringResultAsync("RI.SET", ["ridel-acl-idx", $"field-{count}", "val"]).ConfigureAwait(false); + var val = await client.ExecuteForStringResultAsync("RI.DEL", ["ridel-acl-idx", $"field-{count}"]).ConfigureAwait(false); + count++; + ClassicAssert.AreEqual("1", val); + } + } + + [Test] + public async Task RIGetACLsAsync() + { + // Pre-create the index and insert a field using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["riget-acl-idx", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"]).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.SET", ["riget-acl-idx", "field1", "value1"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.GET", + [DoRIGetAsync] + ).ConfigureAwait(false); + + static async Task DoRIGetAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("RI.GET", ["riget-acl-idx", "field1"]).ConfigureAwait(false); + ClassicAssert.AreEqual("value1", val); + } + } + + [Test] + public async Task RISetACLsAsync() + { + // Pre-create the index using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["riset-acl-idx", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.SET", + [DoRISetAsync] + ).ConfigureAwait(false); + + static async Task DoRISetAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("RI.SET", ["riset-acl-idx", "field1", "value1"]).ConfigureAwait(false); + ClassicAssert.AreEqual("OK", val); + } + } + + [Test] + public async Task RIRangeACLsAsync() + { + // Pre-create the index and insert fields using default user (DISK mode required for scan) + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["rirange-acl-idx", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"]).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.SET", ["rirange-acl-idx", "aaa", "val-a"]).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.SET", ["rirange-acl-idx", "bbb", "val-b"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.RANGE", + [DoRIRangeAsync] + ).ConfigureAwait(false); + + static async Task DoRIRangeAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("RI.RANGE", ["rirange-acl-idx", "aaa", "bbb", "FIELDS", "KEY"]).ConfigureAwait(false); + ClassicAssert.IsNotNull(val); + ClassicAssert.AreEqual(2, val.Length); + } + } + + [Test] + public async Task RIScanACLsAsync() + { + // Pre-create the index and insert a field using default user (DISK mode required for scan) + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["riscan-acl-idx", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"]).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.SET", ["riscan-acl-idx", "aaa", "val-a"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.SCAN", + [DoRIScanAsync] + ).ConfigureAwait(false); + + static async Task DoRIScanAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("RI.SCAN", ["riscan-acl-idx", "aaa", "COUNT", "10", "FIELDS", "KEY"]).ConfigureAwait(false); + ClassicAssert.IsNotNull(val); + ClassicAssert.IsTrue(val.Length >= 1); + } + } + + [Test] + public async Task RIExistsACLsAsync() + { + // Pre-create the index using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["riexists-acl-idx", "MEMORY", "CACHESIZE", "65536"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.EXISTS", + [DoRIExistsAsync] + ).ConfigureAwait(false); + + static async Task DoRIExistsAsync(GarnetClient client) + { + var val = await client.ExecuteForStringResultAsync("RI.EXISTS", ["riexists-acl-idx"]).ConfigureAwait(false); + ClassicAssert.AreEqual("1", val); + } + } + + [Test] + public async Task RIConfigACLsAsync() + { + // Pre-create the index using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["riconfig-acl-idx", "MEMORY", "CACHESIZE", "65536"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.CONFIG", + [DoRIConfigAsync] + ).ConfigureAwait(false); + + static async Task DoRIConfigAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("RI.CONFIG", ["riconfig-acl-idx"]).ConfigureAwait(false); + ClassicAssert.IsNotNull(val); + ClassicAssert.AreEqual(12, val.Length); + } + } + + [Test] + public async Task RIMetricsACLsAsync() + { + // Pre-create the index using default user + using var setupClient = await CreateGarnetClientAsync(DefaultUser, DefaultPassword).ConfigureAwait(false); + await setupClient.ExecuteForStringResultAsync("RI.CREATE", ["rimetrics-acl-idx", "MEMORY", "CACHESIZE", "65536"]).ConfigureAwait(false); + + await CheckCommandsAsync( + "RI.METRICS", + [DoRIMetricsAsync] + ).ConfigureAwait(false); + + static async Task DoRIMetricsAsync(GarnetClient client) + { + var val = await client.ExecuteForStringArrayResultAsync("RI.METRICS", ["rimetrics-acl-idx"]).ConfigureAwait(false); + ClassicAssert.IsNotNull(val); + ClassicAssert.AreEqual(8, val.Length); + } + } + [Test] public async Task ReplicaOfACLsAsync() { @@ -5665,6 +5923,24 @@ static async Task DoSetIfGreaterAsync(GarnetClient client) } } + [Test] + public async Task SetWithEtagACLsAsync() + { + int count = 0; + + await CheckCommandsAsync( + "SETWITHETAG", + [DoSetWithEtagAsync] + ).ConfigureAwait(false); + + async Task DoSetWithEtagAsync(GarnetClient client) + { + var res = await client.ExecuteForStringResultAsync("SETWITHETAG", [$"key-{count}", "value"]).ConfigureAwait(false); + count++; + ClassicAssert.IsNotNull(res); + } + } + [Test] public async Task DelIfGreaterACLsAsync() { @@ -7528,7 +7804,7 @@ static async Task DoVAddAsync(GarnetClient client) { var elem = Encoding.ASCII.GetString("\x0\x1\x2\x3"u8); - long val = await client.ExecuteForLongResultAsync("VADD", ["foo", "REDUCE", "50", "VALUES", "4", "1.0", "2.0", "3.0", "4.0", elem, "CAS", "NOQUANT", "EF", "16", "SETATTR", "{ 'hello': 'world' }", "M", "32"]).ConfigureAwait(false); + long val = await client.ExecuteForLongResultAsync("VADD", ["foo", "REDUCE", "2", "VALUES", "4", "1.0", "2.0", "3.0", "4.0", elem, "CAS", "NOQUANT", "EF", "16", "SETATTR", "{ 'hello': 'world' }", "M", "32"]).ConfigureAwait(false); ClassicAssert.AreEqual(1, val); } } diff --git a/test/Garnet.test/Resp/ACL/SetUserTests.cs b/test/standalone/Garnet.test.acl/Resp/ACL/SetUserTests.cs similarity index 99% rename from test/Garnet.test/Resp/ACL/SetUserTests.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/SetUserTests.cs index 22f33a25ee0..3770859729a 100644 --- a/test/Garnet.test/Resp/ACL/SetUserTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/ACL/SetUserTests.cs @@ -4,7 +4,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,7 +12,6 @@ namespace Garnet.test.Resp.ACL /// /// Tests for ACL SETUSER operations. /// - [AllureNUnit] [TestFixture] class SetUserTests : AclTest { diff --git a/test/Garnet.test/Resp/ACL/UserAclResult.cs b/test/standalone/Garnet.test.acl/Resp/ACL/UserAclResult.cs similarity index 100% rename from test/Garnet.test/Resp/ACL/UserAclResult.cs rename to test/standalone/Garnet.test.acl/Resp/ACL/UserAclResult.cs diff --git a/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs b/test/standalone/Garnet.test.acl/Resp/GarnetAuthenticatorTests.cs similarity index 72% rename from test/Garnet.test/Resp/GarnetAuthenticatorTests.cs rename to test/standalone/Garnet.test.acl/Resp/GarnetAuthenticatorTests.cs index b8132167eaf..bda8d4f5a06 100644 --- a/test/Garnet.test/Resp/GarnetAuthenticatorTests.cs +++ b/test/standalone/Garnet.test.acl/Resp/GarnetAuthenticatorTests.cs @@ -2,9 +2,7 @@ // Licensed under the MIT license. using System; -using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using Garnet.server.Auth; using Garnet.server.Auth.Settings; @@ -16,9 +14,8 @@ namespace Garnet.test.Resp /// /// Tests generic to all s. /// - [AllureNUnit] [TestFixture] - public class GarnetAuthenticatorTests : AllureTestBase + public class GarnetAuthenticatorTests : TestBase { private delegate bool AuthenticateDelegate(ReadOnlySpan password, ReadOnlySpan username); @@ -60,25 +57,9 @@ public async Task InvalidatingAuthorizationAsync() auth.HasACLSupport = false; auth.IsAuthenticated = false; - var authCalls = 0; - var authingAsFoo = false; - var authedAsFoo = false; - auth.AuthenticateCallback = (p, u) => { - if (!authingAsFoo) - { - ClassicAssert.AreEqual("default", Encoding.UTF8.GetString(u)); - } - else - { - ClassicAssert.AreEqual("foo", Encoding.UTF8.GetString(u)); - authedAsFoo = true; - } - - authCalls++; - auth.IsAuthenticated = true; return true; }; @@ -87,21 +68,12 @@ public async Task InvalidatingAuthorizationAsync() server.Start(); using var c = TestUtils.GetGarnetClientSession(); - c.Connect(); - - // Initial command runs under default user - _ = await c.ExecuteAsync("PING").ConfigureAwait(false); - - // Auth as proper user, should get another call - authingAsFoo = true; - _ = await c.ExecuteAsync("AUTH", "foo", "bar").ConfigureAwait(false); - ClassicAssert.IsTrue(authedAsFoo); + await c.ConnectAsync().ConfigureAwait(false); + // Initial command should work _ = await c.ExecuteAsync("PING").ConfigureAwait(false); // Command after auth invalidation fails as no auth - - var oldAuthCalls = authCalls; auth.IsAuthenticated = false; try { @@ -113,8 +85,11 @@ public async Task InvalidatingAuthorizationAsync() ClassicAssert.AreEqual("NOAUTH Authentication required.", e.Message); } - _ = await c.ExecuteAsync("AUTH", "foo", "bar").ConfigureAwait(false); - ClassicAssert.True(authCalls > oldAuthCalls); + // Re-auth + _ = await c.ExecuteAsync("AUTH", "bar").ConfigureAwait(false); + + // Should be authed again + _ = await c.ExecuteAsync("PING").ConfigureAwait(false); } } } \ No newline at end of file diff --git a/test/standalone/Garnet.test.acl/TestProjectSetup.cs b/test/standalone/Garnet.test.acl/TestProjectSetup.cs new file mode 100644 index 00000000000..9f4a4893453 --- /dev/null +++ b/test/standalone/Garnet.test.acl/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestAcl); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.collections/Garnet.test.collections.csproj b/test/standalone/Garnet.test.collections/Garnet.test.collections.csproj new file mode 100644 index 00000000000..5fa45ed1d93 --- /dev/null +++ b/test/standalone/Garnet.test.collections/Garnet.test.collections.csproj @@ -0,0 +1,51 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + $(DefineConstants);TEST_PROJECT + + + + + + diff --git a/test/standalone/Garnet.test.collections/GarnetObjectTests.cs b/test/standalone/Garnet.test.collections/GarnetObjectTests.cs new file mode 100644 index 00000000000..01c243fb8ad --- /dev/null +++ b/test/standalone/Garnet.test.collections/GarnetObjectTests.cs @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Threading.Tasks; +using Garnet.common; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using Tsavorite.core; + +namespace Garnet.test +{ + [TestFixture] + public class GarnetObjectTests : TestBase + { + TsavoriteKV store; + IDevice logDevice, objectLogDevice; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + CreateStore(); + } + + [TearDown] + public void TearDown() + { + store.Dispose(); + logDevice.Dispose(); + objectLogDevice.Dispose(); + logDevice = objectLogDevice = null; + TestUtils.OnTearDown(); + } + + [Test] + public void WriteRead() + { + using var session = store.NewSession(new SimpleGarnetObjectSessionFunctions()); + var bContext = session.BasicContext; + + var key = new ReadOnlySpan([0]); + var obj = new SortedSetObject(); + + _ = bContext.Upsert((FixedSpanByteKey)key, obj); + + IGarnetObject output = null; + var status = bContext.Read((FixedSpanByteKey)key, ref output); + + ClassicAssert.IsTrue(status.Found); + ClassicAssert.AreEqual(obj, output); + } + + const int keyNum = 0; + + [Test] + public async Task WriteCheckpointRead() + { + var obj = new SortedSetObject(); + + LocalWrite(); + _ = await store.TakeHybridLogCheckpointAsync(CheckpointType.FoldOver).ConfigureAwait(false); + store.Dispose(); + CreateStore(); + _ = store.Recover(); + LocalRead(); + + void LocalWrite() + { + using var session = store.NewSession(new MyFunctions()); + var bContext = session.BasicContext; + + var key = new ReadOnlySpan([keyNum]); + obj.Add([15], 10); + + _ = bContext.Upsert((FixedSpanByteKey)key, obj); + } + + void LocalRead() + { + using var session = store.NewSession(new MyFunctions()); + var bContext = session.BasicContext; + + IGarnetObject output = null; + var key = new ReadOnlySpan([keyNum]); + var status = bContext.Read((FixedSpanByteKey)key, ref output); + + ClassicAssert.IsTrue(status.Found); + ClassicAssert.IsTrue(obj.Equals((SortedSetObject)output)); + } + } + + [Test] + public async Task WriteCheckpointCopyUpdate() + { + IGarnetObject obj = new SortedSetObject(); + + LocalWrite(); + _ = await store.TakeHybridLogCheckpointAsync(CheckpointType.FoldOver).ConfigureAwait(false); + store.Dispose(); + CreateStore(); + _ = store.Recover(); + LocalRead(); + + void LocalWrite() + { + using var session = store.NewSession(new MyFunctions()); + var bContext = session.BasicContext; + + var key = new ReadOnlySpan([keyNum]); + ((SortedSetObject)obj).Add([15], 10); + + _ = bContext.Upsert((FixedSpanByteKey)key, obj); + store.Log.Flush(true); + _ = bContext.RMW((FixedSpanByteKey)key, ref obj); + } + + void LocalRead() + { + using var session = store.NewSession(new MyFunctions()); + var bContext = session.BasicContext; + + IGarnetObject output = null; + var key = new ReadOnlySpan([keyNum]); + var status = bContext.Read((FixedSpanByteKey)key, ref output); + + ClassicAssert.IsTrue(status.Found); + ClassicAssert.IsTrue(((SortedSetObject)obj).Equals((SortedSetObject)output)); + } + } + + private class MyFunctions : SessionFunctionsBase + { + public MyFunctions() + { } + + public override bool Reader(in TSourceLogRecord srcLogRecord, ref IGarnetObject input, ref IGarnetObject output, ref ReadInfo readInfo) + { + output = (IGarnetObject)srcLogRecord.ValueObject; + return true; + } + + public override bool CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref IGarnetObject input, ref IGarnetObject output, ref RMWInfo rmwInfo) + { + Assert.That(dstLogRecord.Info.ValueIsObject, Is.True); + dstLogRecord.TrySetValueObject(srcLogRecord.ValueObject.Clone()); + return true; + } + + public override RecordFieldInfo GetRMWModifiedFieldInfo(in TSourceLogRecord srcLogRecord, ref IGarnetObject input) + => new() { KeySize = srcLogRecord.Key.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetRMWInitialFieldInfo(TKey key, ref IGarnetObject input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, ReadOnlySpan value, ref IGarnetObject input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = value.Length, ValueIsObject = false }; + public override RecordFieldInfo GetUpsertFieldInfo(TKey key, IHeapObject value, ref IGarnetObject input) + => new() { KeySize = key.KeyBytes.Length, ValueSize = ObjectIdMap.ObjectIdSize, ValueIsObject = true }; + } + + private void CreateStore() + { + logDevice ??= Devices.CreateLogDevice(TestUtils.MethodTestDir + "/hlog.log"); + objectLogDevice ??= Devices.CreateLogDevice(TestUtils.MethodTestDir + "/hlog.obj.log"); + + var kvSettings = new KVSettings + { + IndexSize = 1L << 13, + LogDevice = logDevice, + ObjectLogDevice = objectLogDevice, + CheckpointDir = TestUtils.MethodTestDir + }; + + store = new(kvSettings + , Tsavorite.core.StoreFunctions.Create(new GarnetKeyComparer(), () => new MyGarnetObjectSerializer(), + new GarnetRecordTriggers()) + , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); + } + } + + /// + /// Serializer for IGarnetObject + /// + sealed class MyGarnetObjectSerializer : BinaryObjectSerializer + { + /// + public override void Deserialize(out IHeapObject obj) + { + var type = (GarnetObjectType)reader.ReadByte(); + obj = type switch + { + GarnetObjectType.SortedSet => new SortedSetObject(reader), + GarnetObjectType.List => new ListObject(reader), + GarnetObjectType.Hash => new HashObject(reader), + GarnetObjectType.Set => new SetObject(reader), + _ => null, + }; + } + + /// + public override void Serialize(IHeapObject obj) + { + if (obj == null) + writer.Write((byte)GarnetObjectType.Null); + else + ((IGarnetObject)obj).Serialize(writer); + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/GeoHashTests.cs b/test/standalone/Garnet.test.collections/GeoHashTests.cs similarity index 95% rename from test/Garnet.test/GeoHashTests.cs rename to test/standalone/Garnet.test.collections/GeoHashTests.cs index 1b93a84d1d4..4ca0d1e53dc 100644 --- a/test/Garnet.test/GeoHashTests.cs +++ b/test/standalone/Garnet.test.collections/GeoHashTests.cs @@ -1,18 +1,16 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Globalization; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class GeoHashTests : AllureTestBase + public class GeoHashTests : TestBase { [Test] [TestCase(30.5388942218, 104.0555758833)] diff --git a/test/Garnet.test/ObjectTestsForOutput.cs b/test/standalone/Garnet.test.collections/ObjectTestsForOutput.cs similarity index 99% rename from test/Garnet.test/ObjectTestsForOutput.cs rename to test/standalone/Garnet.test.collections/ObjectTestsForOutput.cs index 440e4f87419..f352bdc764e 100644 --- a/test/Garnet.test/ObjectTestsForOutput.cs +++ b/test/standalone/Garnet.test.collections/ObjectTestsForOutput.cs @@ -2,16 +2,13 @@ // Licensed under the MIT license. using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - - [AllureNUnit] [TestFixture] - public class ObjectTestsForOutput : AllureTestBase + public class ObjectTestsForOutput : TestBase { protected GarnetServer server; diff --git a/test/Garnet.test/RespBlockingCollectionTests.cs b/test/standalone/Garnet.test.collections/RespBlockingCollectionTests.cs similarity index 99% rename from test/Garnet.test/RespBlockingCollectionTests.cs rename to test/standalone/Garnet.test.collections/RespBlockingCollectionTests.cs index 06874f7cb98..b725430f90e 100644 --- a/test/Garnet.test/RespBlockingCollectionTests.cs +++ b/test/standalone/Garnet.test.collections/RespBlockingCollectionTests.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespBlockingCollectionTests : AllureTestBase + public class RespBlockingCollectionTests : TestBase { GarnetServer server; private static readonly Random random = Random.Shared; diff --git a/test/Garnet.test/RespHashTests.cs b/test/standalone/Garnet.test.collections/RespHashTests.cs similarity index 99% rename from test/Garnet.test/RespHashTests.cs rename to test/standalone/Garnet.test.collections/RespHashTests.cs index 2465c425d78..877a1434310 100644 --- a/test/Garnet.test/RespHashTests.cs +++ b/test/standalone/Garnet.test.collections/RespHashTests.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespHashTests : AllureTestBase + public class RespHashTests : TestBase { GarnetServer server; @@ -26,7 +24,7 @@ public class RespHashTests : AllureTestBase public void Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableObjectStoreReadCache: true, enableAOF: true, lowMemory: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableAOF: true, lowMemory: true); server.Start(); } @@ -1135,7 +1133,7 @@ public async Task CanDoHashExpireLTM() db.HashSet(key, [new HashEntry("Field1", "StringValue"), new HashEntry("Field2", "1")]); } - var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); + var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); // Ensure data has spilled to disk ClassicAssert.Greater(info.HeadAddress, info.BeginAddress); @@ -1427,7 +1425,7 @@ public void CanSetAndGetMultiplepairLC(int bytesSent) TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":680\r\n"; + expectedResponse = ":712\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); // multiple get @@ -1491,7 +1489,7 @@ public void CanDeleteOnepairLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":408\r\n"; + expectedResponse = ":440\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("HDEL myhash field1"); @@ -1499,7 +1497,7 @@ public void CanDeleteOnepairLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":272\r\n"; + expectedResponse = ":304\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); //HDEL with nonexisting key @@ -1594,7 +1592,7 @@ public void CanDoIncrByLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":264\r\n"; + expectedResponse = ":296\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); // do hincrby @@ -1603,7 +1601,7 @@ public void CanDoIncrByLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":264\r\n"; + expectedResponse = ":296\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); } @@ -1616,7 +1614,7 @@ public void CanDoIncrByFloatLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":264\r\n"; + expectedResponse = ":296\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("HINCRBYFLOAT myhash field 0.1"); @@ -1624,7 +1622,7 @@ public void CanDoIncrByFloatLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":264\r\n"; + expectedResponse = ":296\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); // exponential notation @@ -1633,7 +1631,7 @@ public void CanDoIncrByFloatLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":392\r\n"; + expectedResponse = ":424\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommands("HINCRBYFLOAT myhash field2 2.0e2", "PING HELLO"); @@ -1641,7 +1639,7 @@ public void CanDoIncrByFloatLC() TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); response = lightClientRequest.SendCommand("MEMORY USAGE myhash"); - expectedResponse = ":392\r\n"; + expectedResponse = ":424\r\n"; TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); } diff --git a/test/Garnet.test/RespListGarnetClientTests.cs b/test/standalone/Garnet.test.collections/RespListGarnetClientTests.cs similarity index 99% rename from test/Garnet.test/RespListGarnetClientTests.cs rename to test/standalone/Garnet.test.collections/RespListGarnetClientTests.cs index f9a151bba58..0337b42a3df 100644 --- a/test/Garnet.test/RespListGarnetClientTests.cs +++ b/test/standalone/Garnet.test.collections/RespListGarnetClientTests.cs @@ -3,16 +3,14 @@ using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.client; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespListGarnetClientTests : AllureTestBase + public class RespListGarnetClientTests : TestBase { private GarnetServer server; diff --git a/test/Garnet.test/RespListTests.cs b/test/standalone/Garnet.test.collections/RespListTests.cs similarity index 92% rename from test/Garnet.test/RespListTests.cs rename to test/standalone/Garnet.test.collections/RespListTests.cs index 4f5495bc34a..e33ee2dd822 100644 --- a/test/Garnet.test/RespListTests.cs +++ b/test/standalone/Garnet.test.collections/RespListTests.cs @@ -7,7 +7,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,9 +14,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - class RespListTests : AllureTestBase + class RespListTests : TestBase { GarnetServer server; Random r; @@ -54,7 +52,7 @@ public void BasicLPUSHAndLPOP() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 184; + var expectedResponse = 224; ClassicAssert.AreEqual(expectedResponse, actualValue); string popval = db.ListLeftPop(key); @@ -85,7 +83,7 @@ public void MultiLPUSHAndLTRIMWithMemoryCheck() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 904; + var expectedResponse = 944; ClassicAssert.AreEqual(expectedResponse, actualValue); db.ListTrim(key, 1, 5); @@ -95,7 +93,7 @@ public void MultiLPUSHAndLTRIMWithMemoryCheck() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 504; + expectedResponse = 544; ClassicAssert.AreEqual(expectedResponse, actualValue); //all elements remain @@ -105,7 +103,7 @@ public void MultiLPUSHAndLTRIMWithMemoryCheck() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 504; + expectedResponse = 544; ClassicAssert.AreEqual(expectedResponse, actualValue); db.ListTrim(key, 0, -3); @@ -173,11 +171,9 @@ public void MultiLPUSHAndLLENWithPendingStatus() var db = redis.GetDatabase(0); var nVals = 100; - RedisValue[] values = new RedisValue[nVals]; + var values = new RedisValue[nVals]; for (int i = 0; i < 100; i++) - { - values[i] = ($"val-{i + 1}"); - } + values[i] = $"val-{i + 1}"; for (int j = 0; j < 25; j++) { @@ -185,7 +181,7 @@ public void MultiLPUSHAndLLENWithPendingStatus() ClassicAssert.AreEqual(nVals, nAdded); } - long nLen = db.ListLength("List_Test-10"); + var nLen = db.ListLength("List_Test-10"); ClassicAssert.AreEqual(100, nLen); } @@ -295,7 +291,7 @@ public void BasicRPUSHAndLINSERT() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 344; + var expectedResponse = 384; ClassicAssert.AreEqual(expectedResponse, actualValue); long nLen = db.ListLength(key); @@ -309,7 +305,7 @@ public void BasicRPUSHAndLINSERT() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 432; + expectedResponse = 472; ClassicAssert.AreEqual(expectedResponse, actualValue); // test after @@ -320,7 +316,7 @@ public void BasicRPUSHAndLINSERT() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 520; + expectedResponse = 560; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -344,7 +340,7 @@ public void BasicRPUSHAndLREM() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 584; + var expectedResponse = 624; ClassicAssert.AreEqual(expectedResponse, actualValue); long nLen = db.ListLength(key); @@ -353,7 +349,7 @@ public void BasicRPUSHAndLREM() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 424; + expectedResponse = 464; ClassicAssert.AreEqual(expectedResponse, actualValue); ret = db.ListRemove(key, "val_4", -1); @@ -362,7 +358,7 @@ public void BasicRPUSHAndLREM() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 344; + expectedResponse = 384; ClassicAssert.AreEqual(expectedResponse, actualValue); ret = db.ListRemove(key, "val_2", 0); @@ -394,7 +390,7 @@ public void MultiLPUSHAndLPOPV1() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 904; + var expectedResponse = 944; ClassicAssert.AreEqual(expectedResponse, actualValue); long nLen = db.ListLength(key); @@ -440,7 +436,7 @@ public void MultiLPUSHAndLPOPV2() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 904; + var expectedResponse = 944; ClassicAssert.AreEqual(expectedResponse, actualValue); long nLen = db.ListLength(key); @@ -450,7 +446,7 @@ public void MultiLPUSHAndLPOPV2() result = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 744; + expectedResponse = 784; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -473,7 +469,7 @@ public void MultiRPUSHAndRPOP() var result = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 904; + var expectedResponse = 944; ClassicAssert.AreEqual(expectedResponse, actualValue); string popval = string.Empty; @@ -541,7 +537,7 @@ public void CanDoRPopLPush() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 272; + var expectedResponse = 304; ClassicAssert.AreEqual(expectedResponse, actualValue); var lrange = db.ListRange(key, 0, -1); @@ -558,7 +554,7 @@ public void CanDoRPopLPush() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 272; + expectedResponse = 304; ClassicAssert.AreEqual(expectedResponse, actualValue); lrange = db.ListRange(key, 0, -1); @@ -1078,14 +1074,25 @@ public void CanHandleNoPrexistentKey() [Test] [Repeat(10)] - public void ListPushPopStressTest() + public async Task ListPushPopStressTest() { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + // Custom config with a generous timeout for this stress test specifically — the default + // 30s is normally plenty, but under heavy CI load a queued response can be delayed. + static ConfigurationOptions StressConfig() + { + var cfg = TestUtils.GetConfig(); + cfg.SyncTimeout = (int)TimeSpan.FromMinutes(2).TotalMilliseconds; + cfg.AsyncTimeout = (int)TimeSpan.FromMinutes(2).TotalMilliseconds; + return cfg; + } + + using var redis = ConnectionMultiplexer.Connect(StressConfig()); var db = redis.GetDatabase(0); - int keyCount = 10; + // Keep concurrency modest so we don't outpace small CI runners while still exercising + // real LPUSH/RPOP concurrency on multiple keys. + int keyCount = 5; int ppCount = 100; - //string[] keys = new string[keyCount]; HashSet keys = []; for (int i = 0; i < keyCount; i++) while (!keys.Add(r.Next().ToString())) { } @@ -1093,38 +1100,88 @@ public void ListPushPopStressTest() ClassicAssert.AreEqual(keyCount, keys.Count, "Unique key initialization failed!"); var keyArray = keys.ToArray(); - Task[] tasks = new Task[keyArray.Length << 1]; - for (int i = 0; i < tasks.Length; i += 2) + + // Cancellation token stops in-flight workers on first failure and provides a hard deadline. + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + + // Pre-create one ConnectionMultiplexer per worker so each "client" has its own socket. + // A single shared mux serializes all writes through one background writer; under CI load + // that writer can fall behind enough to exceed timeouts. Connect up front to also avoid + // a connect storm racing ConnectTimeout. + var workerCount = keyCount * 2; + var workerMuxes = new ConnectionMultiplexer[workerCount]; + try { - int idx = i; - tasks[i] = Task.Run(async () => - { - var key = keyArray[idx >> 1]; - for (int j = 0; j < ppCount; j++) - await db.ListLeftPushAsync(key, j).ConfigureAwait(false); - }); + for (int i = 0; i < workerCount; i++) + workerMuxes[i] = ConnectionMultiplexer.Connect(StressConfig()); + } + catch + { + foreach (var mux in workerMuxes) + mux?.Dispose(); + throw; + } - tasks[i + 1] = Task.Run(() => + try + { + // Use async Redis APIs scheduled on the threadpool. Each await releases the worker + // thread while the response is in flight, so SE.Redis's IO-completion continuations + // always have a free worker. + var tasks = new Task[workerCount]; + for (int i = 0; i < keyCount; i++) { - var key = keyArray[idx >> 1]; - for (int j = 0; j < ppCount; j++) + var key = keyArray[i]; + var pushDb = workerMuxes[i * 2].GetDatabase(0); + var popDb = workerMuxes[i * 2 + 1].GetDatabase(0); + + tasks[i * 2] = Task.Run(async () => { - var value = db.ListRightPop(key); - while (value.IsNull) + for (int j = 0; j < ppCount && !cts.IsCancellationRequested; j++) + _ = await pushDb.ListLeftPushAsync(key, j).ConfigureAwait(false); + }); + + tasks[i * 2 + 1] = Task.Run(async () => + { + for (int j = 0; j < ppCount && !cts.IsCancellationRequested; j++) { - Thread.Yield(); - value = db.ListRightPop(key); + var value = await popDb.ListRightPopAsync(key).ConfigureAwait(false); + while (value.IsNull && !cts.IsCancellationRequested) + { + await Task.Delay(1, cts.Token).ConfigureAwait(false); + value = await popDb.ListRightPopAsync(key).ConfigureAwait(false); + } + if (!cts.IsCancellationRequested) + ClassicAssert.IsTrue((int)value >= 0 && (int)value < ppCount, "Pop value inconsistency"); } - ClassicAssert.IsTrue((int)value >= 0 && (int)value < ppCount, "Pop value inconsistency"); - } - }); - } - Task.WaitAll(tasks); + }); + } - foreach (var key in keyArray) + // Await all tasks; observe ALL faults (Task.WhenAll's awaiter only re-throws the + // first exception, but Task.WhenAll(...).Exception is the full aggregate). + var allDone = Task.WhenAll(tasks); + try + { + await allDone.ConfigureAwait(false); + } + catch + { + cts.Cancel(); + } + + if (allDone.Exception != null) + throw allDone.Exception; + + foreach (var key in keyArray) + { + var count = db.ListLength(key); + ClassicAssert.AreEqual(0, count); + } + } + finally { - var count = db.ListLength(key); - ClassicAssert.AreEqual(0, count); + cts.Cancel(); + foreach (var mux in workerMuxes) + mux?.Dispose(); } } @@ -1239,7 +1296,7 @@ public void CanDoLPushXRpushX() response = db.Execute("MEMORY", "USAGE", "mylist"); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 904; + expectedResponse = 936; ClassicAssert.AreEqual(expectedResponse, actualValue); //this should not create a new list @@ -1257,7 +1314,7 @@ public void CanDoLPushXRpushX() response = db.Execute("MEMORY", "USAGE", "myaux-list"); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 912; + expectedResponse = 952; ClassicAssert.AreEqual(expectedResponse, actualValue); } diff --git a/test/Garnet.test/RespSetTest.cs b/test/standalone/Garnet.test.collections/RespSetTest.cs similarity index 99% rename from test/Garnet.test/RespSetTest.cs rename to test/standalone/Garnet.test.collections/RespSetTest.cs index 46b56d556aa..ea4df3d1743 100644 --- a/test/Garnet.test/RespSetTest.cs +++ b/test/standalone/Garnet.test.collections/RespSetTest.cs @@ -6,7 +6,6 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,9 +14,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespSetTest : AllureTestBase + public class RespSetTest : TestBase { GarnetServer server; @@ -92,7 +90,7 @@ public void CanAddAndListMembers() var response = db.Execute("MEMORY", "USAGE", "user1:set"); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 272; + var expectedResponse = 312; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -185,7 +183,7 @@ public void CanRemoveField() var memresponse = db.Execute("MEMORY", "USAGE", "user1:set"); var actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - var expectedResponse = 424; + var expectedResponse = 464; ClassicAssert.AreEqual(expectedResponse, actualValue); var response = db.SetRemove(key, new RedisValue("ItemOne")); @@ -193,7 +191,7 @@ public void CanRemoveField() memresponse = db.Execute("MEMORY", "USAGE", "user1:set"); actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - expectedResponse = 352; + expectedResponse = 392; ClassicAssert.AreEqual(expectedResponse, actualValue); response = db.SetRemove(key, new RedisValue("ItemFive")); @@ -201,7 +199,7 @@ public void CanRemoveField() memresponse = db.Execute("MEMORY", "USAGE", "user1:set"); actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - expectedResponse = 352; + expectedResponse = 392; ClassicAssert.AreEqual(expectedResponse, actualValue); var longResponse = db.SetRemove(key, ["ItemTwo", "ItemThree"]); @@ -209,7 +207,7 @@ public void CanRemoveField() memresponse = db.Execute("MEMORY", "USAGE", "user1:set"); actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - expectedResponse = 200; + expectedResponse = 240; ClassicAssert.AreEqual(expectedResponse, actualValue); var members = db.SetMembers(key); diff --git a/test/Garnet.test/RespSortedSetGarnetClientTests.cs b/test/standalone/Garnet.test.collections/RespSortedSetGarnetClientTests.cs similarity index 99% rename from test/Garnet.test/RespSortedSetGarnetClientTests.cs rename to test/standalone/Garnet.test.collections/RespSortedSetGarnetClientTests.cs index de279f9460b..e259915d32e 100644 --- a/test/Garnet.test/RespSortedSetGarnetClientTests.cs +++ b/test/standalone/Garnet.test.collections/RespSortedSetGarnetClientTests.cs @@ -9,7 +9,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.client; using Garnet.client.GarnetClientAPI; using Garnet.common; @@ -19,9 +18,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespSortedSetGarnetClientTests : AllureTestBase + public class RespSortedSetGarnetClientTests : TestBase { protected GarnetServer server; ManualResetEventSlim waiter; @@ -171,7 +169,7 @@ public async Task CanDoZAddGarnetMultithread() { tasks[i] = Task.Run(async () => { - for (var ii = 0; ii < numIterations; ++ii) + for (var ii = 0; ii < numIterations; ii++) { string name = GetUniqueName(ss, ii); if (name == String.Empty) diff --git a/test/Garnet.test/RespSortedSetGeoTests.cs b/test/standalone/Garnet.test.collections/RespSortedSetGeoTests.cs similarity index 99% rename from test/Garnet.test/RespSortedSetGeoTests.cs rename to test/standalone/Garnet.test.collections/RespSortedSetGeoTests.cs index ad8a5b4fa80..a6a4a4888a8 100644 --- a/test/Garnet.test/RespSortedSetGeoTests.cs +++ b/test/standalone/Garnet.test.collections/RespSortedSetGeoTests.cs @@ -1,11 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Globalization; using System.Linq; using System.Text; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -14,9 +13,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespSortedSetGeoTests : AllureTestBase + public class RespSortedSetGeoTests : TestBase { GarnetServer server; @@ -162,7 +160,7 @@ public void CanUseGeoAdd() var memresponse = db.Execute("MEMORY", "USAGE", "cities"); var actualValue = ResultType.Integer == memresponse.Resp2Type ? int.Parse(memresponse.ToString()) : -1; - var expectedResponse = 3944; + var expectedResponse = 3976; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -206,7 +204,7 @@ public void CanUseGeoPos() var memresponse = db.Execute("MEMORY", "USAGE", "Sicily"); var actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - var expectedResponse = 344; + var expectedResponse = 376; ClassicAssert.AreEqual(expectedResponse, actualValue); db.GeoAdd(new RedisKey("SecondKey"), 13.361389, 38.115556, new RedisValue("Palermo")); @@ -216,7 +214,7 @@ public void CanUseGeoPos() memresponse = db.Execute("MEMORY", "USAGE", "SecondKey"); actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - expectedResponse = 352; + expectedResponse = 392; ClassicAssert.AreEqual(expectedResponse, actualValue); var responseHash = db.GeoHash(new RedisKey("SecondKey"), ["Palermo"]); @@ -225,7 +223,7 @@ public void CanUseGeoPos() memresponse = db.Execute("MEMORY", "USAGE", "SecondKey"); actualValue = ResultType.Integer == memresponse.Resp2Type ? Int32.Parse(memresponse.ToString()) : -1; - expectedResponse = 352; + expectedResponse = 392; ClassicAssert.AreEqual(expectedResponse, actualValue); } diff --git a/test/Garnet.test/RespSortedSetTests.cs b/test/standalone/Garnet.test.collections/RespSortedSetTests.cs similarity index 98% rename from test/Garnet.test/RespSortedSetTests.cs rename to test/standalone/Garnet.test.collections/RespSortedSetTests.cs index 64a481981d5..3fbb4004731 100644 --- a/test/Garnet.test/RespSortedSetTests.cs +++ b/test/standalone/Garnet.test.collections/RespSortedSetTests.cs @@ -8,7 +8,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Embedded.server; using Garnet.common; using Garnet.server; @@ -20,19 +19,8 @@ namespace Garnet.test { - using TestBasicGarnetApi = GarnetApi, - SpanByteAllocator>>, - BasicContext>, - GenericAllocator>>>, - BasicContext, - SpanByteAllocator>>>; - - [AllureNUnit] [TestFixture] - public class RespSortedSetTests : AllureTestBase + public class RespSortedSetTests : TestBase { protected GarnetServer server; @@ -83,7 +71,7 @@ public class RespSortedSetTests : AllureTestBase public void Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableObjectStoreReadCache: true, enableAOF: true, lowMemory: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableAOF: true, lowMemory: true); server.Start(); } @@ -105,16 +93,17 @@ public unsafe void SortedSetPopTest() db.SortedSetAdd("key1", "b", 2); var session = new RespServerSession(0, new EmbeddedNetworkSender(), server.Provider.StoreWrapper, null, null, false); - var api = new TestBasicGarnetApi(session.storageSession, session.storageSession.basicContext, session.storageSession.objectStoreBasicContext); + var api = new BasicGarnetApi(session.storageSession, session.storageSession.stringBasicContext, + session.storageSession.objectBasicContext, session.storageSession.unifiedBasicContext); var key = Encoding.ASCII.GetBytes("key1"); fixed (byte* keyPtr = key) { - var result = api.SortedSetPop(new ArgSlice(keyPtr, key.Length), out var items); + var result = api.SortedSetPop(PinnedSpanByte.FromPinnedPointer(keyPtr, key.Length), out var items); ClassicAssert.AreEqual(1, items.Length); ClassicAssert.AreEqual("a", Encoding.ASCII.GetString(items[0].member.ReadOnlySpan)); ClassicAssert.AreEqual("1", Encoding.ASCII.GetString(items[0].score.ReadOnlySpan)); - result = api.SortedSetPop(new ArgSlice(keyPtr, key.Length), out items); + result = api.SortedSetPop(PinnedSpanByte.FromPinnedPointer(keyPtr, key.Length), out items); ClassicAssert.AreEqual(1, items.Length); ClassicAssert.AreEqual("b", Encoding.ASCII.GetString(items[0].member.ReadOnlySpan)); ClassicAssert.AreEqual("2", Encoding.ASCII.GetString(items[0].score.ReadOnlySpan)); @@ -137,11 +126,12 @@ public unsafe void SortedSetPopWithExpire() Thread.Sleep(200); var session = new RespServerSession(0, new EmbeddedNetworkSender(), server.Provider.StoreWrapper, null, null, false); - var api = new TestBasicGarnetApi(session.storageSession, session.storageSession.basicContext, session.storageSession.objectStoreBasicContext); + var api = new BasicGarnetApi(session.storageSession, session.storageSession.stringBasicContext, + session.storageSession.objectBasicContext, session.storageSession.unifiedBasicContext); var key = Encoding.ASCII.GetBytes("key1"); fixed (byte* keyPtr = key) { - var result = api.SortedSetPop(new ArgSlice(keyPtr, key.Length), out var items); + var result = api.SortedSetPop(PinnedSpanByte.FromPinnedPointer(keyPtr, key.Length), out var items); ClassicAssert.AreEqual(1, items.Length); ClassicAssert.AreEqual("b", Encoding.ASCII.GetString(items[0].member.ReadOnlySpan)); ClassicAssert.AreEqual("2", Encoding.ASCII.GetString(items[0].score.ReadOnlySpan)); @@ -209,7 +199,7 @@ public void AddAndLength() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1792; + var expectedResponse = 1832; ClassicAssert.AreEqual(expectedResponse, actualValue); var entries2 = new SortedSetEntry[entries.Length + 1]; @@ -222,7 +212,7 @@ public void AddAndLength() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1952; + expectedResponse = 1992; ClassicAssert.AreEqual(expectedResponse, actualValue); // no new entries get added @@ -231,7 +221,6 @@ public void AddAndLength() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1952; ClassicAssert.AreEqual(expectedResponse, actualValue); card = db.SortedSetLength(key); @@ -242,7 +231,6 @@ public void AddAndLength() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1952; ClassicAssert.AreEqual(expectedResponse, actualValue); var deleted = db.KeyDelete(key); @@ -508,7 +496,7 @@ public void CanCreateLeaderBoard() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1792; + var expectedResponse = 1832; ClassicAssert.AreEqual(expectedResponse, actualValue); var card = db.SortedSetLength(key); @@ -592,7 +580,7 @@ public void AddRemove() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1800; + var expectedResponse = 1848; ClassicAssert.AreEqual(expectedResponse, actualValue); // remove all entries @@ -618,7 +606,7 @@ public void AddRemove() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 360; + expectedResponse = 408; ClassicAssert.AreEqual(expectedResponse, actualValue); // remove the single entry @@ -647,7 +635,7 @@ public void AddRemove() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1800; + expectedResponse = 1848; ClassicAssert.AreEqual(expectedResponse, actualValue); // 1 entry removed @@ -660,7 +648,7 @@ public void AddRemove() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1640; + expectedResponse = 1688; ClassicAssert.AreEqual(expectedResponse, actualValue); // remaining entries removed @@ -735,7 +723,7 @@ public void AddPopDesc() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1792; + var expectedResponse = 1840; ClassicAssert.AreEqual(expectedResponse, actualValue); var last = db.SortedSetPop(key, Order.Descending); @@ -745,7 +733,7 @@ public void AddPopDesc() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1632; + expectedResponse = 1680; ClassicAssert.AreEqual(expectedResponse, actualValue); var last2 = db.SortedSetPop(key, 2, Order.Descending); @@ -756,7 +744,7 @@ public void AddPopDesc() response = db.Execute("MEMORY", "USAGE", key); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 1312; + expectedResponse = 1360; ClassicAssert.AreEqual(expectedResponse, actualValue); var last3 = db.SortedSetPop(key, 999, Order.Descending); @@ -791,7 +779,7 @@ public void AddScore() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1800; + var expectedResponse = 1848; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -816,7 +804,7 @@ public void CanDoZMScore() var memResponse = db.Execute("MEMORY", "USAGE", key); var memActualValue = ResultType.Integer == memResponse.Resp2Type ? Int32.Parse(memResponse.ToString()) : -1; - var memExpectedResponse = 1808; + var memExpectedResponse = 1864; ClassicAssert.AreEqual(memExpectedResponse, memActualValue); } @@ -856,7 +844,7 @@ public void CandDoZIncrby() var response = db.Execute("MEMORY", "USAGE", key); var actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - var expectedResponse = 1792; + var expectedResponse = 1832; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -915,7 +903,7 @@ public void CanManageNotExistingKeySE() response = db.Execute("MEMORY", "USAGE", "nokey"); actualValue = ResultType.Integer == response.Resp2Type ? Int32.Parse(response.ToString()) : -1; - expectedResponse = 344; + expectedResponse = 376; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -2194,7 +2182,7 @@ public async Task CanDoSortedSetExpireLTM() ]); } - var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); + var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); // Ensure data has spilled to disk ClassicAssert.Greater(info.HeadAddress, info.BeginAddress); @@ -2676,10 +2664,10 @@ public void ZLexCountWithExpiredAndExpiringItems() db.SortedSetAdd("key1", "d", 4); db.SortedSetAdd("key1", "e", 5); - db.Execute("ZPEXPIRE", "key1", "200", "MEMBERS", "3", "a", "e", "c"); - db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "1", "b"); + db.Execute("ZPEXPIRE", "key1", "500", "MEMBERS", "3", "a", "e", "c"); + db.Execute("ZPEXPIRE", "key1", "2000", "MEMBERS", "1", "b"); - Thread.Sleep(300); + Thread.Sleep(1000); var lexCount = (int)db.Execute("ZLEXCOUNT", "key1", "-", "+"); // SortedSetLengthByValue will check - and + to [- and [+ ClassicAssert.AreEqual(2, lexCount); // Only "b" and "d" should remain @@ -2687,7 +2675,7 @@ public void ZLexCountWithExpiredAndExpiringItems() var lexCountRange = db.SortedSetLengthByValue("key1", "b", "d", Exclude.Stop); ClassicAssert.AreEqual(1, lexCountRange); // Only "b" should remain within the range - Thread.Sleep(300); + Thread.Sleep(1500); lexCount = (int)db.Execute("ZLEXCOUNT", "key1", "-", "+"); ClassicAssert.AreEqual(1, lexCount); // Only "d" should remain diff --git a/test/standalone/Garnet.test.collections/TestProjectSetup.cs b/test/standalone/Garnet.test.collections/TestProjectSetup.cs new file mode 100644 index 00000000000..8855c649a09 --- /dev/null +++ b/test/standalone/Garnet.test.collections/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestCollections); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.complexstring/Garnet.test.complexstring.csproj b/test/standalone/Garnet.test.complexstring/Garnet.test.complexstring.csproj new file mode 100644 index 00000000000..340c2ffce68 --- /dev/null +++ b/test/standalone/Garnet.test.complexstring/Garnet.test.complexstring.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test/GarnetBitmapTests.cs b/test/standalone/Garnet.test.complexstring/GarnetBitmapTests.cs similarity index 91% rename from test/Garnet.test/GarnetBitmapTests.cs rename to test/standalone/Garnet.test.complexstring/GarnetBitmapTests.cs index 746d3b2894b..767a8b48499 100644 --- a/test/Garnet.test/GarnetBitmapTests.cs +++ b/test/standalone/Garnet.test.complexstring/GarnetBitmapTests.cs @@ -1,11 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; using System.Numerics.Tensors; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -14,12 +13,11 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class GarnetBitmapTests : AllureTestBase + public class GarnetBitmapTests : TestBase { GarnetServer server; - Random r; + Random rng; [SetUp] public void Setup() @@ -38,17 +36,17 @@ public void Setup() server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, useReviv: useReviv); server.Start(); - r = new Random(674386); + rng = new Random(674386); } [TearDown] public void TearDown() { - server.Dispose(); + server?.Dispose(); TestUtils.OnTearDown(); } - private long LongRandom() => r.NextInt64(long.MinValue, long.MaxValue); + private long LongRandom() => rng.NextInt64(long.MinValue, long.MaxValue); private ulong ULongRandom() => (ulong)LongRandom(); @@ -232,8 +230,8 @@ public void BitmapSetGetBitTest_LTM(bool preSet) for (int j = 0; j < keyIter; j++) { - long offset = r.Next(0, bitmapBytes << 3); - bool set = r.Next(0, 1) == 0 ? false : true; + long offset = rng.Next(0, bitmapBytes << 3); + bool set = rng.Next(0, 1) == 0 ? false : true; bool returnedVal = db.StringSetBit(sKey, offset, set); bool expectedVal = false; @@ -265,12 +263,12 @@ public void BitmapSetGetBitTest_LTM(bool preSet) for (int j = 0; j < keyIter; j++) { - long offset = r.Next(0, bitmapBytes << 3); + long offset = rng.Next(0, bitmapBytes << 3); bool returnedVal = db.StringGetBit(sKey, offset); bool expectedVal = false; if (state.ContainsKey(key) && state[key].ContainsKey(offset)) expectedVal = state[key][offset]; - ClassicAssert.AreEqual(expectedVal, returnedVal, $"{offset}"); + ClassicAssert.AreEqual(expectedVal, returnedVal, $"offset {offset}"); } } } @@ -282,11 +280,13 @@ public void BitmapSetGetBitTest_LTM(bool preSet) [TestCase("DOTNET_EnableHWIntrinsic", "0")] public void BitmapSimpleBitCountTest(string arg, string val) { - using var server = new GarnetServerTestProcess(new() { [arg] = val }); + // Dispose the [SetUp] server to free the port for the external GarnetServerTestProcess + server.Dispose(); + server = null; + using var testProcess = new GarnetServerTestProcess(new() { [arg] = val }, TestUtils.TestPort); try { - - using var redis = ConnectionMultiplexer.Connect(server.Options); + using var redis = ConnectionMultiplexer.Connect(testProcess.Options); var db = redis.GetDatabase(0); var maxBitmapLen = 1 << 12; @@ -296,7 +296,7 @@ public void BitmapSimpleBitCountTest(string arg, string val) for (var i = 0; i < iter; i++) { - var offset = r.Next(1, maxBitmapLen); + var offset = rng.Next(1, maxBitmapLen); var set = !db.StringSetBit(key, offset, true); expectedCount += set ? 1 : 0; } @@ -306,8 +306,7 @@ public void BitmapSimpleBitCountTest(string arg, string val) } catch { - server.RecordTestOutput(); - + testProcess.RecordTestOutput(); throw; } } @@ -366,7 +365,7 @@ public void BitmapBitCountBetweenOffsetsTest() long maxOffset = 0; for (int i = 0; i < iter; i++) { - long offset = r.Next(1, maxBitmapLen); + long offset = rng.Next(1, maxBitmapLen); db.StringSetBit(key, offset, true); maxOffset = Math.Max(offset, maxOffset); offsets.Add(offset); @@ -387,27 +386,27 @@ public void BitmapBitCountBetweenOffsetsTest() long expectedCount = Count(bitmap, 0, -1); count = db.StringBitCount(key, 0, -1); - ClassicAssert.AreEqual(count, expectedCount, $"{0} {-1} {bitmap.Length}"); + ClassicAssert.AreEqual(expectedCount, count, $"startOffset {0}, endOffset {-1}, bitmapLength {bitmap.Length}"); //Test with startOffset for (int i = 0; i < iter; i++) { - int startOffset = r.Next(1, (int)maxSizeInBytes); + int startOffset = rng.Next(1, (int)maxSizeInBytes); expectedCount = Count(bitmap, startOffset, -1); count = db.StringBitCount(key, startOffset); - ClassicAssert.AreEqual(expectedCount, count, $"{startOffset} {-1} {maxSizeInBytes}"); + ClassicAssert.AreEqual(expectedCount, count, $"startOffset {startOffset}, endOffset {-1}, maxSizeInBytes {maxSizeInBytes}"); } //Test with startOffset and endOffset for (int i = 0; i < iter; i++) { - int startOffset = r.Next(1, (int)maxSizeInBytes); - int endOffset = r.Next(startOffset, (int)maxSizeInBytes); + int startOffset = rng.Next(1, (int)maxSizeInBytes); + int endOffset = rng.Next(startOffset, (int)maxSizeInBytes); expectedCount = Count(bitmap, startOffset, endOffset); count = db.StringBitCount(key, startOffset, endOffset); - ClassicAssert.AreEqual(expectedCount, count, $"{startOffset} {endOffset} {maxSizeInBytes}"); + ClassicAssert.AreEqual(expectedCount, count, $"startOffset {startOffset}, endOffset {endOffset}, maxSizeInBytes {maxSizeInBytes}"); } } @@ -430,12 +429,12 @@ public void BitmapBitCountBetweenOffsetsTestV2() for (int j = 0; j < iter; j++) { for (int i = 0; i < buf.Length; i++) - buf[i] = (byte)r.Next(0, 128); + buf[i] = (byte)rng.Next(0, 128); db.StringSet(key, buf); - int startOffset = r.Next(1, buf.Length); - int endOffset = r.Next(startOffset, buf.Length); + int startOffset = rng.Next(1, buf.Length); + int endOffset = rng.Next(startOffset, buf.Length); long expectedCount = Count(buf, startOffset, endOffset); count = db.StringBitCount(key, startOffset, endOffset); @@ -462,11 +461,11 @@ public void BitmapBitCountNegativeOffsets() //check offsets in range for (int j = 0; j < iter; j++) { - r.NextBytes(buf); + rng.NextBytes(buf); db.StringSet(key, buf); - int startOffset = j == 0 ? -10 : r.Next(-maxByteLen, 0); - int endOffset = j == 0 ? -1 : r.Next(startOffset, 0); + int startOffset = j == 0 ? -10 : rng.Next(-maxByteLen, 0); + int endOffset = j == 0 ? -1 : rng.Next(startOffset, 0); expectedCount = Count(buf, startOffset, endOffset); count = db.StringBitCount(key, startOffset, endOffset); @@ -477,11 +476,11 @@ public void BitmapBitCountNegativeOffsets() //check negative offsets beyond range for (int j = 0; j < iter; j++) { - r.NextBytes(buf); + rng.NextBytes(buf); db.StringSet(key, buf); - int startOffset = j == 0 ? -10 : r.Next(-maxByteLen << 1, -maxByteLen); - int endOffset = j == 0 ? -1 : r.Next(startOffset, -maxByteLen); + int startOffset = j == 0 ? -10 : rng.Next(-maxByteLen << 1, -maxByteLen); + int endOffset = j == 0 ? -1 : rng.Next(startOffset, -maxByteLen); expectedCount = Count(buf, startOffset, endOffset); count = db.StringBitCount(key, startOffset, endOffset); @@ -498,7 +497,7 @@ public void BitmapBitCountTest_LTM() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: (bitmapBytes << 2).ToString(), + pageCount: 2, // Specify pageCount instead of memorySize to avoid LogSizeTracker.MinTargetPageCount requirement pageSize: (bitmapBytes << 1).ToString()); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -511,7 +510,7 @@ public void BitmapBitCountTest_LTM() for (int i = 0; i < keyCount; i++) { string sKey = i.ToString(); - r.NextBytes(bitmap); + rng.NextBytes(bitmap); bitmapList.Add(Count(bitmap)); db.StringSet(sKey, bitmap); @@ -520,7 +519,7 @@ public void BitmapBitCountTest_LTM() int iter = 128; for (int i = 0; i < iter; i++) { - int key = r.Next(0, keyCount); + int key = rng.Next(0, keyCount); string sKey = key.ToString(); long count = db.StringBitCount(sKey); long expectedCount = bitmapList[key]; @@ -542,7 +541,7 @@ public unsafe void BitmapSimpleBITCOUNT_PCT(int bytesPerSend) string key = "mykey"; int maxBitmapLen = 1 << 12; byte[] buf = new byte[maxBitmapLen >> 3]; - r.NextBytes(buf); + rng.NextBytes(buf); db.StringSet(key, buf); long expectedCount = Count(buf); @@ -607,7 +606,7 @@ public void BitmapSimpleBitPosTests() long maxOffset = 0; for (var i = 0; i < iter; i++) { - long offset = r.Next(1, maxBitmapLen); + long offset = rng.Next(1, maxBitmapLen); _ = db.StringSetBit(key, offset, true); buf = db.StringGet(key); @@ -630,7 +629,7 @@ public void BitmapSimpleBitPosTests() for (var i = 0; i < iter; i++) { - long offset = r.Next(1, (int)maxOffset); + long offset = rng.Next(1, (int)maxOffset); _ = db.StringSetBit(key, offset, false); buf = db.StringGet(key); @@ -663,13 +662,13 @@ public void BitmapBitPosOffsetsTest() for (var j = 0; j < iter; j++) { - r.NextBytes(buf); + rng.NextBytes(buf); _ = db.StringSet(key, buf); - var startOffset = r.Next(0, maxByteLen); - var endOffset = r.Next(startOffset, maxByteLen); + var startOffset = rng.Next(0, maxByteLen); + var endOffset = rng.Next(startOffset, maxByteLen); - var set = r.Next(0, 1) == 0 ? false : true; + var set = rng.Next(0, 1) == 0 ? false : true; expectedPos = Bitpos(buf, startOffset, endOffset, set); pos = db.StringBitPosition(key, set, startOffset, endOffset); @@ -684,13 +683,13 @@ public void BitmapBitPosOffsetsTest() // check negative offsets in range for (var j = 0; j < iter; j++) { - r.NextBytes(buf); + rng.NextBytes(buf); _ = db.StringSet(key, buf); - var startOffset = j == 0 ? -10 : r.Next(-maxByteLen, 0); - var endOffset = j == 0 ? -1 : r.Next(startOffset, 0); + int startOffset = j == 0 ? -10 : rng.Next(-maxByteLen, 0); + var endOffset = j == 0 ? -1 : rng.Next(startOffset, 0); - var set = r.Next(0, 1) != 0; + var set = rng.Next(0, 1) != 0; expectedPos = Bitpos(buf, startOffset, endOffset, set); pos = db.StringBitPosition(key, set, startOffset, endOffset); ClassicAssert.AreEqual(expectedPos, pos, $"{j} {set} {startOffset} {endOffset}"); @@ -710,7 +709,7 @@ public void BitmapBitPosTest_LTM() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: (bitmapBytes << 2).ToString(), + memorySize: (bitmapBytes << 3).ToString(), // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: (bitmapBytes << 1).ToString()); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -722,8 +721,8 @@ public void BitmapBitPosTest_LTM() for (var i = 0; i < keyCount; i++) { - var sKey = i.ToString(); - r.NextBytes(bitmap); + string sKey = i.ToString(); + rng.NextBytes(bitmap); bitmapList.Add(Bitpos(bitmap, set: true)); _ = db.StringSet(sKey, bitmap); @@ -732,7 +731,7 @@ public void BitmapBitPosTest_LTM() var iter = 128; for (var i = 0; i < iter; i++) { - var key = r.Next(0, keyCount); + int key = rng.Next(0, keyCount); var sKey = key.ToString(); var pos = db.StringBitPosition(sKey, true); var expectedPos = bitmapList[key]; @@ -773,10 +772,10 @@ public unsafe void BitmapSimpleBITPOS_PCT(int bytesPerSend) using var lightClientRequest = TestUtils.CreateRequest(); var db = redis.GetDatabase(0); - var key = "mykey"; + string key = "mykey"; var maxBitmapLen = 1 << 12; var buf = new byte[maxBitmapLen >> 3]; - r.NextBytes(buf); + rng.NextBytes(buf); db.StringSet(key, buf); var expectedPos = Bitpos(buf); @@ -830,7 +829,7 @@ public void BitOp_Unary_BitwiseNot( var dstKey = "dst"; var srcKeyBitmap = new byte[bitmapLength]; - r.NextBytes(srcKeyBitmap); + rng.NextBytes(srcKeyBitmap); var expectedBitmap = CopyBitmap(srcKeyBitmap, invert: true); db.StringSet(srcKey, srcKeyBitmap); @@ -856,15 +855,17 @@ public void BitOp_Binary_SameSize( args[environment[i]] = environment[i + 1]; } - using var server = new GarnetServerTestProcess(args); + // Dispose the [SetUp] server to free the port for the external GarnetServerTestProcess + server.Dispose(); + server = null; + using var testProcess = new GarnetServerTestProcess(args, TestUtils.TestPort); try { - BitOp_Binary_SameSize(server.Options, op, bitmapSize, keys); + BitOp_Binary_SameSize(testProcess.Options, op, bitmapSize, keys); } catch { - server.RecordTestOutput(); - + testProcess.RecordTestOutput(); throw; } } @@ -907,7 +908,7 @@ private void BitOp_Binary_SameSize( for (var i = 0; i < srcKeys.Length; i++) { srcKeyBitmaps[i] = new byte[bitmapSize]; - r.NextBytes(srcKeyBitmaps[i]); + rng.NextBytes(srcKeyBitmaps[i]); srcKeys[i] = "src" + i; db.StringSet(srcKeys[i], srcKeyBitmaps[i]); @@ -958,7 +959,7 @@ public void BitOp_Binary_DifferentTails( for (var i = 0; i < srcKeys.Length; i++) { srcKeyBitmaps[i] = new byte[sharedLength + additionalLengths[i]]; - r.NextBytes(srcKeyBitmaps[i]); + rng.NextBytes(srcKeyBitmaps[i]); srcKeys[i] = "src" + i; db.StringSet(srcKeys[i], srcKeyBitmaps[i]); @@ -977,6 +978,94 @@ public void BitOp_Binary_DifferentTails( ClassicAssert.AreEqual(expectedBitmap, actualBitmap); } + // Regression test for a use-after-fixed bug in the BITOP read callback: + // for overflow values (byte[] > MaxInlineValueSize, default 4 KB) the callback used to + // capture a pointer inside a `fixed` block and dereference it later from the BITOP + // execution path; GC compaction between the two could relocate the byte[], causing + // BITOP to read garbage. Running BITOP on overflow-sized values while a background + // thread periodically triggers compacting GCs must produce stable, correct results. + [Test, Order(21)] + [Category("BITOP")] + public void BitOp_OverflowValues_StableUnderGCCompaction( + [Values(Bitwise.And, Bitwise.Or, Bitwise.Xor, Bitwise.Diff)] Bitwise op) + { + // Pick lengths that exceed the default ObjectAllocator MaxInlineValueSize (4 KB), + // which forces values into the overflow (heap byte[]) path inside Tsavorite. + const int sharedLength = 4096 + 32 + 3; + var additionalLengths = new[] { 0, 7 }; + + Func opFunc = op switch + { + Bitwise.And => static (a, b) => (byte)(a & b), + Bitwise.Or => static (a, b) => (byte)(a | b), + Bitwise.Xor => static (a, b) => (byte)(a ^ b), + Bitwise.Diff => static (a, b) => (byte)(a & ~b), + _ => throw new NotSupportedException() + }; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var srcKeyCount = additionalLengths.Length; + var srcKeys = new RedisKey[srcKeyCount]; + var srcKeyBitmaps = new byte[srcKeyCount][]; + var srcMaxLength = sharedLength + Enumerable.Max(additionalLengths); + + const string dstKey = "dst"; + var expectedBitmap = new byte[srcMaxLength]; + + for (var i = 0; i < srcKeys.Length; i++) + { + srcKeyBitmaps[i] = new byte[sharedLength + additionalLengths[i]]; + rng.NextBytes(srcKeyBitmaps[i]); + + srcKeys[i] = "src" + i; + db.StringSet(srcKeys[i], srcKeyBitmaps[i]); + + if (i == 0) + srcKeyBitmaps[i].AsSpan().CopyTo(expectedBitmap); + else + ApplyBitop(ref expectedBitmap, srcKeyBitmaps[i], opFunc); + } + + // Background thread that periodically forces compacting GCs to maximize the chance + // of the GC running between the BITOP read callback's `fixed` block and the + // subsequent in-server BITOP execution. Before the fix this exposed a stale-pointer + // dereference that produced wrong bytes. + using var stop = new System.Threading.CancellationTokenSource(); + var gcThread = new System.Threading.Thread(() => + { + while (!stop.IsCancellationRequested) + { + System.Runtime.GCSettings.LargeObjectHeapCompactionMode = + System.Runtime.GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, blocking: true, compacting: true); + System.Threading.Thread.Sleep(1); + } + }) + { IsBackground = true, Name = "BitOpGCStress" }; + gcThread.Start(); + + try + { + const int iterations = 100; + for (var iter = 0; iter < iterations; iter++) + { + var size = db.StringBitOperation(op, dstKey, srcKeys); + ClassicAssert.AreEqual(expectedBitmap.Length, size, $"Iteration {iter}: BITOP returned wrong size"); + + byte[] actualBitmap = db.StringGet(dstKey); + ClassicAssert.AreEqual(expectedBitmap.Length, actualBitmap.Length, $"Iteration {iter}: GET returned wrong length"); + ClassicAssert.AreEqual(expectedBitmap, actualBitmap, $"Iteration {iter}: BITOP {op} produced incorrect bytes"); + } + } + finally + { + stop.Cancel(); + gcThread.Join(); + } + } + private static long GetValueFromBitmap(ref byte[] bitmap, long offset, int bitCount, bool signed) { long startBit = offset; @@ -1064,10 +1153,10 @@ public void BitmapBitfieldGetTest([Values(RespCommand.BITFIELD, RespCommand.BITF long expectedValue; long returnedValue; long redisValue; - r = new Random(Guid.NewGuid().GetHashCode()); + rng = new Random(Guid.NewGuid().GetHashCode()); bitmapData = new byte[16]; - r.NextBytes(bitmapData); + rng.NextBytes(bitmapData); db.StringSet(key, bitmapData); for (int i = 0; i < (bitmapData.Length << 3) + 64; i++)//offset in bits { @@ -1121,7 +1210,7 @@ public unsafe void BitmapBitfieldGetTest_PCT([Values(RespCommand.BITFIELD, RespC //r = new Random(Guid.NewGuid().GetHashCode()); bitmapData = new byte[16]; - r.NextBytes(bitmapData); + rng.NextBytes(bitmapData); db.StringSet(key, bitmapData); for (int i = 0; i < (bitmapData.Length << 3) + 64; i++)//offset in bits { @@ -1159,9 +1248,9 @@ public void BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand. server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: (bitmapBytes << 2).ToString(), + memorySize: (bitmapBytes << 3).ToString(), // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: (bitmapBytes << 1).ToString()); - //MemorySize: "16g", + //LogMemorySize: "16g", //PageSize: "32m"); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -1176,7 +1265,7 @@ public void BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand. for (int i = 0; i < keyCount; i++) { bitmapData[i] = new byte[bitmapBytes]; - r.NextBytes(bitmapData[i]); + rng.NextBytes(bitmapData[i]); int key = i; string sKey = i.ToString(); @@ -1186,11 +1275,11 @@ public void BitmapBitfieldGetTest_LTM([Values(RespCommand.BITFIELD, RespCommand. int iter = 1 << 12; for (int i = 0; i < iter; i++) { - int key = r.Next(0, keyCount); + int key = rng.Next(0, keyCount); byte[] currBitmap = bitmapData[key]; string sKey = key.ToString(); - int offset = r.Next(0, (bitmapData.Length << 3)); - int bitCount = r.Next(1, 65); + int offset = rng.Next(0, (bitmapData.Length << 3)); + int bitCount = rng.Next(1, 65); //signed expectedValue = GetValueFromBitmap(ref currBitmap, offset, bitCount, true); @@ -1216,7 +1305,7 @@ private long RandomIntBitRange(int bitCount, bool signed) long value = LongRandom(); - value = (r.Next() & 0x1) == 0x1 ? -value : value; + value = (rng.Next() & 0x1) == 0x1 ? -value : value; value = value >> (64 - bitCount); ClassicAssert.IsTrue(value >= minVal); @@ -1277,7 +1366,7 @@ public unsafe void BitmapBitfieldSetTest_PCT(int bytesPerSend) //r = new Random(Guid.NewGuid().GetHashCode()); bitmapData = new byte[16]; - r.NextBytes(bitmapData); + rng.NextBytes(bitmapData); db.StringSet(key, bitmapData); long oldVal, expectedOldVal; @@ -1286,8 +1375,8 @@ public unsafe void BitmapBitfieldSetTest_PCT(int bytesPerSend) //1. Test signed set bitfield for (int i = 0; i < tests; i++) { - int bitCount = r.Next(1, 64); - long offset = r.Next(0, (bitmapData.Length << 3) - bitCount - 1); + int bitCount = rng.Next(1, 64); + long offset = rng.Next(0, (bitmapData.Length << 3) - bitCount - 1); //expectedReturnVal = RandomIntBitRange(bitCount); expectedReturnVal = RandomIntBitRange(bitCount, true); @@ -1325,7 +1414,7 @@ public void BitmapBitfieldSetTest() //r = new Random(Guid.NewGuid().GetHashCode()); bitmapData = new byte[16]; - r.NextBytes(bitmapData); + rng.NextBytes(bitmapData); db.StringSet(key, bitmapData); long oldVal, expectedOldVal; @@ -1334,8 +1423,8 @@ public void BitmapBitfieldSetTest() //1. Test signed set bitfield for (int i = 0; i < tests; i++) { - int bitCount = r.Next(1, 64); - long offset = r.Next(0, (bitmapData.Length << 3) - bitCount - 1); + int bitCount = rng.Next(1, 64); + long offset = rng.Next(0, (bitmapData.Length << 3) - bitCount - 1); //expectedReturnVal = RandomIntBitRange(bitCount); expectedReturnVal = RandomIntBitRange(bitCount, true); @@ -1360,9 +1449,9 @@ public void BitmapBitfieldSetTest_LTM() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: (bitmapBytes << 2).ToString(), + pageCount: 2, // Specify pageCount instead of memorySize to avoid LogSizeTracker.MinTargetPageCount requirement pageSize: (bitmapBytes << 1).ToString()); - //MemorySize: "16g", + //LogMemorySize: "16g", //PageSize: "32m"); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -1375,7 +1464,7 @@ public void BitmapBitfieldSetTest_LTM() for (int i = 0; i < keyCount; i++) { bitmapData[i] = new byte[bitmapBytes]; - r.NextBytes(bitmapData[i]); + rng.NextBytes(bitmapData[i]); int key = i; string sKey = i.ToString(); @@ -1390,11 +1479,11 @@ public void BitmapBitfieldSetTest_LTM() int iter = 1 << 12; for (int i = 0; i < iter; i++) { - int key = r.Next(0, keyCount); + int key = rng.Next(0, keyCount); byte[] currBitmap = bitmapData[key]; string sKey = key.ToString(); - int offset = r.Next(0, (bitmapData.Length << 3)); - int bitCount = r.Next(1, 65); + int offset = rng.Next(0, (bitmapData.Length << 3)); + int bitCount = rng.Next(1, 65); setNewValue = RandomIntBitRange(bitCount, true); @@ -1531,7 +1620,7 @@ public unsafe void BitmapBitfieldSignedIncrTest_PCT(int bytesPerSend) int testCheckOverflow = 1 << 15; for (int i = 0; i < testCheckOverflow; i++) { - bitCount = r.Next(1, 64); + bitCount = rng.Next(1, 64); long value = RandomIntBitRange(bitCount, true); long incrBy = RandomIntBitRange(bitCount, true); @@ -1611,7 +1700,7 @@ public unsafe void BitmapBitfieldSignedIncrTest_PCT(int bytesPerSend) //signed overflow with wrap and sat for (int i = 0; i < tests; i++) { - bitCount = r.Next(1, 64); + bitCount = rng.Next(1, 64); long value = RandomIntBitRange(bitCount, true); long incrBy = RandomIntBitRange(bitCount, true); @@ -1701,7 +1790,7 @@ public void BitmapBitfieldSignedIncrTest() int testCheckOverflow = 1 << 15; for (int i = 0; i < testCheckOverflow; i++) { - bitCount = r.Next(1, 64); + bitCount = rng.Next(1, 64); long value = RandomIntBitRange(bitCount, true); long incrBy = RandomIntBitRange(bitCount, true); @@ -1770,7 +1859,7 @@ public void BitmapBitfieldSignedIncrTest() //signed overflow with wrap and sat for (int i = 0; i < tests; i++) { - bitCount = r.Next(1, 64); + bitCount = rng.Next(1, 64); long value = RandomIntBitRange(bitCount, true); long incrBy = RandomIntBitRange(bitCount, true); @@ -1826,9 +1915,9 @@ public void BitmapBitfieldIncrTest_LTM() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: (bitmapBytes << 2).ToString(), + pageCount: 2, // Specify pageCount instead of memorySize to avoid LogSizeTracker.MinTargetPageCount requirement pageSize: (bitmapBytes << 1).ToString()); - //MemorySize: "16g", + //LogMemorySize: "16g", //PageSize: "32m"); server.Start(); using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); @@ -1841,7 +1930,7 @@ public void BitmapBitfieldIncrTest_LTM() for (int i = 0; i < keyCount; i++) { bitmapData[i] = new byte[bitmapBytes]; - r.NextBytes(bitmapData[i]); + rng.NextBytes(bitmapData[i]); int key = i; string sKey = i.ToString(); @@ -1857,11 +1946,11 @@ public void BitmapBitfieldIncrTest_LTM() int iter = 1 << 12; for (int i = 0; i < iter; i++) { - int key = r.Next(0, keyCount); + int key = rng.Next(0, keyCount); byte[] currBitmap = bitmapData[key]; string sKey = key.ToString(); - int offset = r.Next(0, (bitmapData.Length << 3)); - int bitCount = r.Next(1, 65); + int offset = rng.Next(0, (bitmapData.Length << 3)); + int bitCount = rng.Next(1, 65); setNewValue = RandomIntBitRange(bitCount, true); incrByValue = RandomIntBitRange(bitCount, true); @@ -1929,7 +2018,7 @@ public void BitmapBitfieldUnsignedIncrTest() for (int i = 0; i < tests; i++) { - bitCount = r.Next(1, 63); + bitCount = rng.Next(1, 63); long value = RandomIntBitRange(bitCount, false); long incrBy = RandomIntBitRange(bitCount, true); @@ -2013,11 +2102,11 @@ public void BitmapBitfieldGrowingTest([Values] RevivificationMode revivification long incrBy = RandomIntBitRange(bitCount, true); result = (long)db.Execute("BITFIELD", (RedisKey)key, "OVERFLOW", "WRAP", "INCRBY", "i" + bitCount.ToString(), "#" + offset.ToString(), value); - ClassicAssert.AreEqual(result, value); + ClassicAssert.AreEqual(value, result); result = (long)db.Execute("BITFIELD", (RedisKey)key, "OVERFLOW", "WRAP", "INCRBY", "i" + bitCount.ToString(), "#" + offset.ToString(), incrBy); (expectedResult, overflow) = CheckSignedBitfieldOverflow(value, incrBy, (byte)bitCount, 0); - ClassicAssert.AreEqual(result, expectedResult); + ClassicAssert.AreEqual(expectedResult, result); } //sat incrby @@ -2400,7 +2489,7 @@ public void BitmapBitPosBitSearchSingleBitRangeTests() var valueLenBits = valueLen << 3; for (var i = 0; i < iter; i++) { - var offset = r.NextInt64(0, valueLenBits); + var offset = rng.NextInt64(0, valueLenBits); BitSearch(offset, searchFor: true); BitSearch(offset, searchFor: false); } diff --git a/test/Garnet.test/HyperLogLogTests.cs b/test/standalone/Garnet.test.complexstring/HyperLogLogTests.cs similarity index 97% rename from test/Garnet.test/HyperLogLogTests.cs rename to test/standalone/Garnet.test.complexstring/HyperLogLogTests.cs index d1fefef613e..8231003369a 100644 --- a/test/Garnet.test/HyperLogLogTests.cs +++ b/test/standalone/Garnet.test.complexstring/HyperLogLogTests.cs @@ -1,11 +1,10 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Buffers.Binary; using System.Collections.Generic; using System.Linq; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -13,9 +12,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public unsafe class HyperLogLogTests : AllureTestBase + public unsafe class HyperLogLogTests : TestBase { GarnetServer server; Random r; @@ -138,7 +136,7 @@ public void HyperLogLogSimpleInvalidHLLTypeTest() } catch (Exception ex) { - ClassicAssert.AreEqual(ex.Message, "WRONGTYPE Key is not a valid HyperLogLog string value."); + Assert.That(ex.Message, Does.EndWith("WRONGTYPE Key is not a valid HyperLogLog string value.")); } try @@ -147,7 +145,7 @@ public void HyperLogLogSimpleInvalidHLLTypeTest() } catch (Exception ex) { - ClassicAssert.AreEqual(ex.Message, "WRONGTYPE Key is not a valid HyperLogLog string value."); + Assert.That(ex.Message, Does.EndWith("WRONGTYPE Key is not a valid HyperLogLog string value.")); } try @@ -156,7 +154,7 @@ public void HyperLogLogSimpleInvalidHLLTypeTest() } catch (Exception ex) { - ClassicAssert.AreEqual(ex.Message, "WRONGTYPE Key is not a valid HyperLogLog string value."); + Assert.That(ex.Message, Does.EndWith("WRONGTYPE Key is not a valid HyperLogLog string value.")); } try @@ -165,7 +163,7 @@ public void HyperLogLogSimpleInvalidHLLTypeTest() } catch (Exception ex) { - ClassicAssert.AreEqual(ex.Message, "WRONGTYPE Key is not a valid HyperLogLog string value."); + Assert.That(ex.Message, Does.EndWith("WRONGTYPE Key is not a valid HyperLogLog string value.")); } try @@ -174,7 +172,7 @@ public void HyperLogLogSimpleInvalidHLLTypeTest() } catch (Exception ex) { - ClassicAssert.AreEqual(ex.Message, "WRONGTYPE Key is not a valid HyperLogLog string value."); + Assert.That(ex.Message, Does.EndWith("WRONGTYPE Key is not a valid HyperLogLog string value.")); } } @@ -575,12 +573,12 @@ public void HyperLogLogPFADD_LTM(int seqSize) if (seqSize < 128) server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: "1024", + memorySize: "2k", // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: "512"); else server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: "32k", + memorySize: "64k", // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: "16k"); server.Start(); @@ -693,7 +691,7 @@ public void HyperLogLogTestPFMERGE_LTM_SparseToSparse() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: "1024", + memorySize: "2k", // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: "512"); server.Start(); @@ -802,7 +800,7 @@ public void HyperLogLogTestPFMERGE_LTM_SparseToDense(bool reverse) { server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, - memorySize: "32k", + memorySize: "64k", // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: "16k"); server.Start(); @@ -912,7 +910,7 @@ public void HyperLogLogTestPFMERGE_LTM_DenseToDense() server.Dispose(); server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, - memorySize: "32k", + memorySize: "64k", // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking pageSize: "16k"); server.Start(); diff --git a/test/standalone/Garnet.test.complexstring/TestProjectSetup.cs b/test/standalone/Garnet.test.complexstring/TestProjectSetup.cs new file mode 100644 index 00000000000..08a8d89705d --- /dev/null +++ b/test/standalone/Garnet.test.complexstring/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestComplexString); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.extensions/CacheSizeTrackerTests.cs b/test/standalone/Garnet.test.extensions/CacheSizeTrackerTests.cs new file mode 100644 index 00000000000..4b4b91a863e --- /dev/null +++ b/test/standalone/Garnet.test.extensions/CacheSizeTrackerTests.cs @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +using System; +using System.Threading; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; +using Tsavorite.core; + +namespace Garnet.test +{ + [TestFixture] + public class CacheSizeTrackerTests : TestBase + { + GarnetServer server; + TsavoriteKV store; + CacheSizeTracker cacheSizeTracker; + + // The HLOG will always have at least two pages allocated. + const int MinLogAllocatedPageCount = 2; + const int PageSize = 512; + const int TargetSize = 9000; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + + // memorySizeStr is 2k for inline pages (hence pageCount: 4) plus 7k for heap allocations so we end up in the middle of the third page (see individual test notes). + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, memorySize: $"{TargetSize}", pageSize: $"{PageSize}", pageCount: 4, lowMemory: true, indexSize: "1k"); + server.Start(); + store = server.Provider.StoreWrapper.store; + cacheSizeTracker = server.Provider.StoreWrapper.sizeTracker; + } + + [TearDown] + public void TearDown() + { + server?.Dispose(); + TestUtils.OnTearDown(); + } + + [Test] + public void RecordHeapSizeValidationTest() + { + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + db.HashSet("usr01", [new HashEntry("Title", "Faster")]); + string r = db.HashGet("usr01", "Title"); + ClassicAssert.AreEqual("Faster", r); + + // This will count only the value object; there is no key overflow. + ClassicAssert.AreEqual(208, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + } + + [Test] + public void SmallMainLogFineGrainedEvictionTest() + { + using var epcEvent = new ManualResetEventSlim(false); + bool evicted = false; + cacheSizeTracker.mainLogTracker.PostMemoryTrim = (allocatedPageCount, headAddress) => { evicted = true; epcEvent.Set(); }; + + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + ClassicAssert.AreEqual(MinLogAllocatedPageCount, cacheSizeTracker.mainLogTracker.logAccessor.AllocatedPageCount); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // This will count only the value object; there is no key overflow. This is for an object with one entry; the dictionary + // overhead for the entire object is 80, and the per-entry overhead is 128. So for each record we have 80 + 128 = 208 heap bytes. + const int MemorySizePerObject = 208; + const int RecordSize = 32; + + // Add one record to verify expected memory size. + db.HashSet("u00", [new HashEntry("Title", "Faster")]); + string r = db.HashGet("u00", "Title"); + ClassicAssert.AreEqual("Faster", r); + + ClassicAssert.AreEqual(MemorySizePerObject, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + ClassicAssert.AreEqual(MinLogAllocatedPageCount, cacheSizeTracker.mainLogTracker.logAccessor.AllocatedPageCount); // Ensure APC hasn't changed as memory is still within the min & max limits + + // Inline size: + // K/V lengths fit into a single byte each, so the record size is: RecordInfo, MinLengthMetadataBytes, valueLength, keyLength, objectLogPosition; the total rounded up to record alignment. + // ValueLength is 4 for the ObjectId, so this becomes 8 + 5 + 3(key) + 4(value) totalling 20, plus 8 for objectLogPosition totaling 32 which is already rounded to record alignment + // and is a even divisor for the page size. First valid address is 64, so a 512b page allows 14 records evenly; a memory size of 2k allows 56 total records. + Assert.That(store.Log.TailAddress, Is.EqualTo(PageHeader.Size + RecordSize)); + + // Heap size: + // MemorySizePerEntry is 208 heap bytes per record, which x 14 records per page is 2912 heap bytes per page. + // Total size: + // Per-page, heap size plus the 512 bytes of the page itself is 3424 tatal bytes per page. + // We've limited ourselves to 9k memory size; so initially we'll have two fully allocated pages (6848 bytes) and then the third page will be partially allocated with the remaining memory + // before we need to start HeadAddress moving up (by 32 bytes each record), and then we'll evict pages as we pass their boundaries. For this test, the heap size per page being larger than + // the inline page size makes it easy to track (a page's inline size is less than 3 records' worth of heap data), so we'll stay at 3 pages and advance HeadAddress. + + // Allocate the first two pages (remember we added one record above); we should not evict. + int numRecords = 28; + for (var ii = 1; ii < numRecords; ii++) + db.HashSet($"u{ii:00}", [new HashEntry("Title", "Faster")]); + Assert.That(evicted, Is.False, "Eviction should not have occurred yet"); + Assert.That(cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, Is.EqualTo(numRecords * MemorySizePerObject)); + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo(numRecords * MemorySizePerObject + 3 * PageSize)); // We are at the end of the second page and have the "allocate one page ahead" allocated + Assert.That(store.Log.HeadAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.Log.TailAddress, Is.EqualTo(PageSize * 2)); + + var (highTarget, lowTarget) = cacheSizeTracker.mainLogTracker.TargetDeltaRange; + Assert.That(highTarget, Is.EqualTo(9900)); + Assert.That(lowTarget, Is.EqualTo(8820)); + var remaining = highTarget - cacheSizeTracker.mainLogTracker.TotalSize; + Assert.That(remaining, Is.EqualTo(2540)); + + // Our next allocation will add a page and thus subtract 512 from our budget, leaving 2028. This is enough for 9 more records (1872 bytes), which will put us in the middle of the third page + // with 156 bytes remaining in our budget. We should still not have evicted yet. + int batchSize = 9; + for (var ii = 0; ii < batchSize; ii++) + db.HashSet($"u{ii + numRecords:00}", [new HashEntry("Title", "Faster")]); + numRecords += batchSize; + Assert.That(numRecords, Is.EqualTo(37)); + Assert.That(evicted, Is.False, "Eviction should not have occurred yet"); + Assert.That(cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, Is.EqualTo(numRecords * MemorySizePerObject)); + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo(numRecords * MemorySizePerObject + 4 * PageSize)); // We are now on the third page and have the "allocate one page ahead" allocated + Assert.That(store.Log.HeadAddress, Is.EqualTo(PageHeader.Size)); + Assert.That(store.Log.TailAddress, Is.EqualTo(PageSize * 2 + PageHeader.Size + RecordSize * batchSize)); + + // Now we start evicting. Add one record, then wait for the eviction of one record. This will signal the completion event. + // Eviction will proceed until it goes at or below lowTarget. We will go 208-156=52 bytes over size, then evict until we have gained + // 9900-8820+52 (highTarget - lowTarget plus overage) = 1132 bytes, which is more than 5 records so we'll evict 6, leaving us with + // a total size of 6*208-1132=116 bytes under the low target. + db.HashSet($"u{numRecords++:00}", [new HashEntry("Title", "Faster")]); + Assert.That(numRecords, Is.EqualTo(38)); + Assert.That(epcEvent.Wait(TimeSpan.FromSeconds(2 * LogSizeTracker.ResizeTaskDelaySeconds)), Is.True, "Timeout occurred. Resizing did not happen within the specified time, pt 1"); + Assert.That(evicted, Is.True, "Eviction should have occurred"); + var evictedRecords = 6; + + // HeadAddress will have advanced those 6 records, but we've added only one record so Tail will only grow by one. + batchSize = 6; // reuse this for the "batch" of evicted records + Assert.That(store.Log.HeadAddress, Is.EqualTo(PageHeader.Size + RecordSize * batchSize)); + Assert.That(store.Log.TailAddress, Is.EqualTo(PageSize * 2 + PageHeader.Size + RecordSize * 10)); // was 9 records in, now 10 + Assert.That(cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, Is.EqualTo((numRecords - evictedRecords) * MemorySizePerObject)); + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo((numRecords - batchSize) * MemorySizePerObject + 4 * PageSize)); // We are now on the third page and have the "allocate one page ahead" allocated + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo(lowTarget - 116)); // alternate verification of TotalSize, calculated from expected eviction amount + + // We have space for 4 records on the current page, and we are 116 bytes below lowTarget, and highTarget-lowTarget = 1080, + // so we have 1196 / 208 = 5 records worth of budget with 156 bytes left over. So add 6 records, which will go over budget by + // 208-156=52 again, so again we will evict 6 records, with HeadAddress staying on the same page. + evicted = false; + epcEvent.Reset(); + batchSize = 6; + for (var ii = 0; ii < batchSize; ii++) + db.HashSet($"u{ii + numRecords:00}", [new HashEntry("Title", "Faster")]); + numRecords += batchSize; + Assert.That(numRecords, Is.EqualTo(44)); + evictedRecords += 6; + Assert.That(evictedRecords, Is.EqualTo(12)); + + Assert.That(epcEvent.Wait(TimeSpan.FromSeconds(2 * LogSizeTracker.ResizeTaskDelaySeconds)), Is.True, "Timeout occurred. Resizing did not happen within the specified time, pt 2"); + Assert.That(evicted, Is.True, "Eviction should have occurred"); + + // HeadAddress was at 6 records into its page, so it has 8 records to go on the page; we evicted 6 of them, so are now 12 in. + Assert.That(store.Log.HeadAddress, Is.EqualTo(PageHeader.Size + RecordSize * 12)); + Assert.That(store.Log.TailAddress, Is.EqualTo(PageSize * 3 + PageHeader.Size + RecordSize * 2)); // We are two records into the fourth page + Assert.That(cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, Is.EqualTo((numRecords - evictedRecords) * MemorySizePerObject)); + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo((numRecords - evictedRecords) * MemorySizePerObject + 4 * PageSize)); // We are now on the third page and have the "allocate one page ahead" allocated + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo(lowTarget - 116)); // alternate verification of TotalSize, calculated from expected eviction amount + + // As before, evicting 6 left us with a total size of 6*208-1132=116 bytes under the low target, so have 1196 / 208 = 5 records worth of budget with 156 bytes left over. + // So add 6 records, which will go over budget by 208-156=52 again. This time, however, we have only 2 records for HeadAddress to advance before it can evict its page, + // which is more than twice a record size. That means we can evict to clear up the 1132 bytes by evicting 2 records for 416 bytes, then the page for 512 bytes, which + // totals 928 bytes leaving us still 204 bytes over budget, which means HeadAddress will advance one record in to the next page (rather than 4 in, which it would be if + // we could not reclaim the page space). + evicted = false; + epcEvent.Reset(); + batchSize = 6; + for (var ii = 0; ii < batchSize; ii++) + db.HashSet($"u{ii + numRecords:00}", [new HashEntry("Title", "Faster")]); + numRecords += batchSize; + evictedRecords += 3; + Assert.That(evictedRecords, Is.EqualTo(15)); + Assert.That(numRecords, Is.EqualTo(50)); + + Assert.That(epcEvent.Wait(TimeSpan.FromSeconds(2 * LogSizeTracker.ResizeTaskDelaySeconds)), Is.True, "Timeout occurred. Resizing did not happen within the specified time, pt 3"); + Assert.That(evicted, Is.True, "Eviction should have occurred"); + + // HeadAddress should be one record in to its new page. + Assert.That(store.Log.HeadAddress, Is.EqualTo(PageSize + PageHeader.Size + RecordSize)); + Assert.That(store.Log.TailAddress, Is.EqualTo(PageSize * 3 + PageHeader.Size + RecordSize * 8)); // We were two records into the fourth page, now we're 8 records in + Assert.That(cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, Is.EqualTo((numRecords - evictedRecords) * MemorySizePerObject)); + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo((numRecords - evictedRecords) * MemorySizePerObject + 3 * PageSize)); // We trimmed an allocated page + Assert.That(cacheSizeTracker.mainLogTracker.TotalSize, Is.EqualTo(lowTarget - 4)); // alternate verification of TotalSize, calculated from expected eviction amount + } + + [Test] + [Explicit("Revivification for readcache: update to be like SmallMainLogFineGrainedEvictionTest for readcache")] + public void ReadCacheIncreaseEmptyPageCountTest() + { + server?.Dispose(); + + // Create with a main-log heapMemorySize we won't hit, just to instantiate the tracker to ensure record heapMemory size. + // memorySizeStr is 1GB for that, while readCache has only 1k for its limit; both have 1k for inline pages (hence pageCounts are 2) + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, memorySize: "1GB", pageSize: "512", pageCount: 2, lowMemory: true, indexSize: "1k", + readCacheMemorySize: "2k", readCachePageSize: "1k", readCachePageCount: 2, enableReadCache: true); + + server.Start(); + store = server.Provider.StoreWrapper.store; + cacheSizeTracker = server.Provider.StoreWrapper.sizeTracker; + + var readCacheEmptyPageCountIncrements = 0; + using var readCacheEpcEvent = new ManualResetEventSlim(false); + + cacheSizeTracker.readCacheTracker.PostMemoryTrim = (allocatedPageDCount, headAddress) => { readCacheEmptyPageCountIncrements++; readCacheEpcEvent.Set(); }; + + ClassicAssert.AreEqual(0, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); + ClassicAssert.AreEqual(MinLogAllocatedPageCount, cacheSizeTracker.readCacheTracker.logAccessor.AllocatedPageCount); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + + // This will count only the value object; there is no key overflow. + const int MemorySizePerEntry = 208; + + // K/V lengths fit into a single byte each, so the record size is: RecordInfo, MinLengthMetadataSize, keyLength, valueLength; the total rounded up to record alignment. + // RecordInfo.Size + MinLengthMetadataSize (5) + keyLength (12) + valueLength (4) + ObjectLogPosition (8) rounded up to record alignment (8) = 40 + // With PageHeader.Size (64) and 1024-byte memory with 512-byte pages, we can fit 11 records per page: (512 - 64 = 448) / 40 = 11 (with 8 bytes left over) + const int InlineRecordSize = 40; + + // Insert one record to verify MemorySizePerEntry + db.HashSet($"usr{0:000}", [new HashEntry("Title", "Faster")]); + ClassicAssert.AreEqual(MemorySizePerEntry, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + + // Insert the rest of the records, enough to spill over to disk. + for (var i = 1; i < 100; i++) + db.HashSet($"usr{i:000}", [new HashEntry("Title", "Faster")]); + + var info = TestUtils.GetStoreAddressInfo(redis.GetServer(TestUtils.EndPoint), includeReadCache: true); + ClassicAssert.AreEqual(PageHeader.Size, info.ReadCacheTailAddress); + + // Now read back the earlier records, which were evicted to disk and will come back into the readcache. With 20 we will have one full and one partial page. + const int NumReadCacheRecords = 20; + for (var i = 0; i < NumReadCacheRecords; i++) + { + var value = db.HashGet($"usr{i:000}", "Title"); + ClassicAssert.AreEqual("Faster", (string)value, i.ToString()); + } + ClassicAssert.AreEqual(NumReadCacheRecords * MemorySizePerEntry, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); + + // We have two pages in the read cache now: one full (11 records) and one partial (9 records). So we will have one 8-byte leftover at the end of the first page. + info = TestUtils.GetStoreAddressInfo(redis.GetServer(TestUtils.EndPoint), includeReadCache: true); + ClassicAssert.AreEqual(PageHeader.Size * 2 + InlineRecordSize * NumReadCacheRecords + 8, info.ReadCacheTailAddress); + + if (!readCacheEpcEvent.Wait(TimeSpan.FromSeconds(3 * 3 * LogSizeTracker.ResizeTaskDelaySeconds))) + Assert.Fail("Timeout occurred. Resizing did not happen within the specified time."); + + ClassicAssert.AreEqual(1, readCacheEmptyPageCountIncrements); + // The first page of the read cache has been evicted => 11 records removed, 9 remain. + ClassicAssert.AreEqual(9 * MemorySizePerEntry, cacheSizeTracker.readCacheTracker.LogHeapSizeBytes); + } + + /// + /// Verifies that removing the last element from an object collection (triggering HasRemoveKey → + /// ExpireAndStop) correctly returns the heap tracker to zero. This exercises the IPU path where + /// Operate mutates the object in-place, the sizeChange delta is applied before the ExpireAndStop + /// early return, and OnDispose(Deleted) subtracts the remaining empty-collection overhead. + /// + [Test] + public void RemoveLastElementReturnsTrackerToZero_IPU() + { + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add a single element to a set — creates the object record with heap tracking. + db.SetAdd("myset", "value1"); + var heapAfterAdd = cacheSizeTracker.mainLogTracker.LogHeapSizeBytes; + ClassicAssert.Greater(heapAfterAdd, 0, "Heap should be positive after SADD"); + + // Remove the only element — triggers HasRemoveKey → ExpireAndStop → tombstone. + var removed = db.SetRemove("myset", "value1"); + ClassicAssert.IsTrue(removed, "SREM should return true"); + + // The tracker should return to zero: the sizeChange delta from SREM (negative) plus + // OnDispose(Deleted) subtracting the empty-collection overhead should exactly cancel + // the original SADD increment. + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, + "Heap tracker must return to zero after removing the last element (IPU path)"); + } + + /// + /// Same scenario but with the record in the readonly region, forcing the remove-last-element + /// through CopyUpdate → PostCopyUpdater → HasRemoveKey → ExpireAndStop. The PCU path's + /// tombstoned new record must not leak value heap because +value is only added on pcuSuccess=true. + /// + [Test] + public void RemoveLastElementReturnsTrackerToZero_PCU() + { + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Add a single element to a list. + db.ListRightPush("mylist", "value1"); + var heapAfterAdd = cacheSizeTracker.mainLogTracker.LogHeapSizeBytes; + ClassicAssert.Greater(heapAfterAdd, 0, "Heap should be positive after RPUSH"); + + // Shift the record to the readonly region so the next mutation goes through CopyUpdate. + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Remove the only element — forces CopyUpdate → PCU → HasRemoveKey → ExpireAndStop. + var popped = db.ListLeftPop("mylist"); + ClassicAssert.AreEqual("value1", (string)popped, "LPOP should return the value"); + + // Force eviction to flush any remaining sealed source records. + store.Log.FlushAndEvict(wait: true); + + ClassicAssert.AreEqual(0, cacheSizeTracker.mainLogTracker.LogHeapSizeBytes, + "Heap tracker must return to zero after removing the last element (PCU path)"); + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/DiskANN/DiskANNServiceTests.cs b/test/standalone/Garnet.test.extensions/DiskANN/DiskANNServiceTests.cs similarity index 99% rename from test/Garnet.test/DiskANN/DiskANNServiceTests.cs rename to test/standalone/Garnet.test.extensions/DiskANN/DiskANNServiceTests.cs index 383f798f3b1..a76a279606d 100644 --- a/test/Garnet.test/DiskANN/DiskANNServiceTests.cs +++ b/test/standalone/Garnet.test.extensions/DiskANN/DiskANNServiceTests.cs @@ -9,7 +9,6 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -17,9 +16,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class DiskANNServiceTests : AllureTestBase + public class DiskANNServiceTests : TestBase { private delegate void ReadCallbackDelegate(ulong context, uint numKeys, nint keysData, nuint keysLength, nint dataCallback, nint dataCallbackContext); private delegate byte WriteCallbackDelegate(ulong context, nint keyData, nuint keyLength, nint writeData, nuint writeLength); diff --git a/test/Garnet.test/DiskANN/DiskANNSyntheticRecallTests.cs b/test/standalone/Garnet.test.extensions/DiskANN/DiskANNSyntheticRecallTests.cs similarity index 99% rename from test/Garnet.test/DiskANN/DiskANNSyntheticRecallTests.cs rename to test/standalone/Garnet.test.extensions/DiskANN/DiskANNSyntheticRecallTests.cs index 00fc01c205a..6bd6bc415ea 100644 --- a/test/Garnet.test/DiskANN/DiskANNSyntheticRecallTests.cs +++ b/test/standalone/Garnet.test.extensions/DiskANN/DiskANNSyntheticRecallTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Globalization; using System.Linq; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -17,9 +16,8 @@ namespace Garnet.test /// Generates vectors → VADD to Garnet → VSIM query every vector → /// compare results against brute-force nearest neighbors → assert recall ≥ threshold. /// - [AllureNUnit] [TestFixture] - public class DiskANNSyntheticRecallTests : AllureTestBase + public class DiskANNSyntheticRecallTests : TestBase { private GarnetServer server; diff --git a/test/standalone/Garnet.test.extensions/Garnet.test.extensions.csproj b/test/standalone/Garnet.test.extensions/Garnet.test.extensions.csproj new file mode 100644 index 00000000000..aebeb903f2e --- /dev/null +++ b/test/standalone/Garnet.test.extensions/Garnet.test.extensions.csproj @@ -0,0 +1,53 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + false + + + + diff --git a/test/Garnet.test/GarnetJSON/JSONPath/JsonAssert.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonAssert.cs similarity index 100% rename from test/Garnet.test/GarnetJSON/JSONPath/JsonAssert.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonAssert.cs diff --git a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathExecuteTests.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathExecuteTests.cs similarity index 99% rename from test/Garnet.test/GarnetJSON/JSONPath/JsonPathExecuteTests.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathExecuteTests.cs index 086ec92af45..0eb83674310 100644 --- a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathExecuteTests.cs +++ b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathExecuteTests.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + #region License // Copyright (c) 2007 James Newton-King @@ -30,16 +33,14 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; -using Allure.NUnit; using GarnetJSON.JSONPath; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test.JSONPath { - [AllureNUnit] [TestFixture] - public class JsonPathExecuteTests : AllureTestBase + public class JsonPathExecuteTests : TestBase { [Test] public void GreaterThanIssue1518() diff --git a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathParseTests.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathParseTests.cs similarity index 99% rename from test/Garnet.test/GarnetJSON/JSONPath/JsonPathParseTests.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathParseTests.cs index 814ce019fd8..2ab278bf8dd 100644 --- a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathParseTests.cs +++ b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathParseTests.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + #region License // Copyright (c) 2007 James Newton-King @@ -29,16 +32,14 @@ using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; -using Allure.NUnit; using GarnetJSON.JSONPath; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test.JSONPath { - [AllureNUnit] [TestFixture] - public class JsonPathParseTests : AllureTestBase + public class JsonPathParseTests : TestBase { [Test] public void BooleanQuery_TwoValues() diff --git a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathRegressionTests.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathRegressionTests.cs similarity index 97% rename from test/Garnet.test/GarnetJSON/JSONPath/JsonPathRegressionTests.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathRegressionTests.cs index ddb7b51ac64..a195ab2a065 100644 --- a/test/Garnet.test/GarnetJSON/JSONPath/JsonPathRegressionTests.cs +++ b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/JsonPathRegressionTests.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + #region License // Copyright (c) 2007 James Newton-King @@ -32,7 +35,6 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; -using Allure.NUnit; using GarnetJSON.JSONPath; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -40,9 +42,8 @@ namespace Garnet.test.JSONPath { - [AllureNUnit] [TestFixture] - public class JsonPathRegressionTests : AllureTestBase + public class JsonPathRegressionTests : TestBase { public class RegressionTestQuery { diff --git a/test/Garnet.test/GarnetJSON/JSONPath/QueryExpressionTests.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/QueryExpressionTests.cs similarity index 98% rename from test/Garnet.test/GarnetJSON/JSONPath/QueryExpressionTests.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/QueryExpressionTests.cs index eff3a084533..7da0c0bc90a 100644 --- a/test/Garnet.test/GarnetJSON/JSONPath/QueryExpressionTests.cs +++ b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/QueryExpressionTests.cs @@ -1,4 +1,7 @@ -#region License +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +#region License // Copyright (c) 2007 James Newton-King // @@ -26,7 +29,6 @@ #endregion using System.Collections.Generic; using System.Text.Json.Nodes; -using Allure.NUnit; using GarnetJSON.JSONPath; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -34,9 +36,8 @@ namespace Garnet.test.JSONPath { - [AllureNUnit] [TestFixture] - public class QueryExpressionTests : AllureTestBase + public class QueryExpressionTests : TestBase { [Test] public void AndExpressionTest() diff --git a/test/Garnet.test/GarnetJSON/JSONPath/RegressionSuite.json b/test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/RegressionSuite.json similarity index 100% rename from test/Garnet.test/GarnetJSON/JSONPath/RegressionSuite.json rename to test/standalone/Garnet.test.extensions/GarnetJSON/JSONPath/RegressionSuite.json diff --git a/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs b/test/standalone/Garnet.test.extensions/GarnetJSON/JsonCommandsTest.cs similarity index 99% rename from test/Garnet.test/GarnetJSON/JsonCommandsTest.cs rename to test/standalone/Garnet.test.extensions/GarnetJSON/JsonCommandsTest.cs index 365fc795c0d..848dd0e5846 100644 --- a/test/Garnet.test/GarnetJSON/JsonCommandsTest.cs +++ b/test/standalone/Garnet.test.extensions/GarnetJSON/JsonCommandsTest.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; @@ -8,7 +8,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using GarnetJSON; using NUnit.Framework; @@ -17,9 +16,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - class JsonCommandsTest : AllureTestBase + class JsonCommandsTest : TestBase { GarnetServer server; string binPath; diff --git a/test/standalone/Garnet.test.extensions/GlobalUsings.cs b/test/standalone/Garnet.test.extensions/GlobalUsings.cs new file mode 100644 index 00000000000..35f46f7207d --- /dev/null +++ b/test/standalone/Garnet.test.extensions/GlobalUsings.cs @@ -0,0 +1,5 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +global using StoreAllocator = Tsavorite.core.ObjectAllocator>; +global using StoreFunctions = Tsavorite.core.StoreFunctions; \ No newline at end of file diff --git a/test/Garnet.test/IndexGrowthTests.cs b/test/standalone/Garnet.test.extensions/IndexGrowthTests.cs similarity index 87% rename from test/Garnet.test/IndexGrowthTests.cs rename to test/standalone/Garnet.test.extensions/IndexGrowthTests.cs index c3afca7907e..371dd281b6a 100644 --- a/test/Garnet.test/IndexGrowthTests.cs +++ b/test/standalone/Garnet.test.extensions/IndexGrowthTests.cs @@ -1,21 +1,19 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Threading; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class IndexGrowthTests : AllureTestBase + public class IndexGrowthTests : TestBase { GarnetServer server; - private int indexResizeTaskDelaySeconds = 10; + private int indexResizeTaskDelaySeconds = 5; private int indexResizeWaitCycles = 2; [SetUp] @@ -72,10 +70,10 @@ public void IndexGrowthTest() [Test] public void ObjectStoreIndexGrowthTest() { - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, objectStoreIndexSize: "64", objectStoreIndexMaxSize: "128", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, indexSize: "64", indexMaxSize: "128", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); server.Start(); - var objectStore = server.Provider.StoreWrapper.objectStore; + var store = server.Provider.StoreWrapper.store; RedisKey[] keys = ["abcdkey", "bcdekey", "cdefkey", "defgkey", "efghkey", "fghikey", "ghijkey", "hijkkey"]; RedisValue[] values = ["abcdval", "bcdeval", "cdefval", "defgval", "efghval", "fghival", "ghijval", "hijkval"]; @@ -83,8 +81,8 @@ public void ObjectStoreIndexGrowthTest() using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { var db = redis.GetDatabase(0); - ClassicAssert.AreEqual(0, objectStore.OverflowBucketAllocations); - ClassicAssert.AreEqual(1, objectStore.IndexSize); + ClassicAssert.AreEqual(0, store.OverflowBucketAllocations); + ClassicAssert.AreEqual(1, store.IndexSize); for (int i = 0; i < keys.Length; i++) { @@ -92,16 +90,16 @@ public void ObjectStoreIndexGrowthTest() } VerifyObjectStoreSetMembers(db, keys, values); - ClassicAssert.AreEqual(1, objectStore.OverflowBucketAllocations); + ClassicAssert.AreEqual(1, store.OverflowBucketAllocations); // Wait for the resizing to happen for (int waitCycles = 0; waitCycles < indexResizeWaitCycles; waitCycles++) { Thread.Sleep(TimeSpan.FromSeconds(indexResizeTaskDelaySeconds)); - if (objectStore.IndexSize > 1) break; + if (store.IndexSize > 1) break; } - ClassicAssert.AreEqual(2, objectStore.IndexSize); + ClassicAssert.AreEqual(2, store.IndexSize); VerifyObjectStoreSetMembers(db, keys, values); } } @@ -134,9 +132,7 @@ public void IndexGrowthTestWithDiskReadAndCheckpoint() ClassicAssert.AreEqual(8, store.IndexSize); for (int i = 0; i < keys.Length; i++) - { db.StringSet(keys[i], values[i]); - } ClassicAssert.AreEqual(values[0], db.StringGet(keys[0]).ToString()); @@ -183,10 +179,10 @@ public void IndexGrowthTestWithDiskReadAndCheckpoint() [Test] public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() { - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, objectStoreIndexSize: "512", objectStoreIndexMaxSize: "1k", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, indexSize: "512", indexMaxSize: "1k", indexResizeFrequencySecs: indexResizeTaskDelaySeconds); server.Start(); - var objectStore = server.Provider.StoreWrapper.objectStore; + var store = server.Provider.StoreWrapper.store; RedisKey[] keys = ["abcdkey", "bcdekey", "cdefkey", "defgkey", "efghkey", "fghikey", "ghijkey", "hijkkey"]; RedisValue[] values = ["abcdval", "bcdeval", "cdefval", "defgval", "efghval", "fghival", "ghijval", "hijkval"]; @@ -195,12 +191,10 @@ public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { var db = redis.GetDatabase(0); - ClassicAssert.AreEqual(8, objectStore.IndexSize); + ClassicAssert.AreEqual(8, store.IndexSize); for (int i = 0; i < keys.Length; i++) - { db.SetAdd(keys[i], values[i]); - } VerifyObjectStoreSetMembers(db, keys, values); @@ -218,10 +212,10 @@ public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() for (int waitCycles = 0; waitCycles < indexResizeWaitCycles; waitCycles++) { Thread.Sleep(TimeSpan.FromSeconds(indexResizeTaskDelaySeconds)); - if (objectStore.IndexSize > 8) break; + if (store.IndexSize > 8) break; } - ClassicAssert.AreEqual(16, objectStore.IndexSize); + ClassicAssert.AreEqual(16, store.IndexSize); // Check if entry created before resizing is still accessible. VerifyObjectStoreSetMembers(db, keys, values); @@ -233,7 +227,7 @@ public void ObjectStoreIndexGrowthTestWithDiskReadAndCheckpoint() } server.Dispose(false); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, objectStoreIndexSize: "512", objectStoreIndexMaxSize: "1k"); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, indexSize: "512", indexMaxSize: "1k"); server.Start(); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) diff --git a/test/Garnet.test/ReadCacheTests.cs b/test/standalone/Garnet.test.extensions/ReadCacheTests.cs similarity index 58% rename from test/Garnet.test/ReadCacheTests.cs rename to test/standalone/Garnet.test.extensions/ReadCacheTests.cs index 981fc4267ad..0340cb14908 100644 --- a/test/Garnet.test/ReadCacheTests.cs +++ b/test/standalone/Garnet.test.extensions/ReadCacheTests.cs @@ -1,16 +1,14 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class ReadCacheTests : AllureTestBase + public class ReadCacheTests : TestBase { GarnetServer server; @@ -18,7 +16,7 @@ public class ReadCacheTests : AllureTestBase public void Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, enableObjectStoreReadCache: true, lowMemory: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableReadCache: true, lowMemory: true); server.Start(); } @@ -26,6 +24,7 @@ public void Setup() public void TearDown() { server.Dispose(); + server = null; TestUtils.OnTearDown(); } @@ -41,12 +40,14 @@ public void MainStoreReadCacheTest() var server = redis.GetServer(TestUtils.EndPoint); var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - // Start at tail address of 64 - ClassicAssert.AreEqual(64, info.ReadCacheBeginAddress); - ClassicAssert.AreEqual(64, info.ReadCacheTailAddress); + // Start at tail address after PageHeader (64) + const int PageHeaderSize = 64; + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheHeadAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheBeginAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheTailAddress); // Do enough writes to overflow memory to push records to disk - for (var i = 0; i < 100; i++) + for (var i = 0; i < 120; i++) { var key = $"key{i:00000}"; var value = $"val{i:00000}"; @@ -58,41 +59,48 @@ public void MainStoreReadCacheTest() ClassicAssert.Greater(info.HeadAddress, info.BeginAddress); // Read cache should not have been updated as there are no reads yet - ClassicAssert.AreEqual(64, info.ReadCacheTailAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheTailAddress); + + // Issue read of initial key to populate read cache. Record size is: + // RecordInfo.Size + NumIndicatorBytes (3) + 1 byte each for lengths + 8 bytes each for key and value + no optionals + roundup to record alignment + const int RecordSize = 32; - // Issue read of initial key to populate read cache var key0 = $"key00000"; var value0 = db.StringGet(key0); ClassicAssert.AreEqual("val00000", (string)value0); info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - ClassicAssert.AreEqual(64 + 40, info.ReadCacheTailAddress); // 40 bytes for one record + ClassicAssert.AreEqual(PageHeaderSize + RecordSize, info.ReadCacheTailAddress); // Issue read again to ensure read cache is not updated value0 = db.StringGet(key0); ClassicAssert.AreEqual("val00000", (string)value0); info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - ClassicAssert.AreEqual(64 + 40, info.ReadCacheTailAddress); + ClassicAssert.AreEqual(PageHeaderSize + RecordSize, info.ReadCacheTailAddress); // Read more keys to update read cache - for (var j = 1; j < 20; j++) + for (var j = 1; j < 40; j++) { var key = $"key{j:00000}"; var value = db.StringGet(key); ClassicAssert.AreEqual($"val{j:00000}", (string)value); } info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - ClassicAssert.AreEqual(64 + 40 * 20 + 8, info.ReadCacheTailAddress); // 40 bytes for 20 records + 8 bytes for page boundary alignment - ClassicAssert.AreEqual(64, info.ReadCacheBeginAddress); // Read cache should not have been evicted yet + // 32 bytes for 14 records plus PageHeader ends on page boundary so no bytes needed for 512b page alignment, but we pick up the next pages' headers. + ClassicAssert.AreEqual(PageHeaderSize * 3 + RecordSize * 40, info.ReadCacheTailAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheBeginAddress); // Read cache should not have been evicted yet + ClassicAssert.AreEqual(info.ReadCacheBeginAddress, info.ReadCacheHeadAddress); // Issue more reads to start evicting read cache entries - for (var j = 20; j < 40; j++) + for (var j = 40; j < 80; j++) { var key = $"key{j:00000}"; var value = db.StringGet(key); ClassicAssert.AreEqual($"val{j:00000}", (string)value); } + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - ClassicAssert.Greater(info.ReadCacheBeginAddress, 64); // Read cache entries should have been evicted + ClassicAssert.Greater(info.ReadCacheBeginAddress, PageHeaderSize); // Read cache entries should have been evicted + ClassicAssert.AreEqual(info.ReadCacheBeginAddress, info.ReadCacheHeadAddress); } [Test] @@ -101,11 +109,13 @@ public void ObjectStoreReadCacheTest() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); var db = redis.GetDatabase(0); var server = redis.GetServer(TestUtils.EndPoint); - var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); + var info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); - // Start at tail address of 24 - ClassicAssert.AreEqual(24, info.ReadCacheBeginAddress); - ClassicAssert.AreEqual(24, info.ReadCacheTailAddress); + // Start at tail address after PageHeader (64) + const int PageHeaderSize = 64; + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheBeginAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheHeadAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheTailAddress); // Do enough list pushes to overflow memory to push records to disk for (var i = 0; i < 100; i++) @@ -115,36 +125,41 @@ public void ObjectStoreReadCacheTest() _ = db.ListRightPush(key, value); } - info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); // Ensure data has spilled to disk ClassicAssert.Greater(info.HeadAddress, info.BeginAddress); // Read cache should not have been updated as there are no reads yet - ClassicAssert.AreEqual(24, info.ReadCacheTailAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheTailAddress); + + // Issue read of initial key to populate read cache. Record size is: + // RecordInfo.Size + NumIndicatorBytes (3) + 1 byte each for lengths + 11 bytes for key + 4 bytes for value (objectId) + objLogPosition (8) + no optionals + roundup to record alignment + const int RecordSize = 40; - // Issue read of initial key to populate read cache var key0 = $"objKey00000"; var value0 = db.ListGetByIndex(key0, 0); ClassicAssert.AreEqual("objVal00000", (string)value0); - info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); - ClassicAssert.AreEqual(24 + 24, info.ReadCacheTailAddress); // 24 bytes for one record + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); + ClassicAssert.AreEqual(PageHeaderSize + RecordSize, info.ReadCacheTailAddress); // Issue read again to ensure read cache is not updated value0 = db.ListGetByIndex(key0, 0); ClassicAssert.AreEqual("objVal00000", (string)value0); - info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); - ClassicAssert.AreEqual(24 + 24, info.ReadCacheTailAddress); + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); + ClassicAssert.AreEqual(PageHeaderSize + RecordSize, info.ReadCacheTailAddress); // Read more keys to update read cache - for (var j = 1; j < 40; j++) + for (var j = 1; j < 20; j++) { var key = $"objKey{j:00000}"; var value = db.ListGetByIndex(key, 0); ClassicAssert.AreEqual($"objVal{j:00000}", (string)value); } - info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); - ClassicAssert.AreEqual(24 + 24 * 40 + 8, info.ReadCacheTailAddress); // 24 bytes for 20 records + 8 bytes for page boundary alignment - ClassicAssert.AreEqual(24, info.ReadCacheBeginAddress); // Read cache should not have been evicted yet + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); + // 40 bytes for 11 records plus PageHeader ends 8 bytes short of page boundary so add 8 bytes needed for page alignment plus next page's header. + ClassicAssert.AreEqual(PageHeaderSize * 2 + RecordSize * 20 + 8, info.ReadCacheTailAddress); + ClassicAssert.AreEqual(PageHeaderSize, info.ReadCacheBeginAddress); // Read cache should not have been evicted yet + ClassicAssert.AreEqual(info.ReadCacheBeginAddress, info.ReadCacheHeadAddress); // Issue more reads to start evicting read cache entries for (var j = 40; j < 80; j++) @@ -153,8 +168,9 @@ public void ObjectStoreReadCacheTest() var value = db.ListGetByIndex(key, 0); ClassicAssert.AreEqual($"objVal{j:00000}", (string)value); } - info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true, isObjectStore: true); - ClassicAssert.Greater(info.ReadCacheBeginAddress, 24); // Read cache entries should have been evicted + info = TestUtils.GetStoreAddressInfo(server, includeReadCache: true); + ClassicAssert.Greater(info.ReadCacheBeginAddress, PageHeaderSize); // Read cache entries should have been evicted + ClassicAssert.AreEqual(info.ReadCacheBeginAddress, info.ReadCacheHeadAddress); } } } \ No newline at end of file diff --git a/test/Garnet.test/ReadOptimizedLockTests.cs b/test/standalone/Garnet.test.extensions/ReadOptimizedLockTests.cs similarity index 98% rename from test/Garnet.test/ReadOptimizedLockTests.cs rename to test/standalone/Garnet.test.extensions/ReadOptimizedLockTests.cs index 43e749b03e7..315723cf24d 100644 --- a/test/Garnet.test/ReadOptimizedLockTests.cs +++ b/test/standalone/Garnet.test.extensions/ReadOptimizedLockTests.cs @@ -1,20 +1,18 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; using System.Threading; -using Allure.NUnit; using Garnet.common; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class ReadOptimizedLockTests : AllureTestBase + public class ReadOptimizedLockTests : TestBase { [TestCase(123)] [TestCase(0)] diff --git a/test/Garnet.test/RespRevivificationTests.cs b/test/standalone/Garnet.test.extensions/RespRevivificationTests.cs similarity index 92% rename from test/Garnet.test/RespRevivificationTests.cs rename to test/standalone/Garnet.test.extensions/RespRevivificationTests.cs index 354fecff7c4..11b9ee75c1d 100644 --- a/test/Garnet.test/RespRevivificationTests.cs +++ b/test/standalone/Garnet.test.extensions/RespRevivificationTests.cs @@ -4,16 +4,14 @@ using System; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - class RespRevivificationTests : AllureTestBase + class RespRevivificationTests : TestBase { GarnetServer server; Random r; @@ -186,7 +184,8 @@ public async Task RevivificationWithRMWWorksForRecordPool() await db.ExecuteAsync("SET", "foo", "c", "PX", 500).ConfigureAwait(false); await db.ExecuteAsync("SET", "michael", "jordan").ConfigureAwait(false); - await db.ExecuteAsync("SET", "johnny", "x", "PX", 500).ConfigureAwait(false); // big record, that we need to make sure active exp picks up for revivification + // A big record that is above minRevivifiable address; after active exp tombstones it, revivification should pick it up + await db.ExecuteAsync("SET", "arnold", "schwarzenegger", "PX", 500).ConfigureAwait(false); await Task.Delay(600).ConfigureAwait(false); // wait for the keys to expire @@ -195,21 +194,21 @@ public async Task RevivificationWithRMWWorksForRecordPool() // when deleting via above, are we potentially resetting the value and it's metadata and stuff? ClassicAssert.IsTrue(exec.Resp2Type != ResultType.Error); - // attempt to do an RCU operation that will reuse the tombstoned record via Recordpool - await db.ExecuteAsync("SETIFGREATER", "michael", "j", 23).ConfigureAwait(false); + // Do a new-record operation that will reuse a tombstoned record via FreeRecordPool + await db.ExecuteAsync("SETIFGREATER", "the", "terminator", 23).ConfigureAwait(false); // confirm we did indeed use a reviv record var stats = await db.ExecuteAsync("INFO", "STOREREVIV").ConfigureAwait(false); - ClassicAssert.IsTrue(stats.ToString().Contains("Successful Takes: 1"), "Expected in-chain revivification to happen, but it did not."); + ClassicAssert.IsTrue(stats.ToString().Contains("Successful Takes: 1"), "Expected FreeRecord revivification to happen, but it did not."); - var res = (RedisResult[])await db.ExecuteAsync("GETWITHETAG", "michael").ConfigureAwait(false); + var res = (RedisResult[])await db.ExecuteAsync("GETWITHETAG", "the").ConfigureAwait(false); ClassicAssert.AreEqual(23, (long)res[0], "Incorrect Etag."); - ClassicAssert.AreEqual("j", res[1].ToString(), "Expected the value to be updated via RMW operation, but it was not."); + ClassicAssert.AreEqual("terminator", res[1].ToString(), "Expected the value to be updated via RMW operation, but it was not."); } [Test] - public async Task RevivifiedRecordsShouldStillEnqueueToAofViaRmwAndClearEtagState() + public async Task RevivifiedRecordsShouldStillEnqueueToAofViaRmw() { server.Dispose(false); SetupServerWithReviv(inChainOnly: true); @@ -218,24 +217,25 @@ public async Task RevivifiedRecordsShouldStillEnqueueToAofViaRmwAndClearEtagStat var db = redis.GetDatabase(0); - long startingAddr = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long startingAddr = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; await db.StringSetAsync("arnold", "schwarzeneggar").ConfigureAwait(false); - long tailAddrAfterInsert = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterInsert = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterInsert > startingAddr, "Expected AOF tail address to move forward on initial SET"); // setup keys for revivifying await db.StringSetAsync("hoo", "kachakahookahooka").ConfigureAwait(false); // both would move the tail address of the AOF forward - long tailAddrAfterInsert2 = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterInsert2 = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterInsert2 > tailAddrAfterInsert, "Expected AOF tail address to move forward on initial SET"); + await db.KeyDeleteAsync("hoo").ConfigureAwait(false); - long tailAddrAfterDelete = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterDelete = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterDelete > tailAddrAfterInsert2, "Expected AOF tail address to move forward on DELETE"); // inchain revivification of the exact same key should take place await db.ExecuteAsync("SETIFGREATER", "hoo", "b", "1").ConfigureAwait(false); - long tailAddrAfterRevivifyRmw = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterRevivifyRmw = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterRevivifyRmw > tailAddrAfterDelete, "Expected AOF tail address to move forward on revivification RMW"); // the unrelated read should not be affected by revivification state @@ -259,22 +259,22 @@ public async Task RevivifiedRecordsShouldStillEnqueueToAofViaUpsert() await db.StringSetAsync("fizz", "buzz").ConfigureAwait(false); - long startingAddr = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long startingAddr = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; // setup keys for revivifying await db.StringSetAsync("hoo", "kachakahookahooka").ConfigureAwait(false); // both would move the tail address of the AOF forward - long tailAddrAfterInsert = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterInsert = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterInsert > startingAddr, "Expected AOF tail address to move forward on initial SET"); // in-chain tombstone await db.KeyDeleteAsync("hoo").ConfigureAwait(false); - long tailAddrAfterDelete = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterDelete = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterDelete > tailAddrAfterInsert, "Expected AOF tail address to move forward on DELETE"); // do an in-chain revivification await db.StringSetAsync("hoo", "b").ConfigureAwait(false); - long tailAddrAfterRevivifySet = server.Provider.StoreWrapper.appendOnlyFile.TailAddress; + long tailAddrAfterRevivifySet = server.Provider.StoreWrapper.appendOnlyFile.Log.TailAddress[0]; ClassicAssert.IsTrue(tailAddrAfterRevivifySet > tailAddrAfterDelete, "Expected AOF tail address to move forward on revivification SET"); // make sure revivification stats reflect that things were indeed revivified diff --git a/test/Garnet.test/ScratchBufferAllocatorTests.cs b/test/standalone/Garnet.test.extensions/ScratchBufferAllocatorTests.cs similarity index 94% rename from test/Garnet.test/ScratchBufferAllocatorTests.cs rename to test/standalone/Garnet.test.extensions/ScratchBufferAllocatorTests.cs index a1f50a1379a..624e699c236 100644 --- a/test/Garnet.test/ScratchBufferAllocatorTests.cs +++ b/test/standalone/Garnet.test.extensions/ScratchBufferAllocatorTests.cs @@ -1,16 +1,14 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class ScratchBufferAllocatorTests : AllureTestBase + public class ScratchBufferAllocatorTests : TestBase { [Test] @@ -20,7 +18,7 @@ public void CreateArgSliceAndRewindTest([Values(0, 127, 8192)] int maxInitialCap var string2 = new string('b', 65); var string3 = new string('c', 6000); - var sam = new ScratchBufferAllocator(maxInitialCapacity: maxInitialCapacity); + var sam = new ScratchBufferAllocator(minSizeBuffer: 2, maxInitialCapacity: maxInitialCapacity); // Data of length 5 - SAM creates a buffer of size 8 var as1 = sam.CreateArgSlice(string1); diff --git a/test/standalone/Garnet.test.extensions/TestProjectSetup.cs b/test/standalone/Garnet.test.extensions/TestProjectSetup.cs new file mode 100644 index 00000000000..16338f541f0 --- /dev/null +++ b/test/standalone/Garnet.test.extensions/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestExtensions); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.rangeindex/Garnet.test.rangeindex.csproj b/test/standalone/Garnet.test.rangeindex/Garnet.test.rangeindex.csproj new file mode 100644 index 00000000000..340c2ffce68 --- /dev/null +++ b/test/standalone/Garnet.test.rangeindex/Garnet.test.rangeindex.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/standalone/Garnet.test.rangeindex/RespRangeIndexTests.cs b/test/standalone/Garnet.test.rangeindex/RespRangeIndexTests.cs new file mode 100644 index 00000000000..8ba41be175c --- /dev/null +++ b/test/standalone/Garnet.test.rangeindex/RespRangeIndexTests.cs @@ -0,0 +1,2448 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; + +namespace Garnet.test +{ + /// + /// Integration tests for RangeIndex (RI.*) commands. + /// + /// These tests cover the full lifecycle of BfTree-backed range indexes including: + /// creation, field CRUD, type safety (WRONGTYPE), eviction, flush/promote, lazy restore, + /// checkpoint/recovery, AOF replay, concurrent access, and diagnostic commands. + /// + /// Each test creates a fresh Garnet server in Setup and disposes it in + /// TearDown. Tests that need low memory, AOF, or checkpoint recovery + /// recreate the server with the appropriate options. + /// + [TestFixture] + public class RespRangeIndexTests : TestBase + { + GarnetServer server; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + /// + /// Verifies basic RI.CREATE with MEMORY backend succeeds. + /// + [Test] + public void RICreateBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // RI.CREATE with MEMORY backend + var result = db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536"); + ClassicAssert.AreEqual("OK", (string)result); + } + + /// + /// Verifies that calling RI.CREATE on an existing key returns an error + /// ("ERR index already exists") and does not overwrite the existing index. + /// + [Test] + public void RICreateDuplicateReturnsErrorTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create first time + var result = db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536"); + ClassicAssert.AreEqual("OK", (string)result); + + // Create again - should fail with error + var ex = Assert.Throws(() => + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536")); + ClassicAssert.IsTrue(ex.Message.Contains("index already exists")); + } + + /// + /// Verifies that DEL on a RangeIndex key removes the key and frees the BfTree. + /// Subsequent RI.SET/RI.GET on the deleted key should return errors. + /// A second DEL returns false (key no longer exists). + /// + [Test] + public void RICreateThenDeleteTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create a range index and insert data + var result = db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + ClassicAssert.AreEqual("OK", (string)result); + result = db.Execute("RI.SET", "myindex", "field1", "value1"); + ClassicAssert.AreEqual("OK", (string)result); + + // Delete the index with DEL + var deleted = db.KeyDelete("myindex"); + ClassicAssert.IsTrue(deleted); + + // Delete again - should return false (not found) + deleted = db.KeyDelete("myindex"); + ClassicAssert.IsFalse(deleted); + + // RI.SET should fail — index no longer exists after DEL + var ex = Assert.Throws(() => + db.Execute("RI.SET", "myindex", "field1", "value1")); + ClassicAssert.IsNotNull(ex); + + // RI.GET should also fail + ex = Assert.Throws(() => + db.Execute("RI.GET", "myindex", "field1")); + ClassicAssert.IsNotNull(ex); + } + + /// + /// Verifies RI.CREATE with minimal arguments (defaults for all numeric parameters). + /// + [Test] + public void RICreateWithDefaultsTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // RI.CREATE with minimal args (defaults) + var result = db.Execute("RI.CREATE", "myindex", "MEMORY"); + ClassicAssert.AreEqual("OK", (string)result); + } + + /// + /// Verifies RI.CREATE with all optional parameters explicitly specified. + /// + [Test] + public void RICreateWithAllOptionsTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // RI.CREATE with all options (PAGESIZE auto-computed from MAXRECORD) + var result = db.Execute("RI.CREATE", "myindex", "MEMORY", + "CACHESIZE", "131072", + "MINRECORD", "8", + "MAXRECORD", "1024", + "MAXKEYLEN", "128"); + ClassicAssert.AreEqual("OK", (string)result); + } + + /// + /// Verifies basic RI.SET + RI.GET round-trip: set a field, then read it back. + /// + [Test] + public void RISetAndGetBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + + var setResult = db.Execute("RI.SET", "myindex", "field1", "value1"); + ClassicAssert.AreEqual("OK", (string)setResult); + + var getResult = db.Execute("RI.GET", "myindex", "field1"); + ClassicAssert.AreEqual("value1", (string)getResult); + } + + /// + /// Verifies that RI.SET overwrites existing field values (upsert semantics). + /// + [Test] + public void RISetOverwriteTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "field1", "value1"); + db.Execute("RI.SET", "myindex", "field1", "value2"); + + var getResult = db.Execute("RI.GET", "myindex", "field1"); + ClassicAssert.AreEqual("value2", (string)getResult); + } + + /// + /// Verifies that RI.GET on a non-existent field returns null (not an error). + /// + [Test] + public void RIGetNonExistentFieldTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + + var getResult = db.Execute("RI.GET", "myindex", "nosuchfield"); + ClassicAssert.IsTrue(getResult.IsNull); + } + + /// + /// Verifies that RI.GET on a non-existent key returns an error (not null). + /// + [Test] + public void RIGetNonExistentIndexTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var ex = Assert.Throws(() => + db.Execute("RI.GET", "noindex", "field1")); + ClassicAssert.IsTrue(ex.Message.Contains("range index")); + } + + /// + /// Verifies that RI.DEL removes a field and subsequent RI.GET returns null. + /// + [Test] + public void RIDelFieldTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "field1", "value1"); + + var delResult = (int)db.Execute("RI.DEL", "myindex", "field1"); + ClassicAssert.AreEqual(1, delResult); + + var getResult = db.Execute("RI.GET", "myindex", "field1"); + ClassicAssert.IsTrue(getResult.IsNull); + } + + /// + /// Verifies that RI.SET on a non-existent key returns an error. + /// + [Test] + public void RISetOnNonExistentIndexTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var ex = Assert.Throws(() => + db.Execute("RI.SET", "noindex", "field1", "value1")); + ClassicAssert.IsTrue(ex.Message.Contains("range index")); + } + + /// + /// Verifies that multiple fields can be independently stored and retrieved. + /// + [Test] + public void RIMultipleFieldsTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + db.Execute("RI.SET", "myindex", "ccc", "val-c"); + + ClassicAssert.AreEqual("val-a", (string)db.Execute("RI.GET", "myindex", "aaa")); + ClassicAssert.AreEqual("val-b", (string)db.Execute("RI.GET", "myindex", "bbb")); + ClassicAssert.AreEqual("val-c", (string)db.Execute("RI.GET", "myindex", "ccc")); + } + + /// + /// Verifies WRONGTYPE error when RI.SET is used on a normal string key. + /// + [Test] + public void RIWrongTypeOnNormalKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // SET a normal string key + db.StringSet("normalkey", "hello"); + + // RI.SET on a normal key should fail + var ex = Assert.Throws(() => + db.Execute("RI.SET", "normalkey", "field1", "value1")); + ClassicAssert.IsNotNull(ex); + } + + /// + /// Verifies WRONGTYPE error when RI.GET is used on a normal string key. + /// + [Test] + public void RIWrongTypeGetOnNormalKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // SET a normal string key, then try RI.GET + db.StringSet("normalkey", "hello"); + + var ex = Assert.Throws(() => + db.Execute("RI.GET", "normalkey", "field1")); + ClassicAssert.IsNotNull(ex); + } + + /// + /// Verifies WRONGTYPE error when a normal GET is used on a RangeIndex key. + /// + [Test] + public void RINormalGetOnRangeIndexKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536"); + + // GET on a RI key returns WRONGTYPE error + var ex = Assert.Throws(() => db.StringGet("myindex")); + ClassicAssert.IsTrue(ex.Message.StartsWith("WRONGTYPE")); + } + + /// + /// Verifies WRONGTYPE error when SET is used on an existing RangeIndex key, + /// and confirms the RI key's data is not corrupted. + /// + [Test] + public void RINormalSetOnRangeIndexKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "field1", "value1"); + + // SET on a RI key returns WRONGTYPE error + var ex = Assert.Throws(() => db.StringSet("myindex", "overwrite")); + ClassicAssert.IsTrue(ex.Message.StartsWith("WRONGTYPE")); + + // Verify the RI key is still intact + var val = db.Execute("RI.GET", "myindex", "field1"); + ClassicAssert.AreEqual("value1", (string)val); + } + + /// + /// Verifies AOF replay: checkpoint base state, then apply post-checkpoint mutations + /// (RI.SET update, RI.SET insert, RI.DEL) via AOF replay on recovery. + /// + [Test] + public void RIAofReplayTest() + { + // Insert data, then recover WITHOUT checkpoint — relies on AOF replay + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "aoftest", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "aoftest", "key1", "val1"); + db.Execute("RI.SET", "aoftest", "key2", "val2"); + db.Execute("RI.SET", "aoftest", "key3", "val3"); + + // Checkpoint to establish base state + db.Execute("SAVE"); + + // Post-checkpoint mutations — these are only in the AOF + db.Execute("RI.SET", "aoftest", "key4", "val4"); + db.Execute("RI.SET", "aoftest", "key1", "val1-updated"); + db.Execute("RI.DEL", "aoftest", "key2"); + + // Commit AOF + db.Execute("COMMITAOF"); + } + + // Recover — checkpoint restores base state, AOF replays post-checkpoint ops + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // key1 should have updated value (from AOF replay) + var val = db.Execute("RI.GET", "aoftest", "key1"); + ClassicAssert.AreEqual("val1-updated", (string)val, "key1 should have AOF-replayed update"); + + // key2 should be deleted (from AOF replay) + val = db.Execute("RI.GET", "aoftest", "key2"); + ClassicAssert.IsTrue(val.IsNull, "key2 should be deleted via AOF replay"); + + // key3 should exist (from checkpoint) + val = db.Execute("RI.GET", "aoftest", "key3"); + ClassicAssert.AreEqual("val3", (string)val, "key3 should survive from checkpoint"); + + // key4 should exist (from AOF replay) + val = db.Execute("RI.GET", "aoftest", "key4"); + ClassicAssert.AreEqual("val4", (string)val, "key4 should be added via AOF replay"); + } + } + + /// + /// Verifies RI.SCAN returns records ordered by key with COUNT limit. + /// Default FIELDS mode (BOTH) returns [key, value] pairs. + /// + [Test] + public void RIScanBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + db.Execute("RI.SET", "myindex", "ccc", "val-c"); + db.Execute("RI.SET", "myindex", "ddd", "val-d"); + db.Execute("RI.SET", "myindex", "eee", "val-e"); + + var result = (RedisResult[])db.Execute("RI.SCAN", "myindex", "aaa", "COUNT", "3"); + ClassicAssert.AreEqual(3, result.Length); + + // Each element is [key, value] since default FIELDS is BOTH + var first = (RedisResult[])result[0]; + ClassicAssert.AreEqual("aaa", (string)first[0]); + ClassicAssert.AreEqual("val-a", (string)first[1]); + + var second = (RedisResult[])result[1]; + ClassicAssert.AreEqual("bbb", (string)second[0]); + ClassicAssert.AreEqual("val-b", (string)second[1]); + + var third = (RedisResult[])result[2]; + ClassicAssert.AreEqual("ccc", (string)third[0]); + ClassicAssert.AreEqual("val-c", (string)third[1]); + } + + /// + /// Verifies RI.SCAN FIELDS KEY returns only key strings (no nested arrays). + /// + [Test] + public void RIScanFieldsKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + + var result = (RedisResult[])db.Execute("RI.SCAN", "myindex", "aaa", "COUNT", "10", "FIELDS", "KEY"); + ClassicAssert.AreEqual(2, result.Length); + ClassicAssert.AreEqual("aaa", (string)result[0]); + ClassicAssert.AreEqual("bbb", (string)result[1]); + } + + /// + /// Verifies RI.SCAN FIELDS VALUE returns only value strings (no nested arrays). + /// + [Test] + public void RIScanFieldsValueTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + + var result = (RedisResult[])db.Execute("RI.SCAN", "myindex", "aaa", "COUNT", "10", "FIELDS", "VALUE"); + ClassicAssert.AreEqual(2, result.Length); + ClassicAssert.AreEqual("val-a", (string)result[0]); + ClassicAssert.AreEqual("val-b", (string)result[1]); + } + + /// + /// Verifies RI.RANGE returns all entries in the closed [start, end] range. + /// + [Test] + public void RIRangeBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + db.Execute("RI.SET", "myindex", "ccc", "val-c"); + db.Execute("RI.SET", "myindex", "ddd", "val-d"); + db.Execute("RI.SET", "myindex", "eee", "val-e"); + + var result = (RedisResult[])db.Execute("RI.RANGE", "myindex", "bbb", "ddd"); + ClassicAssert.AreEqual(3, result.Length); + + var first = (RedisResult[])result[0]; + ClassicAssert.AreEqual("bbb", (string)first[0]); + ClassicAssert.AreEqual("val-b", (string)first[1]); + + var second = (RedisResult[])result[1]; + ClassicAssert.AreEqual("ccc", (string)second[0]); + ClassicAssert.AreEqual("val-c", (string)second[1]); + + var third = (RedisResult[])result[2]; + ClassicAssert.AreEqual("ddd", (string)third[0]); + ClassicAssert.AreEqual("val-d", (string)third[1]); + } + + /// + /// Verifies RI.SCAN on a non-existent key returns an error. + /// + [Test] + public void RIScanOnNonExistentIndexTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var ex = Assert.Throws(() => + db.Execute("RI.SCAN", "noindex", "aaa", "COUNT", "10")); + ClassicAssert.IsTrue(ex.Message.Contains("range index")); + } + + /// + /// Verifies RI.RANGE on a non-existent key returns an error. + /// + [Test] + public void RIRangeOnNonExistentIndexTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var ex = Assert.Throws(() => + db.Execute("RI.RANGE", "noindex", "aaa", "zzz")); + ClassicAssert.IsTrue(ex.Message.Contains("range index")); + } + + /// + /// Verifies that page eviction frees BfTree instances on evicted pages, + /// while trees on recent (mutable) pages remain live and functional. + /// + [Test] + public void RIEvictionFreesEvictedTreeButKeepsLiveTest() + { + // Create several RI trees on early pages, then fill the log to evict those + // pages (freeing the native BfTrees). Create new trees on recent pages and + // verify they are still fully functional. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create several RI trees — their stubs land on the first pages + for (int i = 0; i < 3; i++) + { + db.Execute("RI.CREATE", $"early{i}", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", $"early{i}", "field1", $"val{i}xx"); + } + ClassicAssert.AreEqual(3, rangeIndexManager.LiveIndexCount, "3 trees should be live after creation"); + + // Fill the log with string keys to push early pages below HeadAddress. + // Eviction calls DisposeRecord on each RI stub, freeing the native BfTrees. + for (int i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + // The 3 early trees should have been freed by eviction + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, + "All 3 early trees should have been freed by page eviction"); + + // Create one new RI tree on a recent (mutable) page — above HeadAddress + db.Execute("RI.CREATE", "live", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "live", "field1", "alive"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Only the new tree should be live"); + + // Verify the live tree is fully functional — its BfTree was NOT freed + var val = db.Execute("RI.GET", "live", "field1"); + ClassicAssert.AreEqual("alive", (string)val, + "Live tree should still be accessible after evicting early pages"); + } + + /// + /// Verifies that evicting a page with a deleted RI stub doesn't crash. + /// The BfTree was already freed by DEL; eviction sees handle=0 and skips. + /// + [Test] + public void RIEvictionAfterDeleteTest() + { + // Test that evicting a page with a deleted RI stub doesn't crash. + // The BfTree was already freed by DEL; DisposeRecord sees handle=0 and skips. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create and then delete a range index + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "key1", "value1"); + db.KeyDelete("myindex"); + + // Fill log to evict the page containing the deleted RI stub + for (int i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + // Create a new index after eviction — should work fine + db.Execute("RI.CREATE", "newidx", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "newidx", "field1", "hello"); + var val = db.Execute("RI.GET", "newidx", "field1"); + ClassicAssert.AreEqual("hello", (string)val); + } + + /// + /// Verifies that multiple indexes can be created, deleted, evicted, and new indexes + /// remain functional after the eviction of deleted stubs. + /// + [Test] + public void RIEvictionMultipleIndexesTest() + { + // Create indexes, delete them, evict their pages, then verify new indexes work + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create and delete several indexes on early pages + for (int idx = 0; idx < 3; idx++) + { + db.Execute("RI.CREATE", $"old{idx}", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", $"old{idx}", "field1", $"value{idx}"); + db.KeyDelete($"old{idx}"); + } + + // Fill log to evict deleted stubs + for (int i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + // Create a live index after eviction — should be fully functional + db.Execute("RI.CREATE", "live", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "live", "field1", "alive"); + var val = db.Execute("RI.GET", "live", "field1"); + ClassicAssert.AreEqual("alive", (string)val); + } + + /// + /// Verifies create/delete/evict/recreate cycle under memory pressure. + /// Each cycle creates an index, deletes it, evicts, then creates a new one. + /// + [Test] + public void RICreateDeleteRecreateWithEvictionTest() + { + // Test create/delete/evict/recreate cycle under memory pressure. + // Each cycle creates an index, deletes it, evicts, then creates a new one + // and verifies the new one is live. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + for (int round = 0; round < 3; round++) + { + // Create and delete — BfTree is freed by delete + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "field1", $"round{round}"); + db.KeyDelete("myindex"); + + // Fill log to evict the deleted stub page + for (int i = 0; i < 100; i++) + db.StringSet($"pad{round}_{i:D4}", $"x{round}_{i:D4}"); + } + + // After all cycles, create a live index and verify it works + db.Execute("RI.CREATE", "final", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "final", "field1", "works"); + var val = db.Execute("RI.GET", "final", "field1"); + ClassicAssert.AreEqual("works", (string)val); + } + + /// + /// Verifies RI.SET returns an error when the field exceeds MAXKEYLEN. + /// + [Test] + public void RISetInvalidKVFieldTooLongTest() + { + // RI.SET with a field exceeding MAXKEYLEN should return an error. + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", + "CACHESIZE", "65536", + "MINRECORD", "8", + "MAXRECORD", "256", + "MAXKEYLEN", "16"); + + // Field of 17 bytes exceeds MAXKEYLEN=16 + var longField = new string('k', 17); + var ex = Assert.Throws(() => + db.Execute("RI.SET", "myindex", longField, "value1")); + ClassicAssert.IsTrue(ex.Message.Contains("key+value size must be between"), + $"Expected InvalidKV error, got: {ex.Message}"); + } + + /// + /// Verifies RI.SET returns an error when the value exceeds MAXRECORD. + /// + [Test] + public void RISetInvalidKVValueTooLongTest() + { + // RI.SET with a value exceeding MAXRECORD should return an error. + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", + "CACHESIZE", "65536", + "MINRECORD", "8", + "MAXRECORD", "64", + "MAXKEYLEN", "16"); + + // Value larger than MAXRECORD=64 + var longValue = new string('v', 128); + var ex = Assert.Throws(() => + db.Execute("RI.SET", "myindex", "field1", longValue)); + ClassicAssert.IsTrue(ex.Message.Contains("key+value size must be between"), + $"Expected InvalidKV error, got: {ex.Message}"); + } + + /// + /// Verifies RI.SET returns an error when the record (key+value) is below MINRECORD. + /// + [Test] + public void RISetInvalidKVRecordTooSmallTest() + { + // RI.SET with a record (key+value) smaller than MINRECORD should return an error. + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Set a high MINRECORD so that small key+value pairs are rejected + db.Execute("RI.CREATE", "myindex", "MEMORY", + "CACHESIZE", "65536", + "MINRECORD", "128", + "MAXRECORD", "1024", + "MAXKEYLEN", "64"); + + // key="a" (1 byte) + value="b" (1 byte) = 2 bytes, well below MINRECORD=128 + var ex = Assert.Throws(() => + db.Execute("RI.SET", "myindex", "a", "b")); + ClassicAssert.IsTrue(ex.Message.Contains("key+value size must be between"), + $"Expected InvalidKV error for record below MINRECORD, got: {ex.Message}"); + } + + /// + /// Verifies thread-safety of concurrent RI.SET/RI.GET/RI.DEL from multiple clients + /// on the same RangeIndex. Tests the shared-lock path under contention. + /// + [Test] + public async Task RIConcurrentMultiClientTest() + { + // Multiple clients concurrently writing to and reading from the same + // RangeIndex. Verifies thread-safety of the shared-lock path. + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", + "CACHESIZE", "1048576", + "MINRECORD", "8", + "MAXRECORD", "256", + "MAXKEYLEN", "64"); + + const int numTasks = 8; + const int opsPerTask = 50; + + // Phase 1: concurrent writes + var writeTasks = new Task[numTasks]; + for (int t = 0; t < numTasks; t++) + { + int taskId = t; + writeTasks[t] = Task.Run(async () => + { + using var conn = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var tdb = conn.GetDatabase(0); + for (int i = 0; i < opsPerTask; i++) + { + var field = $"t{taskId}:f{i:D4}"; + var value = $"val-{taskId}-{i}"; + await tdb.ExecuteAsync("RI.SET", "myindex", field, value); + } + }); + } + await Task.WhenAll(writeTasks); + + // Phase 2: concurrent reads — verify every written value + var readTasks = new Task[numTasks]; + for (int t = 0; t < numTasks; t++) + { + int taskId = t; + readTasks[t] = Task.Run(async () => + { + using var conn = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var tdb = conn.GetDatabase(0); + for (int i = 0; i < opsPerTask; i++) + { + var field = $"t{taskId}:f{i:D4}"; + var expected = $"val-{taskId}-{i}"; + var actual = (string)await tdb.ExecuteAsync("RI.GET", "myindex", field); + ClassicAssert.AreEqual(expected, actual, + $"Mismatch for {field}: expected '{expected}', got '{actual}'"); + } + }); + } + await Task.WhenAll(readTasks); + + // Phase 3: concurrent mixed read/write/delete + var mixedTasks = new Task[numTasks]; + for (int t = 0; t < numTasks; t++) + { + int taskId = t; + mixedTasks[t] = Task.Run(async () => + { + using var conn = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var tdb = conn.GetDatabase(0); + for (int i = 0; i < opsPerTask; i++) + { + // Write a new key + var field = $"mix{taskId}:f{i:D4}"; + await tdb.ExecuteAsync("RI.SET", "myindex", field, $"mixed-{taskId}-{i}"); + + // Read it back + var val = (string)await tdb.ExecuteAsync("RI.GET", "myindex", field); + ClassicAssert.AreEqual($"mixed-{taskId}-{i}", val); + + // Delete it + await tdb.ExecuteAsync("RI.DEL", "myindex", field); + } + }); + } + await Task.WhenAll(mixedTasks); + } + + /// + /// Verifies that DEL on a mutable-region RI key immediately frees the BfTree + /// (LiveIndexCount drops to 0). + /// + [Test] + public void RIDeleteInMutableRegionFreesResourcesTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + var store = server.Provider.StoreWrapper.store; + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "field1", "value1"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should be live after creation"); + ClassicAssert.IsTrue(store.Log.TailAddress > store.Log.ReadOnlyAddress, + "Stub should be in the mutable region"); + + db.KeyDelete("myindex"); + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, + "BfTree should be freed immediately after DEL in mutable region"); + } + + /// + /// Verifies that DEL on a read-only-region RI key frees the BfTree via CopyUpdater. + /// Also verifies a new index can be created and used after the delete. + /// + [Test] + public void RIDeleteInReadOnlyRegionFreesResourcesTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + var store = server.Provider.StoreWrapper.store; + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "field1", "value1"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should be live after creation"); + + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + db.KeyDelete("myindex"); + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, + "BfTree should be freed immediately after DEL in read-only region"); + + db.Execute("RI.CREATE", "newidx", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "newidx", "f1xx", "v1xx"); + var val = db.Execute("RI.GET", "newidx", "f1xx"); + ClassicAssert.AreEqual("v1xx", (string)val); + } + + /// + /// Verifies the flush→promote→access cycle: when pages move to read-only, + /// OnFlush snapshots the BfTree and sets IsFlushed. The next RI.GET detects + /// the flag, promotes the stub to tail via RMW, and data remains accessible. + /// + [Test] + public void RIFlushPromotesToTailOnNextAccessTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + var store = server.Provider.StoreWrapper.store; + + // Create a memory-backed index and insert data + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "aaa", "val-a"); + db.Execute("RI.SET", "myindex", "bbb", "val-b"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should be live"); + + // Record the tail address before flush — stub is in mutable + var tailBeforeFlush = store.Log.TailAddress; + + // Force pages into read-only region — triggers OnFlushRecord which + // snapshots BfTree and sets the Flushed flag on the in-memory stub. + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // The BfTree should still be live (not evicted, just flushed) + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should still be live after flush"); + + // Next RI.GET detects flushed flag → promotes stub to tail via RMW → clears flag + var result = db.Execute("RI.GET", "myindex", "aaa"); + ClassicAssert.AreEqual("val-a", (string)result, "Data should be readable after flush+promote"); + + // Tail should have advanced because the stub was copied to mutable region + ClassicAssert.IsTrue(store.Log.TailAddress > tailBeforeFlush, + "Tail should advance after promote-to-tail RMW"); + + // Subsequent operations should work normally (no more promote needed) + db.Execute("RI.SET", "myindex", "ccc", "val-c"); + result = db.Execute("RI.GET", "myindex", "ccc"); + ClassicAssert.AreEqual("val-c", (string)result, "New insert should work after promote"); + + result = db.Execute("RI.GET", "myindex", "bbb"); + ClassicAssert.AreEqual("val-b", (string)result, "Pre-flush data should still be accessible"); + } + + /// + /// Verifies two consecutive flush→promote cycles: flush, promote, mutate, flush + /// again, promote again. All data should survive both cycles. + /// + [Test] + public void RIFlushPromoteThenSecondFlushTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + var store = server.Provider.StoreWrapper.store; + + // Create index, insert data + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "key-one", "value-1"); + + // First flush cycle: flush → promote → mutate + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + var r1 = db.Execute("RI.GET", "myindex", "key-one"); + ClassicAssert.AreEqual("value-1", (string)r1, "Read after first flush should promote and return data"); + + // Mutate after promote (stub is now back in mutable) + db.Execute("RI.SET", "myindex", "key-two", "value-2"); + + // Second flush cycle: flush again → should snapshot with latest data + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Read again — should promote again and return both values + r1 = db.Execute("RI.GET", "myindex", "key-one"); + ClassicAssert.AreEqual("value-1", (string)r1, "key-one should survive second flush cycle"); + + var r2 = db.Execute("RI.GET", "myindex", "key-two"); + ClassicAssert.AreEqual("value-2", (string)r2, "key-two (added after first promote) should survive second flush cycle"); + + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should still be live"); + } + + /// + /// Verifies full eviction-to-disk and lazy restore cycle: create, insert, evict + /// past HeadAddress, then access triggers pending disk read → invalidate → promote + /// → RestoreTreeFromFlush → data accessible again. + /// + [Test] + public void RIEvictToDiskThenLazyRestoreTest() + { + // Create a BfTree, insert data, evict past HeadAddress (to disk), + // then access again — should promote from disk + lazy restore from flush file. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create disk-backed index and insert data (disk backend supports snapshot/restore) + db.Execute("RI.CREATE", "myindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "myindex", "alpha", "value-alpha"); + db.Execute("RI.SET", "myindex", "bravo", "value-bravo"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should be live"); + + // Fill the log to push the RI stub below HeadAddress (eviction). + // OnFlushRecord snapshots the BfTree to flush.bftree. + // DisposeRecord(PageEviction) frees the native BfTree. + for (int i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, + "BfTree should have been freed by eviction"); + + // Now access the RI key — this triggers: + // 1. Pending read from disk → OnDiskReadRecord invalidates TreeHandle + // 2. ReadRangeIndex detects IsFlushed → promotes stub to tail + // 3. Promoted stub has TreeHandle == 0 → RestoreTreeFromFlush recovers BfTree + // 4. RI.GET returns data from the restored BfTree + var result = db.Execute("RI.GET", "myindex", "alpha"); + ClassicAssert.AreEqual("value-alpha", (string)result, + "Data should be recoverable after eviction + lazy restore"); + + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, + "Restored BfTree should be registered as live"); + + // Verify second key is also available + result = db.Execute("RI.GET", "myindex", "bravo"); + ClassicAssert.AreEqual("value-bravo", (string)result, + "Second key should also be recoverable"); + + // Verify writes work on the restored tree + db.Execute("RI.SET", "myindex", "charlie", "value-charlie"); + result = db.Execute("RI.GET", "myindex", "charlie"); + ClassicAssert.AreEqual("value-charlie", (string)result, + "Writes should work on restored tree"); + } + + /// + /// Verifies checkpoint + recovery: create tree, insert data, BGSAVE, dispose, + /// recover. All data should be present after recovery and new writes should work. + /// + [Test] + public void RICheckpointAndRecoverTest() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Create a disk-backed BfTree and insert data + db.Execute("RI.CREATE", "cpindex", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "cpindex", "alpha", "value-alpha"); + db.Execute("RI.SET", "cpindex", "bravo", "value-bravo"); + db.Execute("RI.SET", "cpindex", "charlie", "value-charlie"); + + // Verify data before checkpoint + var val = db.Execute("RI.GET", "cpindex", "alpha"); + ClassicAssert.AreEqual("value-alpha", (string)val); + + // Take checkpoint via BGSAVE + db.Execute("SAVE"); + } + + // Dispose and recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Verify data recovered from checkpoint + var val = db.Execute("RI.GET", "cpindex", "alpha"); + ClassicAssert.AreEqual("value-alpha", (string)val, "alpha should survive checkpoint+recovery"); + + val = db.Execute("RI.GET", "cpindex", "bravo"); + ClassicAssert.AreEqual("value-bravo", (string)val, "bravo should survive checkpoint+recovery"); + + val = db.Execute("RI.GET", "cpindex", "charlie"); + ClassicAssert.AreEqual("value-charlie", (string)val, "charlie should survive checkpoint+recovery"); + + // Verify writes work after recovery + db.Execute("RI.SET", "cpindex", "delta", "value-delta"); + val = db.Execute("RI.GET", "cpindex", "delta"); + ClassicAssert.AreEqual("value-delta", (string)val, "new insert should work after recovery"); + } + } + + /// + /// Full lifecycle test: create → insert → flush to read-only → promote → mutate → + /// evict to disk → restore from flush → checkpoint → recover. Verifies data + /// survives every transition in the store lifecycle. + /// + [Test] + public void RIFlushEvictRestoreCheckpointCycleTest() + { + // Full lifecycle: create → insert → flush to read-only → promote → mutate → + // evict to disk → restore from flush → checkpoint → recover + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + var store = server.Provider.StoreWrapper.store; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Create disk-backed tree and insert data + db.Execute("RI.CREATE", "lifecycle", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "lifecycle", "key-aaa", "val-aaa"); + db.Execute("RI.SET", "lifecycle", "key-bbb", "val-bbb"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + // Force flush to read-only (sets Flushed flag) + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Access triggers promote to tail (clears Flushed flag) + var val = db.Execute("RI.GET", "lifecycle", "key-aaa"); + ClassicAssert.AreEqual("val-aaa", (string)val, "Should read after flush+promote"); + + // Mutate after promote (stub now in mutable) + db.Execute("RI.SET", "lifecycle", "key-ccc", "val-ccc"); + + // Fill log to evict the old pages past HeadAddress + for (int i = 0; i < 200; i++) + db.StringSet($"fill{i:D4}", $"data{i:D4}"); + + // Tree might still be live (promote moved stub to tail) + // Access should work regardless + val = db.Execute("RI.GET", "lifecycle", "key-bbb"); + ClassicAssert.AreEqual("val-bbb", (string)val, "Should read after eviction of old pages"); + + val = db.Execute("RI.GET", "lifecycle", "key-ccc"); + ClassicAssert.AreEqual("val-ccc", (string)val, "Post-promote insert should survive eviction"); + + // Take checkpoint + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + var val = db.Execute("RI.GET", "lifecycle", "key-aaa"); + ClassicAssert.AreEqual("val-aaa", (string)val, "key-aaa should survive full lifecycle"); + + val = db.Execute("RI.GET", "lifecycle", "key-bbb"); + ClassicAssert.AreEqual("val-bbb", (string)val, "key-bbb should survive full lifecycle"); + + val = db.Execute("RI.GET", "lifecycle", "key-ccc"); + ClassicAssert.AreEqual("val-ccc", (string)val, "key-ccc should survive full lifecycle"); + } + } + + /// + /// Verifies that multiple BfTrees are independently evicted and lazily restored. + /// 3 evicted trees + 1 live tree = 4 total after all restores. + /// + [Test] + public void RIMultipleTreesEvictAndRestoreTest() + { + // Multiple BfTrees, some evicted, some live — verify independent lifecycle + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create 3 trees early — they'll be on early pages + for (int i = 0; i < 3; i++) + { + db.Execute("RI.CREATE", $"tree{i}", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", $"tree{i}", "field1", $"value{i}-1"); + db.Execute("RI.SET", $"tree{i}", "field2", $"value{i}-2"); + } + ClassicAssert.AreEqual(3, rangeIndexManager.LiveIndexCount); + + // Fill log to evict early pages + for (int i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + // Early trees should be evicted + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "All early trees should be evicted"); + + // Create a new tree on recent pages (still in memory) + db.Execute("RI.CREATE", "tree-live", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "tree-live", "field1", "live-val"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + // Access evicted trees — triggers lazy restore + for (int i = 0; i < 3; i++) + { + var val = db.Execute("RI.GET", $"tree{i}", "field1"); + ClassicAssert.AreEqual($"value{i}-1", (string)val, $"tree{i} field1 should restore from flush"); + + val = db.Execute("RI.GET", $"tree{i}", "field2"); + ClassicAssert.AreEqual($"value{i}-2", (string)val, $"tree{i} field2 should restore from flush"); + } + + // All trees should now be live (3 restored + 1 live) + ClassicAssert.AreEqual(4, rangeIndexManager.LiveIndexCount); + + // Live tree should still work + var liveVal = db.Execute("RI.GET", "tree-live", "field1"); + ClassicAssert.AreEqual("live-val", (string)liveVal); + } + + /// + /// Verifies that multiple trees survive checkpoint + recovery with all data intact. + /// + [Test] + public void RICheckpointWithMultipleTreesAndRecoverTest() + { + // Multiple trees, checkpoint, recover — all trees should be restored + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Create several trees with different data + for (int i = 0; i < 5; i++) + { + db.Execute("RI.CREATE", $"idx{i}", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + for (int j = 0; j < 10; j++) + db.Execute("RI.SET", $"idx{i}", $"field-{j:D3}", $"value-{i}-{j}"); + } + + // Checkpoint + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Verify all trees and all fields recovered + for (int i = 0; i < 5; i++) + { + for (int j = 0; j < 10; j++) + { + var val = db.Execute("RI.GET", $"idx{i}", $"field-{j:D3}"); + ClassicAssert.AreEqual($"value-{i}-{j}", (string)val, + $"idx{i} field-{j:D3} should survive checkpoint+recovery"); + } + } + + // Verify writes work on all recovered trees + for (int i = 0; i < 5; i++) + { + db.Execute("RI.SET", $"idx{i}", "new-field", $"new-value-{i}"); + var val = db.Execute("RI.GET", $"idx{i}", "new-field"); + ClassicAssert.AreEqual($"new-value-{i}", (string)val); + } + } + } + + /// + /// Verifies flush → promote → mutate → checkpoint → recover captures post-flush mutations. + /// + [Test] + public void RIFlushPromoteCheckpointRecoverTest() + { + // Flush → promote → mutate → checkpoint → recover + // Tests that post-flush mutations are captured by checkpoint + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + var store = server.Provider.StoreWrapper.store; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "fpcp", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "fpcp", "before-flush", "original"); + + // Flush to read-only + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Promote by accessing + var val = db.Execute("RI.GET", "fpcp", "before-flush"); + ClassicAssert.AreEqual("original", (string)val); + + // Mutate after promote (these are in mutable region now) + db.Execute("RI.SET", "fpcp", "after-flush", "mutated"); + db.Execute("RI.SET", "fpcp", "before-flush", "updated"); + + // Checkpoint should capture the mutated state + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + var val = db.Execute("RI.GET", "fpcp", "before-flush"); + ClassicAssert.AreEqual("updated", (string)val, "Updated value should survive checkpoint"); + + val = db.Execute("RI.GET", "fpcp", "after-flush"); + ClassicAssert.AreEqual("mutated", (string)val, "Post-flush insert should survive checkpoint"); + } + } + + /// + /// Verifies that evicting a deleted stub doesn't crash — the BfTree was already freed + /// by DEL, so eviction sees handle=0 and safely skips. New indexes work after eviction. + /// + [Test] + public void RIDeleteDuringEvictionCycleTest() + { + // Create tree → delete → fill log to trigger eviction of deleted record + // Verify eviction of deleted record doesn't crash + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create and populate + db.Execute("RI.CREATE", "deltest", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "deltest", "field1", "value1"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + // Delete the tree + db.KeyDelete("deltest"); + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount); + + // Fill log to evict the page containing the deleted stub + for (int i = 0; i < 200; i++) + db.StringSet($"fill{i:D4}", $"data{i:D4}"); + + // Create new tree to verify manager is still functional + db.Execute("RI.CREATE", "newtest", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "newtest", "field1", "alive"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + var val = db.Execute("RI.GET", "newtest", "field1"); + ClassicAssert.AreEqual("alive", (string)val); + } + + /// + /// Verifies double flush → promote → checkpoint → recover: data from both flush + /// cycles and post-promote mutations all survive recovery. + /// + [Test] + public void RIDoubleFlushCycleWithCheckpointTest() + { + // flush → promote → mutate → flush again → promote → checkpoint → recover + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + var store = server.Provider.StoreWrapper.store; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "dblflush", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "dblflush", "round1", "value-r1"); + + // First flush cycle + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + var val = db.Execute("RI.GET", "dblflush", "round1"); + ClassicAssert.AreEqual("value-r1", (string)val); + + // Mutate after first promote + db.Execute("RI.SET", "dblflush", "round2", "value-r2"); + + // Second flush cycle + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + val = db.Execute("RI.GET", "dblflush", "round2"); + ClassicAssert.AreEqual("value-r2", (string)val); + + // Mutate after second promote + db.Execute("RI.SET", "dblflush", "round3", "value-r3"); + + // Checkpoint + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + var val = db.Execute("RI.GET", "dblflush", "round1"); + ClassicAssert.AreEqual("value-r1", (string)val, "round1 should survive double-flush + checkpoint"); + + val = db.Execute("RI.GET", "dblflush", "round2"); + ClassicAssert.AreEqual("value-r2", (string)val, "round2 should survive double-flush + checkpoint"); + + val = db.Execute("RI.GET", "dblflush", "round3"); + ClassicAssert.AreEqual("value-r3", (string)val, "round3 should survive double-flush + checkpoint"); + } + } + + /// + /// Verifies eviction → lazy restore → mutate → checkpoint → recover. + /// Post-restore mutations should be captured by the checkpoint. + /// + [Test] + public void RIEvictRestoreAndCheckpointTest() + { + // Evict to disk → lazy restore → mutate → checkpoint → recover + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "evictcp", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "evictcp", "pre-evict", "original"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + // Fill to evict + for (int i = 0; i < 200; i++) + db.StringSet($"fill{i:D4}", $"data{i:D4}"); + + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "Tree should be evicted"); + + // Lazy restore by accessing + var val = db.Execute("RI.GET", "evictcp", "pre-evict"); + ClassicAssert.AreEqual("original", (string)val, "Should restore from flush file"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "Tree should be restored"); + + // Mutate after restore + db.Execute("RI.SET", "evictcp", "post-restore", "added"); + + // Checkpoint + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + var val = db.Execute("RI.GET", "evictcp", "pre-evict"); + ClassicAssert.AreEqual("original", (string)val, "Pre-evict data should survive evict→restore→checkpoint"); + + val = db.Execute("RI.GET", "evictcp", "post-restore"); + ClassicAssert.AreEqual("added", (string)val, "Post-restore data should survive checkpoint"); + } + } + + /// + /// Verifies that taking two checkpoints and recovering always gets the latest data. + /// Checkpoint 2 should contain updates made after checkpoint 1. + /// + [Test] + public void RITwoCheckpointsRecoverToLatestTest() + { + // Take two checkpoints with different data, recover to latest + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Phase 1: create tree and insert first batch + db.Execute("RI.CREATE", "twockpt", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "twockpt", "key-alpha", "val-alpha"); + db.Execute("RI.SET", "twockpt", "key-bravo", "val-bravo"); + + // Checkpoint 1 + db.Execute("SAVE"); + + // Phase 2: insert additional data + update existing + db.Execute("RI.SET", "twockpt", "key-charlie", "val-charlie"); + db.Execute("RI.SET", "twockpt", "key-alpha", "val-alpha-v2"); + + // Checkpoint 2 + db.Execute("SAVE"); + } + + // Recover — should get latest checkpoint (checkpoint 2) + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Should see checkpoint 2 data: updated alpha, bravo, and charlie + var val = db.Execute("RI.GET", "twockpt", "key-alpha"); + ClassicAssert.AreEqual("val-alpha-v2", (string)val, "alpha should have updated value from checkpoint 2"); + + val = db.Execute("RI.GET", "twockpt", "key-bravo"); + ClassicAssert.AreEqual("val-bravo", (string)val, "bravo should exist"); + + val = db.Execute("RI.GET", "twockpt", "key-charlie"); + ClassicAssert.AreEqual("val-charlie", (string)val, "charlie should exist from checkpoint 2"); + + // Verify writes work after recovery + db.Execute("RI.SET", "twockpt", "key-delta", "val-delta"); + val = db.Execute("RI.GET", "twockpt", "key-delta"); + ClassicAssert.AreEqual("val-delta", (string)val, "new insert should work after recovery"); + } + } + + /// + /// Verifies that post-checkpoint mutations are replayed from AOF after recovery. + /// With AOF enabled, v+1 data IS replayed from AOF after checkpoint recovery. + /// + [Test] + public void RIRecoverToEarlierCheckpointTest() + { + // Add keys A,B → checkpoint → add key C, update A → recover → + // With AOF logging, v+1 data IS replayed from AOF after checkpoint recovery + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Phase 1: create tree, insert keys A and B + db.Execute("RI.CREATE", "earlyck", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "earlyck", "key-A", "val-A-original"); + db.Execute("RI.SET", "earlyck", "key-B", "val-B"); + + // Checkpoint + db.Execute("SAVE"); + + // Phase 2: add more data AFTER checkpoint (logged to AOF) + db.Execute("RI.SET", "earlyck", "key-A", "val-A-updated"); + db.Execute("RI.SET", "earlyck", "key-C", "val-C"); + + // Commit AOF + db.Execute("COMMITAOF"); + } + + // Recover — checkpoint restored first, then AOF entries replayed + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // key-A should have the UPDATED value (replayed from AOF) + var val = db.Execute("RI.GET", "earlyck", "key-A"); + ClassicAssert.AreEqual("val-A-updated", (string)val, "key-A should have updated value replayed from AOF"); + + // key-B should exist (it was in the checkpoint) + val = db.Execute("RI.GET", "earlyck", "key-B"); + ClassicAssert.AreEqual("val-B", (string)val, "key-B should exist from checkpoint"); + + // key-C should exist (replayed from AOF) + val = db.Execute("RI.GET", "earlyck", "key-C"); + ClassicAssert.AreEqual("val-C", (string)val, "key-C should exist — replayed from AOF after checkpoint"); + } + } + + /// + /// Verifies that DEL on a recovered key correctly frees the lazily-restored BfTree. + /// + [Test] + public void RIDeleteAfterRecoveryTest() + { + // Create tree, checkpoint, recover, then DEL the recovered key. + // Verifies that DEL correctly frees the lazily-restored BfTree. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + db.Execute("RI.CREATE", "delafter", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "delafter", "key1", "val1"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + db.Execute("SAVE"); + } + + // Recover + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Access triggers lazy restore + var val = db.Execute("RI.GET", "delafter", "key1"); + ClassicAssert.AreEqual("val1", (string)val); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount); + + // DEL on the recovered key should free the BfTree + var deleted = db.KeyDelete("delafter"); + ClassicAssert.IsTrue(deleted, "DEL should return true"); + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "BfTree should be freed after DEL"); + + // Subsequent access should return not found + var ex = Assert.Throws(() => + db.Execute("RI.GET", "delafter", "key1")); + ClassicAssert.IsNotNull(ex); + } + } + + /// + /// Verifies RI.EXISTS returns 1 for existing RI keys, 0 for non-existent + /// and deleted keys. + /// + [Test] + public void RIExistsBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Non-existent key should return 0 + var result = db.Execute("RI.EXISTS", "myindex"); + ClassicAssert.AreEqual(0, (int)result); + + // Create index, should now return 1 + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536"); + result = db.Execute("RI.EXISTS", "myindex"); + ClassicAssert.AreEqual(1, (int)result); + + // Delete the key, should return 0 + db.KeyDelete("myindex"); + result = db.Execute("RI.EXISTS", "myindex"); + ClassicAssert.AreEqual(0, (int)result); + } + + /// + /// Verifies RI.EXISTS returns 0 (not WRONGTYPE) for a normal string key. + /// + [Test] + public void RIExistsOnNormalKeyTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // SET a normal string key + db.StringSet("normalkey", "hello"); + + // RI.EXISTS on a normal string key should return 0 (not WRONGTYPE) + var result = db.Execute("RI.EXISTS", "normalkey"); + ClassicAssert.AreEqual(0, (int)result); + } + + /// + /// Verifies RI.CONFIG returns all 6 configuration fields with correct values. + /// + [Test] + public void RIConfigBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", + "MINRECORD", "32", "MAXRECORD", "512", "MAXKEYLEN", "64"); + + var result = db.Execute("RI.CONFIG", "myindex"); + var arr = (RedisResult[])result; + + // Should be 12 elements (6 field-value pairs) + ClassicAssert.AreEqual(12, arr.Length); + + // Check field names and values + ClassicAssert.AreEqual("storage_backend", (string)arr[0]); + ClassicAssert.AreEqual("MEMORY", (string)arr[1]); + ClassicAssert.AreEqual("cache_size", (string)arr[2]); + ClassicAssert.AreEqual("65536", (string)arr[3]); + ClassicAssert.AreEqual("min_record_size", (string)arr[4]); + ClassicAssert.AreEqual("32", (string)arr[5]); + ClassicAssert.AreEqual("max_record_size", (string)arr[6]); + ClassicAssert.AreEqual("512", (string)arr[7]); + ClassicAssert.AreEqual("max_key_len", (string)arr[8]); + ClassicAssert.AreEqual("64", (string)arr[9]); + ClassicAssert.AreEqual("leaf_page_size", (string)arr[10]); + // leaf_page_size is auto-computed, just ensure it's present + ClassicAssert.IsNotNull((string)arr[11]); + } + + /// + /// Verifies RI.CONFIG returns WRONGTYPE error on a normal string key. + /// + [Test] + public void RIConfigWrongTypeTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.StringSet("normalkey", "hello"); + + var ex = Assert.Throws(() => + db.Execute("RI.CONFIG", "normalkey")); + ClassicAssert.IsNotNull(ex); + } + + /// + /// Verifies RI.METRICS returns runtime state (tree_handle, is_live, is_flushed, is_recovered). + /// + [Test] + public void RIMetricsBasicTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536", "MINRECORD", "8"); + + // Insert some data (use field/value sizes that fit default MINRECORD) + db.Execute("RI.SET", "myindex", "field1", "value1"); + db.Execute("RI.SET", "myindex", "field2", "value2"); + + var result = db.Execute("RI.METRICS", "myindex"); + var arr = (RedisResult[])result; + + // Should be 8 elements (4 field-value pairs) + ClassicAssert.AreEqual(8, arr.Length); + + ClassicAssert.AreEqual("tree_handle", (string)arr[0]); + // tree_handle should be a non-zero number + ClassicAssert.IsNotNull((string)arr[1]); + + ClassicAssert.AreEqual("is_live", (string)arr[2]); + ClassicAssert.AreEqual("true", (string)arr[3]); + + ClassicAssert.AreEqual("is_flushed", (string)arr[4]); + ClassicAssert.AreEqual("false", (string)arr[5]); + + ClassicAssert.AreEqual("is_recovered", (string)arr[6]); + ClassicAssert.AreEqual("false", (string)arr[7]); + } + + /// + /// Verifies that TYPE command returns "rangeindex" for RI keys. + /// + [Test] + public void RITypeCommandTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create a RangeIndex and check TYPE + db.Execute("RI.CREATE", "myindex", "MEMORY", "CACHESIZE", "65536"); + var type = db.KeyType("myindex"); + ClassicAssert.AreEqual(RedisType.Unknown, type); + // StackExchange.Redis maps unknown types to Unknown, verify via Execute + var typeResult = db.Execute("TYPE", "myindex"); + ClassicAssert.AreEqual("rangeindex", (string)typeResult); + + // Normal string key should return "string" + db.StringSet("normalkey", "hello"); + typeResult = db.Execute("TYPE", "normalkey"); + ClassicAssert.AreEqual("string", (string)typeResult); + } + + /// + /// Stress test: 4 worker threads insert concurrently while a 5th thread takes a + /// blocking SAVE checkpoint. Verifies checkpoint contains a strict prefix of each + /// thread's keys (point-in-time snapshot consistency). + /// + [Test] + [CancelAfter(120_000)] + public void RIConcurrentOpsWithCheckpointTest(System.Threading.CancellationToken cancellationToken) + { + // 4 threads insert contiguous keys at full speed. A single SAVE (blocking) + // runs from a 5th thread. After SAVE completes, workers are signaled and + // insert a few more keys before stopping. Recovery should show a strict + // prefix per thread: all keys before some cutoff, none after. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + const int numThreads = 4; + const int postSaveOps = 50; + var saveCompleted = new ManualResetEventSlim(false); + var errors = new System.Collections.Concurrent.ConcurrentBag(); + var insertedCounts = new int[numThreads]; + var barrier = new Barrier(numThreads + 1); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + db.Execute("RI.CREATE", "stress", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + } + + // Worker threads: insert keys at full speed, then postSaveOps more after save + var workers = new Task[numThreads]; + for (int t = 0; t < numThreads; t++) + { + var threadId = t; + workers[t] = Task.Run(() => + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + int i = 0; + barrier.SignalAndWait(); + + // Phase 1: insert at full speed until save completes + while (!saveCompleted.IsSet) + { + try + { + db.Execute("RI.SET", "stress", $"t{threadId}_{i:D6}", $"v{threadId}_{i:D6}"); + i++; + } + catch (Exception ex) + { + errors.Add($"Thread {threadId} op {i}: {ex.Message}"); + } + } + + // Phase 2: insert postSaveOps more (these should NOT be in the checkpoint) + for (int j = 0; j < postSaveOps; j++) + { + try + { + db.Execute("RI.SET", "stress", $"t{threadId}_{i:D6}", $"v{threadId}_{i:D6}"); + i++; + } + catch (Exception ex) + { + errors.Add($"Thread {threadId} op {i}: {ex.Message}"); + } + } + + insertedCounts[threadId] = i; + }); + } + + // Checkpoint thread: wait for workers to start, then do a single blocking SAVE + var checkpointTask = Task.Run(() => + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + barrier.SignalAndWait(); + + // Let workers insert for a bit before taking the checkpoint + Thread.Sleep(200); + db.Execute("SAVE"); + saveCompleted.Set(); + }); + + Task.WaitAll([.. workers, checkpointTask]); + ClassicAssert.IsEmpty(errors, $"Errors during concurrent ops:\n{string.Join("\n", errors)}"); + + // Recover from checkpoint only (no AOF) + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + for (int t = 0; t < numThreads; t++) + { + var totalInserted = insertedCounts[t]; + + // Find the prefix length: first key that is absent + int recovered = 0; + for (int i = 0; i < totalInserted; i++) + { + var val = db.Execute("RI.GET", "stress", $"t{t}_{i:D6}"); + if (val.IsNull) + break; + ClassicAssert.AreEqual($"v{t}_{i:D6}", (string)val, + $"Thread {t} key t{t}_{i:D6}: value mismatch"); + recovered++; + } + + ClassicAssert.Greater(recovered, 0, + $"Thread {t}: no keys recovered"); + + // Strict prefix: ALL keys after the cutoff must be absent + for (int i = recovered; i < totalInserted; i++) + { + var val = db.Execute("RI.GET", "stress", $"t{t}_{i:D6}"); + ClassicAssert.IsTrue(val.IsNull, + $"Thread {t} key t{t}_{i:D6}: present after gap at {recovered} — not a strict prefix"); + } + + // The postSaveOps keys must NOT be in the checkpoint + ClassicAssert.Less(recovered, totalInserted, + $"Thread {t}: all {totalInserted} keys recovered — checkpoint was not taken mid-insertion"); + } + } + } + + /// + /// After checkpoint recovery, IsRecovered must be cleared when the tree + /// is first restored. Otherwise, a second eviction cycle causes + /// RestoreTreeFromFlush to pick the stale checkpoint snapshot instead of the + /// fresh flush.bftree, losing post-recovery writes. + /// + [Test] + public void RIRecoverThenSecondEvictionUsesFlushSnapshotTest() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "stalecp", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "stalecp", "pre-checkpoint", "original"); + db.Execute("SAVE"); + } + + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true, enableAOF: true, tryRecover: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + var val = db.Execute("RI.GET", "stalecp", "pre-checkpoint"); + ClassicAssert.AreEqual("original", (string)val, "Checkpoint data should be restored"); + + db.Execute("RI.SET", "stalecp", "post-recovery", "new-value"); + + // Fill log to trigger flush (writes flush.bftree) then eviction (frees tree) + for (int i = 0; i < 200; i++) + db.StringSet($"fill{i:D4}", $"data{i:D4}"); + + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "Tree should be evicted"); + + val = db.Execute("RI.GET", "stalecp", "pre-checkpoint"); + ClassicAssert.AreEqual("original", (string)val, "Pre-checkpoint data should survive"); + + val = db.Execute("RI.GET", "stalecp", "post-recovery"); + ClassicAssert.AreEqual("new-value", (string)val, + "Post-recovery data must survive eviction (flush.bftree, not stale checkpoint)"); + } + } + + /// + /// Verifies pure AOF-only recovery (no checkpoint). RI.CREATE is replayed to + /// recreate the BfTree, then RI.SET/RI.DEL operations rebuild the data. + /// + [Test] + public void RIAofOnlyRecoveryTest() + { + // No checkpoint at all — AOF replay must recreate the BfTree from scratch. + // RI.CREATE is logged via RMW, RI.SET/RI.DEL via synthetic RMW. + // On recovery, AOF replay re-executes RI.CREATE (creates the tree), + // then replays RI.SET/RI.DEL operations to rebuild the data. + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + db.Execute("RI.CREATE", "aofonly", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "aofonly", "key-a", "val-a"); + db.Execute("RI.SET", "aofonly", "key-b", "val-b"); + db.Execute("RI.SET", "aofonly", "key-c", "val-c"); + db.Execute("RI.SET", "aofonly", "key-a", "val-a-updated"); + db.Execute("RI.DEL", "aofonly", "key-b"); + + // Commit AOF but do NOT take a checkpoint + db.Execute("COMMITAOF"); + } + + // Recover — no checkpoint exists, so everything comes from AOF replay + server.Dispose(); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, enableAOF: true, tryRecover: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // key-a should have updated value + var val = db.Execute("RI.GET", "aofonly", "key-a"); + ClassicAssert.AreEqual("val-a-updated", (string)val, "key-a should have updated value from AOF replay"); + + // key-b should be deleted + val = db.Execute("RI.GET", "aofonly", "key-b"); + ClassicAssert.IsTrue(val.IsNull, "key-b should be deleted via AOF replay"); + + // key-c should exist + val = db.Execute("RI.GET", "aofonly", "key-c"); + ClassicAssert.AreEqual("val-c", (string)val, "key-c should exist from AOF replay"); + + // New writes should work on the AOF-recovered tree + db.Execute("RI.SET", "aofonly", "key-d", "val-d"); + val = db.Execute("RI.GET", "aofonly", "key-d"); + ClassicAssert.AreEqual("val-d", (string)val, "new insert should work after AOF-only recovery"); + } + } + + /// + /// Verifies that DEL on a disk-backed RangeIndex cleans up the WORKING file + /// (<hash>.data.bftree) on disk but PRESERVES per-flush snapshot files + /// (<hash>.<addr>.flush.bftree). Per-flush files are LOG-tied — they may + /// still be required to recover an OLDER checkpoint that was taken BEFORE the DEL. + /// They are reclaimed by OnTruncate when the log's BeginAddress passes their address. + /// + [Test] + public void RIDiskFileCleanupOnDeleteTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create a disk-backed range index + db.Execute("RI.CREATE", "cleanup", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "cleanup", "key1", "val1"); + + // Verify the riLogRoot exists with at least one .data.bftree file + var riLogRoot = Path.Combine(TestUtils.MethodTestDir, "Store", "rangeindex"); + ClassicAssert.IsTrue(Directory.Exists(riLogRoot), "riLogRoot directory should exist after RI.CREATE"); + var dataFiles = Directory.GetFiles(riLogRoot, "*.data.bftree"); + ClassicAssert.AreEqual(1, dataFiles.Length, "should have exactly one data.bftree file"); + + // Delete the range index + var delResult = db.KeyDelete("cleanup"); + ClassicAssert.IsTrue(delResult, "DEL should return true"); + + // Working file should be cleaned up. + dataFiles = Directory.Exists(riLogRoot) ? Directory.GetFiles(riLogRoot, "*.data.bftree") : []; + ClassicAssert.AreEqual(0, dataFiles.Length, "data.bftree files should be deleted after DEL"); + // (Flush files are not asserted here because none were created in this test — no flush + // event fired between RI.SET and DEL. The next test verifies the preservation contract.) + } + + /// + /// Verifies the LOG-tied lifetime of per-flush snapshot files: DEL preserves the + /// <hash>.<addr>.flush.bftree files that were created by prior flush + /// events, because they may be required to recover an OLDER checkpoint that was taken + /// before the DEL. Only OnTruncate (when log BeginAddress passes their address) + /// can safely delete them. + /// + [Test] + public void RIDeletePreservesPerFlushSnapshotFilesTest() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, + lowMemory: true); + server.Start(); + + var store = server.Provider.StoreWrapper.store; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + + // Create RI, write, force flush so a per-flush snapshot file is created. + db.Execute("RI.CREATE", "preservetest", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "preservetest", "field-x", "value-v1"); + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + var riLogRoot = Path.Combine(TestUtils.MethodTestDir, "Store", "rangeindex"); + var flushBefore = Directory.GetFiles(riLogRoot, "*.flush.bftree"); + ClassicAssert.GreaterOrEqual(flushBefore.Length, 1, + "test setup: at least one flush.bftree file should exist after force-flush"); + + // Read once to promote the flushed stub back to the mutable region (RIPROMOTE). + // This is required because DEL operates on the in-memory chain head. + ClassicAssert.AreEqual("value-v1", (string)db.Execute("RI.GET", "preservetest", "field-x")); + + // DEL the key. + ClassicAssert.IsTrue(db.KeyDelete("preservetest"), "DEL should succeed"); + + // Working file must be deleted, but per-flush snapshot file(s) must SURVIVE. + var dataAfter = Directory.GetFiles(riLogRoot, "*.data.bftree"); + ClassicAssert.AreEqual(0, dataAfter.Length, "data.bftree should be deleted on DEL"); + + var flushAfter = Directory.GetFiles(riLogRoot, "*.flush.bftree"); + ClassicAssert.AreEqual(flushBefore.Length, flushAfter.Length, + "per-flush snapshot files MUST be preserved on DEL — they may be required to recover " + + "an older checkpoint that was taken before the DEL. Only OnTruncate (when log " + + "BeginAddress passes their address) can safely delete them."); + + // Verify file content is byte-identical (not corrupted). + for (var i = 0; i < flushBefore.Length; i++) + { + ClassicAssert.IsTrue(File.Exists(flushBefore[i]), $"flush file {Path.GetFileName(flushBefore[i])} must still exist after DEL"); + } + } + + /// + /// Verifies that DEL of a previously-evicted-and-restored RangeIndex correctly cleans + /// up the working file but PRESERVES per-flush snapshot files (LOG-tied lifetime). + /// + [Test] + public void RIDiskFileCleanupOnDeleteAfterEvictionAndRestoreTest() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, lowMemory: true); + server.Start(); + + var rangeIndexManager = server.Provider.StoreWrapper.rangeIndexManager; + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Create a disk-backed range index on an early page + db.Execute("RI.CREATE", "evictdel", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "evictdel", "key1", "val1"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "tree should be live after creation"); + + var riLogRoot = Path.Combine(TestUtils.MethodTestDir, "Store", "rangeindex"); + ClassicAssert.IsTrue(Directory.Exists(riLogRoot), "riLogRoot should exist"); + + // Fill the log with string keys to push RI stub below HeadAddress and trigger eviction + for (var i = 0; i < 200; i++) + db.StringSet($"filler{i:D4}", $"data{i:D4}"); + + // Verify eviction actually occurred + ClassicAssert.AreEqual(0, rangeIndexManager.LiveIndexCount, "tree should have been freed by eviction"); + + // Files should still exist after eviction (preserved for lazy restore) + var dataFiles = Directory.GetFiles(riLogRoot, "*.data.bftree"); + ClassicAssert.AreEqual(1, dataFiles.Length, "data.bftree file should survive eviction"); + var flushFilesPostEvict = Directory.GetFiles(riLogRoot, "*.flush.bftree"); + ClassicAssert.GreaterOrEqual(flushFilesPostEvict.Length, 1, "at least one flush snapshot should exist post-eviction"); + + // Lazy restore brings the record back in-memory (DEL requires the record + // to be in-memory; the unified Delete path does not trigger lazy restore). + var val = db.Execute("RI.GET", "evictdel", "key1"); + ClassicAssert.AreEqual("val1", (string)val, "lazy restore should recover data after eviction"); + ClassicAssert.AreEqual(1, rangeIndexManager.LiveIndexCount, "tree should be live again after lazy restore"); + + // Now delete — only the working data.bftree should be removed; per-flush snapshots + // must survive (LOG-tied lifetime; needed for recovery to a prior checkpoint). + var delResult = db.KeyDelete("evictdel"); + ClassicAssert.IsTrue(delResult, "DEL should return true"); + + dataFiles = Directory.Exists(riLogRoot) ? Directory.GetFiles(riLogRoot, "*.data.bftree") : []; + ClassicAssert.AreEqual(0, dataFiles.Length, "data.bftree should be deleted after DEL"); + + var flushFilesPostDel = Directory.Exists(riLogRoot) ? Directory.GetFiles(riLogRoot, "*.flush.bftree") : []; + ClassicAssert.AreEqual(flushFilesPostEvict.Length, flushFilesPostDel.Length, + "per-flush snapshot files MUST be preserved on DEL even for previously-evicted keys " + + "— they may be required to recover an older checkpoint taken before the DEL."); + } + + // ============================================================================ + // BfTree compaction-lifecycle tests + // ============================================================================ + + /// + /// Helper: count the number of <hash>.<addr>.flush.bftree files in + /// the riLogRoot — used by tests that verify per-flush-file lifecycle. + /// + private static int CountFlushFiles() + { + var riLogRoot = Path.Combine(TestUtils.MethodTestDir, "Store", "rangeindex"); + if (!Directory.Exists(riLogRoot)) return 0; + return Directory.GetFiles(riLogRoot, "*.flush.bftree").Length; + } + + /// + /// Verifies the per-flush snapshot file immutability invariant: a + /// <hash>.<addr>.flush.bftree file, once written, is never overwritten. + /// Subsequent flushes for the same key produce a NEW file at a distinct address, + /// so historical per-flush state is preserved for recovery to older checkpoints. + /// + /// Steps: create a disk-backed RI, set v1, flush (creates file at A1), capture + /// bytes; promote + set v2, flush (must create a new file at A2); assert that the + /// A1 file still exists with byte-identical content AND the post-v2 file count is + /// strictly greater than post-v1. + /// + [Test] + public void RIFlushFilesAreImmutablePerAddressTest() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableRangeIndexPreview: true, + lowMemory: true); + server.Start(); + + var store = server.Provider.StoreWrapper.store; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // v1 state: create disk-backed RI and insert one field. Use long enough field/value + // to satisfy MINRECORD=8. + db.Execute("RI.CREATE", "flushtestkey", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "flushtestkey", "field-x", "value-v1"); + + // Force flush so the v1 stub is on a flushed page (creates ..flush.bftree). + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Snapshot the flush file count and capture the v1 file content. + var afterV1 = ListFlushFiles(); + ClassicAssert.GreaterOrEqual(afterV1.Count, 1, "at least one flush file after v1 flush"); + var v1FileContents = new System.Collections.Generic.Dictionary(); + foreach (var f in afterV1) v1FileContents[f] = File.ReadAllBytes(f); + + // Promote (read forces RIPROMOTE on flushed stub) and mutate to v2. + ClassicAssert.AreEqual("value-v1", (string)db.Execute("RI.GET", "flushtestkey", "field-x")); + db.Execute("RI.SET", "flushtestkey", "field-x", "value-v2"); + + // Force another flush — must create a NEW ..flush.bftree at a distinct addr. + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + // Verify: each pre-existing v1 file STILL exists with byte-identical content. + foreach (var (path, originalBytes) in v1FileContents) + { + ClassicAssert.IsTrue(File.Exists(path), + $"per-flush snapshot file {Path.GetFileName(path)} must remain after subsequent flush"); + var currentBytes = File.ReadAllBytes(path); + ClassicAssert.AreEqual(originalBytes.Length, currentBytes.Length, + $"per-flush snapshot {Path.GetFileName(path)} must NOT be overwritten (size differs)"); + CollectionAssert.AreEqual(originalBytes, currentBytes, + $"per-flush snapshot {Path.GetFileName(path)} must NOT be overwritten (content differs). " + + "Per-flush .flush.bftree files are immutable."); + } + + // Verify: at least one new flush file was created post-v2 (proving second flush + // didn't just overwrite the v1 file in place). + var afterV2 = ListFlushFiles(); + ClassicAssert.Greater(afterV2.Count, afterV1.Count, + "Second flush must create a NEW per-flush file (not overwrite the v1 file). " + + $"Before: {afterV1.Count}, after: {afterV2.Count}"); + } + + // Helper closure + static System.Collections.Generic.List ListFlushFiles() + { + var dir = Path.Combine(TestUtils.MethodTestDir, "Store", "rangeindex"); + if (!Directory.Exists(dir)) return new System.Collections.Generic.List(); + return Directory.GetFiles(dir, "*.flush.bftree").ToList(); + } + } + + /// + /// Verifies that DisposeTreeUnderLock with deleteFiles: false (eviction + /// path) NO-OPS when the source stub has IsTransferred=true, even when its + /// TreeHandle is zero. After PostCopyToTail (compaction) or RIPROMOTE + /// PostCopyUpdater transfers ownership to a destination at the tail, the source + /// record's stub carries IsTransferred=true; the corresponding liveIndexes + /// entry now belongs to the destination, so a later OnEvict on the stale + /// source must NOT remove that entry — doing so would lose checkpoint coverage and + /// DEL-time native-tree disposal. + /// + /// Test mechanics: + /// + /// Construct a 35-byte stub representing the stale source (IsTransferred=true, + /// TreeHandle=0). + /// Call DisposeTreeUnderLock(key, stub, deleteFiles: false). + /// Assert the liveIndexes entry SURVIVED (count unchanged). + /// + /// + /// Discriminating contrast: a stub with IsTransferred=false and + /// TreeHandle=0 (a pure pending entry being evicted before activation) DOES + /// get its liveIndexes entry removed by the same call — proving the + /// IsTransferred check is what makes the no-op precise. + /// + [Test] + public void RIDisposeTreeUnderLockNoOpsOnTransferredSourceTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var rim = server.Provider.StoreWrapper.rangeIndexManager; + ClassicAssert.IsNotNull(rim); + + // Create an RI key so that a real liveIndexes entry exists; this models the scenario + // where we'd later eviction-callback a stale source. + db.Execute("RI.CREATE", "transtest", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + db.Execute("RI.SET", "transtest", "field-x", "value-v1"); + ClassicAssert.AreEqual(1, rim.LiveIndexCount, "tree should be live after creation"); + + // Construct a stub representing a "stale source" (IsTransferred=true, TreeHandle=0) + // with the SAME key as the live entry. This models the byte state of a source record + // after PostCopyToTail-live or RIPROMOTE-live has cleared TreeHandle and set + // IsTransferred on the source. + var staleStub = new byte[RangeIndexManager.IndexSizeBytes]; + // Stub layout: [0..7]=TreeHandle (zero), [33]=flags byte (Transferred bit = 1<<2 = 4) + staleStub[33] = 4; + + // Verify our reading of the stub matches expectation. + ref readonly var stubRef = ref RangeIndexManager.ReadIndex(staleStub); + ClassicAssert.AreEqual(nint.Zero, stubRef.TreeHandle, "test setup: stub TreeHandle should be 0"); + ClassicAssert.IsTrue(stubRef.IsTransferred, "test setup: IsTransferred should be true"); + ClassicAssert.IsFalse(stubRef.IsFlushed, "test setup: IsFlushed should be false"); + + // Call DisposeTreeUnderLock as OnEvict would, with deleteFiles=false (eviction path). + // With IsTransferred=true, this must no-op — NOT remove the entry that belongs to + // the live record at the tail. + // RangeIndexManager hashes the key via PinnedSpanByte.FromPinnedSpan, which captures + // a raw pointer assuming the source is GC-pinned. Use unsafe `fixed` blocks to pin + // the managed byte[] for the duration of each DisposeTreeUnderLock call. + unsafe + { + var keyBytes = System.Text.Encoding.ASCII.GetBytes("transtest"); + fixed (byte* keyPtr = keyBytes) + { + var pinnedKey = new ReadOnlySpan(keyPtr, keyBytes.Length); + rim.DisposeTreeUnderLock(pinnedKey, staleStub, deleteFiles: false); + } + } + + ClassicAssert.AreEqual(1, rim.LiveIndexCount, + "DisposeTreeUnderLock on a stale (IsTransferred=true) source must NOT remove the live entry " + + "that now belongs to the destination at the tail."); + + // Live tree should still be functional. + ClassicAssert.AreEqual("value-v1", (string)db.Execute("RI.GET", "transtest", "field-x")); + + // Discriminating contrast: a stub WITHOUT IsTransferred (pure pending entry, e.g. + // evicted before activation) WOULD remove the entry. This proves the discriminating + // power of the IsTransferred check. + var pendingStub = new byte[RangeIndexManager.IndexSizeBytes]; + // IsTransferred=0, TreeHandle=0 — looks like a pending entry being evicted. + // Use a DIFFERENT key for this part to avoid disturbing the live entry above. + db.Execute("RI.CREATE", "pendkey", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + ClassicAssert.AreEqual(2, rim.LiveIndexCount, "second tree created"); + // Now simulate eviction of a pending-entry stub for "pendkey". + unsafe + { + var pendKeyBytes = System.Text.Encoding.ASCII.GetBytes("pendkey"); + fixed (byte* keyPtr = pendKeyBytes) + { + var pinnedKey = new ReadOnlySpan(keyPtr, pendKeyBytes.Length); + rim.DisposeTreeUnderLock(pinnedKey, pendingStub, deleteFiles: false); + } + } + ClassicAssert.AreEqual(1, rim.LiveIndexCount, + "DisposeTreeUnderLock without IsTransferred should still remove the entry " + + "(pending-eviction case): proves the discriminating power of the IsTransferred check"); + } + + /// + /// Verifies that EnableRangeIndexPreview=true works correctly with + /// CopyReadsToTail=true. RangeIndex Reads go through the dedicated + /// Read_RangeIndex API which suppresses Tsavorite's automatic CTT + /// (RangeIndex performs its own controlled promotion via RIPROMOTE). This test + /// is bounded by [CancelAfter] so a regression that lets CTT reach a + /// RangeIndex stub would surface as a hang/timeout. + /// + [Test] + [CancelAfter(60_000)] + public void RICopyReadsToTailCompatibleTest(System.Threading.CancellationToken cancellationToken) + { + // Recreate the server with CopyReadsToTail=true. + server?.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer( + TestUtils.MethodTestDir, + enableRangeIndexPreview: true, + copyReadsToTail: true); + server.Start(); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + db.Execute("RI.CREATE", "rikey", "DISK", "CACHESIZE", "65536", "MINRECORD", "8"); + for (int i = 0; i < 50; i++) + db.Execute("RI.SET", "rikey", $"field-{i:000}", $"value-{i:000}-pad"); + + // Force the records into the read-only / flushed region so subsequent reads + // would otherwise trigger CTT for the RI stub. With Read_RangeIndex suppressing + // CTT, the read should route through PromoteToTail (RIPROMOTE) instead. + var store = server.Provider.StoreWrapper.store; + store.Log.ShiftReadOnlyAddress(store.Log.TailAddress, wait: true); + + for (int i = 0; i < 50 && !cancellationToken.IsCancellationRequested; i++) + { + var got = (string)db.Execute("RI.GET", "rikey", $"field-{i:000}"); + ClassicAssert.AreEqual($"value-{i:000}-pad", got, $"field-{i:000}"); + } + } + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.rangeindex/TestProjectSetup.cs b/test/standalone/Garnet.test.rangeindex/TestProjectSetup.cs new file mode 100644 index 00000000000..186844656ee --- /dev/null +++ b/test/standalone/Garnet.test.rangeindex/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestRangeIndex); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.scripting/Garnet.test.scripting.csproj b/test/standalone/Garnet.test.scripting/Garnet.test.scripting.csproj new file mode 100644 index 00000000000..dbca1c0c22c --- /dev/null +++ b/test/standalone/Garnet.test.scripting/Garnet.test.scripting.csproj @@ -0,0 +1,55 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test/LuaScriptRunnerTests.cs b/test/standalone/Garnet.test.scripting/LuaScriptRunnerTests.cs similarity index 99% rename from test/Garnet.test/LuaScriptRunnerTests.cs rename to test/standalone/Garnet.test.scripting/LuaScriptRunnerTests.cs index 8ac6ef7d3e0..cf879a46ad9 100644 --- a/test/Garnet.test/LuaScriptRunnerTests.cs +++ b/test/standalone/Garnet.test.scripting/LuaScriptRunnerTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -8,7 +8,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading; -using Allure.NUnit; using Garnet.common; using Garnet.server; using Microsoft.Extensions.Logging; @@ -17,9 +16,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - internal class LuaScriptRunnerTests : AllureTestBase + internal class LuaScriptRunnerTests : TestBase { [Test] public void CannotRunUnsafeScript() diff --git a/test/Garnet.test/LuaScriptTests.cs b/test/standalone/Garnet.test.scripting/LuaScriptTests.cs similarity index 99% rename from test/Garnet.test/LuaScriptTests.cs rename to test/standalone/Garnet.test.scripting/LuaScriptTests.cs index ccf29b52323..67f4b100ce4 100644 --- a/test/Garnet.test/LuaScriptTests.cs +++ b/test/standalone/Garnet.test.scripting/LuaScriptTests.cs @@ -14,7 +14,6 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -23,7 +22,6 @@ namespace Garnet.test { - [AllureNUnit] // Limits chosen here to allow completion - if you have to bump them up, consider that you might have introduced a regression [TestFixture(LuaMemoryManagementMode.Native, "", "")] [TestFixture(LuaMemoryManagementMode.Native, "", "00:00:02")] @@ -31,7 +29,7 @@ namespace Garnet.test [TestFixture(LuaMemoryManagementMode.Tracked, "13m", "")] [TestFixture(LuaMemoryManagementMode.Managed, "", "")] [TestFixture(LuaMemoryManagementMode.Managed, "17m", "")] - public class LuaScriptTests : AllureTestBase + public class LuaScriptTests : TestBase { /// /// Writes it's parameter directly into the response stream, followed by a \r\n. @@ -397,7 +395,7 @@ public void CanDoEvalShaSEMultipleThreads() { tasks[i] = Task.Run(async () => { - for (var ii = 0; ii < numIterations; ++ii) + for (var ii = 0; ii < numIterations; ii++) { _ = db.ScriptEvaluate(script, [(RedisKey)"mykey"]); await Task.Delay(millisecondsDelay: rnd.Next(10, 50)).ConfigureAwait(false); @@ -413,7 +411,7 @@ public void CanDoEvalShaSEMultipleThreads() { tasks[i] = Task.Run(async () => { - for (var ii = 0; ii < numIterations; ++ii) + for (var ii = 0; ii < numIterations; ii++) { _ = db.ScriptEvaluate(script, [(RedisKey)"mykey"]); await Task.Delay(millisecondsDelay: rnd.Next(10, 50)).ConfigureAwait(false); diff --git a/test/Garnet.test/MultiDatabaseTests.cs b/test/standalone/Garnet.test.scripting/MultiDatabaseTests.cs similarity index 94% rename from test/Garnet.test/MultiDatabaseTests.cs rename to test/standalone/Garnet.test.scripting/MultiDatabaseTests.cs index 789ef312d52..36a2ba4222b 100644 --- a/test/Garnet.test/MultiDatabaseTests.cs +++ b/test/standalone/Garnet.test.scripting/MultiDatabaseTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -15,9 +14,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class MultiDatabaseTests : AllureTestBase + public class MultiDatabaseTests : TestBase { GarnetServer server; @@ -29,6 +27,13 @@ public void Setup() server.Start(); } + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + } + [Test] public void MultiDatabaseBasicSelectTestSE() { @@ -37,7 +42,7 @@ public void MultiDatabaseBasicSelectTestSE() var db2Key1 = "db2:key1"; var db2Key2 = "db2:key2"; var db12Key1 = "db12:key1"; - var db12Key2 = "db12:key1"; + var db12Key2 = "db12:key2"; using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db1 = redis.GetDatabase(0); @@ -62,7 +67,7 @@ public void MultiDatabaseBasicSelectTestSE() ClassicAssert.IsFalse(db12.KeyExists(db1Key1)); ClassicAssert.IsFalse(db12.KeyExists(db1Key2)); - db2.StringSet(db12Key2, "db12:value2"); + db2.StringSet(db12Key1, "db12:value2"); db2.SetAdd(db12Key2, [new RedisValue("db12:val2"), new RedisValue("db12:val2")]); ClassicAssert.IsFalse(db12.KeyExists(db12Key1)); @@ -443,7 +448,7 @@ public void MultiDatabaseBasicSelectTestLC() var db1Key1 = "db1:key1"; var db1Key2 = "db1:key2"; var db2Key1 = "db2:key1"; - var db2Key2 = "db2:key1"; + var db2Key2 = "db2:key2"; using var lightClientRequest = TestUtils.CreateRequest(); @@ -1154,7 +1159,8 @@ public void MultiDatabaseSaveRecoverRawStringTest() var garnetServer = redis.GetServer(TestUtils.EndPoint); db1.Execute("SAVE"); //garnetServer.Save(SaveType.BackgroundSave); - while (garnetServer.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); + while (garnetServer.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) + Thread.Sleep(10); } server.Dispose(false); @@ -1369,13 +1375,21 @@ public void MultiDatabaseSaveInProgressTest() db2.ListLeftPush($"k{i}o", new string('x', 256)); } - // Issue general background save - res = db1.Execute("BGSAVE"); - ClassicAssert.AreEqual("Background saving started", res.ToString()); - - // Issue background save to DB 0 while general save is in progress - illegal - Assert.Throws(() => db1.Execute("BGSAVE", "0"), - Encoding.ASCII.GetString(CmdStrings.RESP_ERR_CHECKPOINT_ALREADY_IN_PROGRESS)); + // Issue a general BGSAVE and a per-DB BGSAVE on DB 0 as a single pipelined batch + // via LightClient. Pipelining eliminates the client→server roundtrip between the + // two commands so the per-DB BGSAVE arrives at the server while the general BGSAVE's + // synchronous setup is still holding DB 0's per-DB checkpoint lock — guaranteeing + // the "checkpoint already in progress" error regardless of how fast the actual + // checkpoint completes. Without pipelining, a fast in-memory checkpoint can finish + // before the per-DB BGSAVE arrives over the wire and the test flakes (CI Release). + using (var lcRequest = TestUtils.CreateRequest(countResponseType: CountResponseType.Bytes)) + { + var expectedResponse = + "+Background saving started\r\n" + + $"-{Encoding.ASCII.GetString(CmdStrings.RESP_ERR_CHECKPOINT_ALREADY_IN_PROGRESS)}\r\n"; + var response = lcRequest.Execute("BGSAVE", "BGSAVE 0", expectedResponse.Length); + ClassicAssert.AreEqual(expectedResponse, response); + } int lastsave_old = lastsave; // Wait for save to complete @@ -1388,6 +1402,55 @@ public void MultiDatabaseSaveInProgressTest() } } + [Test] + public void MultiDatabaseGeneralSaveBlocksGeneralSaveTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db0 = redis.GetDatabase(0); + var db1 = redis.GetDatabase(1); + + // Touch DB 1 so there are at least two active databases. With multiple active DBs, + // MultiDatabaseManager.TakeCheckpointAsync acquires multiDbCheckpointingLock, which is + // what makes the second concurrent general BGSAVE fail synchronously below. + db1.StringSet("k", "v"); + + // Add some data so the checkpoint takes a measurable amount of time. + for (var i = 0; i < 1024; i++) + { + db0.StringSet($"k{i}", new string('x', 256)); + db1.StringSet($"k{i}", new string('x', 256)); + } + + // Capture LASTSAVE baseline (long to avoid 2038 truncation in the wait loop below). + var lastsaveBaseline = (long)db0.Execute("LASTSAVE"); + + // Issue general background save. + var res = db0.Execute("BGSAVE"); + ClassicAssert.AreEqual("Background saving started", res.ToString()); + + // Issuing another general BGSAVE while the first is in progress must fail. With multiple + // active DBs, multiDbCheckpointingLock is acquired synchronously by the first BGSAVE, + // so the second one reliably observes the in-progress checkpoint. + // Note: Assert.Throws's second argument is a *failure* message, not the expected exception + // message - assert on Message explicitly. + var ex = Assert.Throws(() => db0.Execute("BGSAVE")); + ClassicAssert.AreEqual( + Encoding.ASCII.GetString(CmdStrings.RESP_ERR_CHECKPOINT_ALREADY_IN_PROGRESS), + ex.Message); + + // Wait (bounded) for the in-flight save to complete by observing LASTSAVE advance past baseline. + var deadline = DateTime.UtcNow.AddSeconds(30); + long lastsave; + do + { + Thread.Sleep(10); + lastsave = (long)db0.Execute("LASTSAVE"); + } + while (lastsave <= lastsaveBaseline && DateTime.UtcNow < deadline); + + ClassicAssert.Greater(lastsave, lastsaveBaseline, "LASTSAVE did not advance within timeout"); + } + [Test] [TestCase(false)] [TestCase(true)] @@ -1677,13 +1740,6 @@ public void MultiDatabaseAofRecoverByDbIdTest() } } - [TearDown] - public void TearDown() - { - server.Dispose(); - TestUtils.OnTearDown(); - } - private (int, int, string, string)[] GenerateDataset(int dbCount, int keyCount) { var data = new (int, int, string, string)[dbCount * keyCount]; diff --git a/test/Garnet.test/RespAofAzureTests.cs b/test/standalone/Garnet.test.scripting/RespAofAzureTests.cs similarity index 99% rename from test/Garnet.test/RespAofAzureTests.cs rename to test/standalone/Garnet.test.scripting/RespAofAzureTests.cs index 7840ffe2846..4bc051294af 100644 --- a/test/Garnet.test/RespAofAzureTests.cs +++ b/test/standalone/Garnet.test.scripting/RespAofAzureTests.cs @@ -1,19 +1,17 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespAofAzureTests : AllureTestBase + public class RespAofAzureTests : TestBase { GarnetServer server; static readonly SortedSetEntry[] entries = diff --git a/test/Garnet.test/RespAofTests.cs b/test/standalone/Garnet.test.scripting/RespAofTests.cs similarity index 89% rename from test/Garnet.test/RespAofTests.cs rename to test/standalone/Garnet.test.scripting/RespAofTests.cs index 1a27f8cd75a..1fcd2392341 100644 --- a/test/Garnet.test/RespAofTests.cs +++ b/test/standalone/Garnet.test.scripting/RespAofTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,7 +6,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespAofTests : AllureTestBase + public class RespAofTests : TestBase { GarnetServer server; private IReadOnlyDictionary respCustomCommandsInfo; @@ -232,7 +230,7 @@ public async Task AofRMWStoreRecoverTestAsync() var db = redis.GetDatabase(0); db.StringSet("SeAofUpsertRecoverTestKey1", "SeAofUpsertRecoverTestValue1", expiry: TimeSpan.FromDays(1), when: When.NotExists); db.StringSet("SeAofUpsertRecoverTestKey2", "SeAofUpsertRecoverTestValue2", expiry: TimeSpan.FromDays(1), when: When.NotExists); - db.Execute("SET", "SeAofUpsertRecoverTestKey3", "SeAofUpsertRecoverTestValue3", "WITHETAG"); + db.Execute("SETWITHETAG", "SeAofUpsertRecoverTestKey3", "SeAofUpsertRecoverTestValue3"); db.Execute("SETIFMATCH", "SeAofUpsertRecoverTestKey3", "UpdatedSeAofUpsertRecoverTestValue3", "1"); db.Execute("SET", "SeAofUpsertRecoverTestKey4", "2"); var res = db.Execute("INCR", "SeAofUpsertRecoverTestKey4"); @@ -315,7 +313,7 @@ public async Task AofExpiryRMWStoreRecoverTestAsync() db.StringSet("AofExpiryRMWStoreRecoverTestKey1", "AofExpiryRMWStoreRecoverTestValue3", expiry: TimeSpan.FromDays(1), when: When.NotExists); db.StringSet("AofExpiryRMWStoreRecoverTestKey2", "AofExpiryRMWStoreRecoverTestValue4", expiry: TimeSpan.FromSeconds(10), when: When.NotExists); - // Set expiry time for 2nd string + // Set expiry time for 1st string db.KeyExpire("AofExpiryRMWStoreRecoverTestKey1", expireTime); Thread.Sleep(2000); @@ -326,7 +324,7 @@ public async Task AofExpiryRMWStoreRecoverTestAsync() // Verify 1st string expiry time var recoveredValueExpTime = db.KeyExpireTime("AofExpiryRMWStoreRecoverTestKey1"); ClassicAssert.IsTrue(recoveredValueExpTime.HasValue); - Assert.That(recoveredValueExpTime.Value, Is.EqualTo(expireTime).Within(TimeSpan.FromMilliseconds(2))); + Assert.That(recoveredValueExpTime.Value, Is.EqualTo(expireTime).Within(TimeSpan.FromMilliseconds(200))); // Verify 2nd string did change recoveredValue = db.StringGet("AofExpiryRMWStoreRecoverTestKey2"); @@ -335,7 +333,7 @@ public async Task AofExpiryRMWStoreRecoverTestAsync() // Verify 2nd string ttl var recoveredValueTtl = db.KeyTimeToLive("AofExpiryRMWStoreRecoverTestKey2"); ClassicAssert.IsTrue(recoveredValueTtl.HasValue); - ClassicAssert.Less(recoveredValueTtl.Value.TotalSeconds, 8); + ClassicAssert.Less(recoveredValueTtl.Value.Milliseconds, 8500); ClassicAssert.Greater(recoveredValueTtl.Value.TotalSeconds, 0); } @@ -370,6 +368,82 @@ public async Task AofExpiryRMWStoreRecoverTestAsync() } } + [Test] + public async Task AofExpiryUpsertStoreRecoverTestAsync() + { + // Test AOF recovery of main store records with an expiry time + + var expireTime = DateTime.UtcNow + TimeSpan.FromMinutes(1); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Add 1st string to main store with long expiry + db.StringSet("AofExpiryUpsertStoreRecoverTestKey1", "AofExpiryUpsertStoreRecoverTestValue1", expiry: TimeSpan.FromDays(1)); + // Add 2nd string to main store with short expiry + db.StringSet("AofExpiryUpsertStoreRecoverTestKey2", "AofExpiryUpsertStoreRecoverTestValue2", expiry: TimeSpan.FromSeconds(1)); + // Wait for 2nd string record to expire + Thread.Sleep(2000); + + // Set value for 2nd record (which has expired) + db.StringSet("AofExpiryUpsertStoreRecoverTestKey2", "AofExpiryUpsertStoreRecoverTestValue4", expiry: TimeSpan.FromSeconds(10)); + + // Set expiry time for 1st string + db.KeyExpire("AofExpiryUpsertStoreRecoverTestKey1", expireTime); + Thread.Sleep(2000); + + // Verify 1st string did not change + var recoveredValue = db.StringGet("AofExpiryUpsertStoreRecoverTestKey1"); + ClassicAssert.AreEqual("AofExpiryUpsertStoreRecoverTestValue1", recoveredValue.ToString()); + + // Verify 1st string expiry time + var recoveredValueExpTime = db.KeyExpireTime("AofExpiryUpsertStoreRecoverTestKey1"); + ClassicAssert.IsTrue(recoveredValueExpTime.HasValue); + Assert.That(recoveredValueExpTime.Value, Is.EqualTo(expireTime).Within(TimeSpan.FromMilliseconds(200))); + + // Verify 2nd string did change + recoveredValue = db.StringGet("AofExpiryUpsertStoreRecoverTestKey2"); + ClassicAssert.AreEqual("AofExpiryUpsertStoreRecoverTestValue4", recoveredValue.ToString()); + + // Verify 2nd string ttl + var recoveredValueTtl = db.KeyTimeToLive("AofExpiryUpsertStoreRecoverTestKey2"); + ClassicAssert.IsTrue(recoveredValueTtl.HasValue); + ClassicAssert.Less(recoveredValueTtl.Value.Milliseconds, 8500); + ClassicAssert.Greater(recoveredValueTtl.Value.TotalSeconds, 0); + } + + // Commit to AOF and restart server + _ = await server.Store.CommitAOFAsync(default); + server.Dispose(false); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, enableAOF: true); + server.Start(); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig())) + { + var db = redis.GetDatabase(0); + + // Verify 1st string value has not changed + var recoveredValue = db.StringGet("AofExpiryUpsertStoreRecoverTestKey1"); + ClassicAssert.AreEqual("AofExpiryUpsertStoreRecoverTestValue1", recoveredValue.ToString()); + + // Verify 1st string expiry time + var recoveredValueExpTime = db.KeyExpireTime("AofExpiryUpsertStoreRecoverTestKey1"); + ClassicAssert.IsTrue(recoveredValueExpTime.HasValue); + Assert.That(recoveredValueExpTime.Value, Is.EqualTo(expireTime).Within(TimeSpan.FromMilliseconds(2))); + + // Verify 2nd string did change + recoveredValue = db.StringGet("AofExpiryUpsertStoreRecoverTestKey2"); + ClassicAssert.AreEqual("AofExpiryUpsertStoreRecoverTestValue4", recoveredValue.ToString()); + + // Verify 2nd string ttl + var recoveredValueTtl = db.KeyTimeToLive("AofExpiryUpsertStoreRecoverTestKey2"); + ClassicAssert.IsTrue(recoveredValueTtl.HasValue); + ClassicAssert.Less(recoveredValueTtl.Value.TotalSeconds, 8); + ClassicAssert.Greater(recoveredValueTtl.Value.TotalSeconds, 0); + } + } + [Test] public async Task AofRMWObjectStoreRecoverTestAsync() { @@ -472,22 +546,40 @@ public async Task AofExpiryRMWObjectStoreRecoverTestAsync() db.ListRightPush(key1, values1_1); db.KeyExpire(key1, expireTime); + var recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + // Add 2nd list to object store with short expiry db.ListRightPush(key2, values2_1); db.KeyExpire(key2, TimeSpan.FromSeconds(1)); + recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + // Wait for 2nd list record to expire Thread.Sleep(2000); + recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + // Push to elements to 1st list and 2nd list (now empty) db.ListRightPush(key1, values1_2); db.ListRightPush(key2, values2_2); + recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + // Add longer expiry to 2nd list db.KeyExpire(key2, TimeSpan.FromSeconds(15)); + recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + Thread.Sleep(2000); + recoveredValuesExpTimeXxx = db.KeyExpireTime(key1); + ClassicAssert.IsTrue(recoveredValuesExpTimeXxx.HasValue); + // Verify 1st list has values from both pushes var recoveredValues = db.ListRange(key1); CollectionAssert.AreEqual(values1_1.Union(values1_2), recoveredValues); @@ -1063,8 +1155,8 @@ public async Task AofTransactionFinalizeStepTestAsync() } // Regression test for https://github.com/microsoft/garnet/issues/1749 - // A SET / RMW / DEL whose AOF entry exceeds AofPageSize used to leave the per-bucket transient X-lock - // held forever (the AOF Enqueue threw "Entry does not fit on page" before TransientXUnlock could run), + // A SET / RMW / DEL whose AOF entry exceeds AofPageSize used to leave the per-bucket ephemeral X-lock + // held forever (the AOF Enqueue threw "Entry does not fit on page" before EphemeralXUnlock could run), // pinning subsequent ops on the same key in an infinite RETRY_LATER loop and burning 100% CPU. // The server is rebuilt with a small AofPageSize to trigger the oversize path with small payloads. [Test] @@ -1087,19 +1179,19 @@ public void OversizedAofEntryDoesNotHangServer() } // 2) From a fresh connection issue several operations on the same key. Before the fix these would - // spin forever inside Tsavorite waiting on the leaked transient X-lock. With the fix they return + // spin forever inside Tsavorite waiting on the leaked ephemeral X-lock. With the fix they return // promptly. We don't care what GET returns, only that the server does not hang. using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig() + ",syncTimeout=3000")) { var db = redis.GetDatabase(0); - // Reads use a transient S-lock on the same hash bucket, so they block on a leaked X-lock. + // Reads use an ephemeral S-lock on the same hash bucket, so they block on a leaked X-lock. Assert.DoesNotThrow(() => _ = db.StringGet(key)); - // RMW operations (e.g. APPEND) use the same TransientX path as SET; verify they don't hang either. + // RMW operations (e.g. APPEND) use the same EphemeralX path as SET; verify they don't hang either. Assert.DoesNotThrow(() => _ = db.StringAppend(key, "x")); - // Delete also takes the transient X-lock; verify it can complete. + // Delete also takes the ephemeral X-lock; verify it can complete. Assert.DoesNotThrow(() => _ = db.KeyDelete(key)); // A small SET on the same key after recovery must succeed end-to-end. diff --git a/test/Garnet.test/RespCustomCommandTests.cs b/test/standalone/Garnet.test.scripting/RespCustomCommandTests.cs similarity index 87% rename from test/Garnet.test/RespCustomCommandTests.cs rename to test/standalone/Garnet.test.scripting/RespCustomCommandTests.cs index 3eb4a98b5ad..e66b7b08cc5 100644 --- a/test/Garnet.test/RespCustomCommandTests.cs +++ b/test/standalone/Garnet.test.scripting/RespCustomCommandTests.cs @@ -11,7 +11,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using GarnetJSON; @@ -22,43 +21,51 @@ namespace Garnet.test { - public class LargeGet : CustomProcedure + /// + /// Validates that data matches the deterministic pattern (i % 251) used by scratch buffer tests. + /// + internal static class ScratchBufferTestHelper { - public override bool Execute(TGarnetApi garnetApi, ref CustomProcedureInput procInput, ref MemoryResult output) + internal static bool ValidateContent(ReadOnlySpan data) { - static bool ResetBuffer(TGarnetApi garnetApi, ref MemoryResult output, int buffOffset) + if (data.Length == 0) return false; + for (int i = 0; i < data.Length; i++) { - bool status = garnetApi.ResetScratchBuffer(buffOffset); - if (!status) - WriteError(ref output, "ERR ResetScratchBuffer failed"); - - return status; + if (data[i] != (byte)(i % 251)) + return false; } + return true; + } + } + public class LargeGet : CustomProcedure + { + public override bool Execute(TGarnetApi garnetApi, ref CustomProcedureInput procInput, ref MemoryResult output) + { var offset = 0; var key = GetNextArg(ref procInput, ref offset); - var buffOffset = garnetApi.GetScratchBufferOffset(); + // Test SBA offset management: GET output goes to ScratchBufferAllocator for (var i = 0; i < 120_000; i++) { - garnetApi.GET(key, out var outval); + garnetApi.GET(key, out PinnedSpanByte outval); if (i % 100 == 0) - { - if (!ResetBuffer(garnetApi, ref output, buffOffset)) - return false; - } + garnetApi.ResetScratchBuffer(); } - buffOffset = garnetApi.GetScratchBufferOffset(); - garnetApi.GET(key, out var outval1); - garnetApi.GET(key, out var outval2); - if (!ResetBuffer(garnetApi, ref output, buffOffset)) return false; + garnetApi.GET(key, out PinnedSpanByte outval1); + garnetApi.GET(key, out PinnedSpanByte outval2); + garnetApi.ResetScratchBuffer(); - buffOffset = garnetApi.GetScratchBufferOffset(); var hashKey = GetNextArg(ref procInput, ref offset); var field = GetNextArg(ref procInput, ref offset); garnetApi.HashGet(hashKey, field, out var value); - if (!ResetBuffer(garnetApi, ref output, buffOffset)) return false; + if (!ScratchBufferTestHelper.ValidateContent(value.ReadOnlySpan)) + { + WriteError(ref output, "ERR HashGet returned corrupted data"); + return false; + } + garnetApi.ResetScratchBuffer(); return true; } @@ -69,7 +76,7 @@ public class LargeGetTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Shared, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Shared, StoreType.Main); return true; } @@ -77,18 +84,13 @@ public override void Main(TGarnetApi garnetApi, ref CustomProcedureI { int offset = 0; var key = GetNextArg(ref procInput, ref offset); - var buffOffset = garnetApi.GetScratchBufferOffset(); + + // Test SBA offset management: GET output goes to ScratchBufferAllocator for (int i = 0; i < 120_000; i++) { - garnetApi.GET(key, out var outval); + garnetApi.GET(key, out PinnedSpanByte outval); if (i % 100 == 0) - { - if (!garnetApi.ResetScratchBuffer(buffOffset)) - { - WriteError(ref output, "ERR ResetScratchBuffer failed"); - return; - } - } + garnetApi.ResetScratchBuffer(); } } } @@ -100,25 +102,27 @@ public override bool Execute(TGarnetApi garnetApi, ref CustomProcedu var offset = 0; var key = GetNextArg(ref procInput, ref offset); - var buffOffset1 = garnetApi.GetScratchBufferOffset(); - garnetApi.GET(key, out var outval1); + // Test scratch buffer reset with GET (output goes to ScratchBufferAllocator) + garnetApi.GET(key, out PinnedSpanByte outval1); + garnetApi.GET(key, out PinnedSpanByte outval2); - var buffOffset2 = garnetApi.GetScratchBufferOffset(); - garnetApi.GET(key, out var outval2); - - if (!garnetApi.ResetScratchBuffer(buffOffset1)) + // Verify GET results contain valid data (i % 251 pattern) + if (!ScratchBufferTestHelper.ValidateContent(outval1.ReadOnlySpan)) { - WriteError(ref output, "ERR ResetScratchBuffer failed"); + WriteError(ref output, "ERR GET returned corrupted data for outval1"); return false; } - // Previous reset call would have shrunk the buffer. This call should fail otherwise it will expand the buffer. - if (garnetApi.ResetScratchBuffer(buffOffset2)) + // Verify both results are identical (validates data integrity) + if (!outval1.ReadOnlySpan.SequenceEqual(outval2.ReadOnlySpan)) { - WriteError(ref output, "ERR ResetScratchBuffer shouldn't expand the buffer"); + WriteError(ref output, "ERR GET results should be identical"); return false; } + // Reset scratch buffer (full reset, discards all GET results) + garnetApi.ResetScratchBuffer(); + return true; } } @@ -154,7 +158,7 @@ public override bool Execute(TGarnetApi garnetApi, ref CustomProcedu garnetApi.Increment(keyToIncrement, out long _, 1); var keyToReturn = GetNextArg(ref procInput, ref offset); - garnetApi.GET(keyToReturn, out ArgSlice outval); + garnetApi.GET(keyToReturn, out PinnedSpanByte outval); WriteBulkString(ref output, outval.Span); return true; } @@ -165,8 +169,8 @@ public class RandomSubstituteOrExpandValForKeyTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); return true; } @@ -179,7 +183,7 @@ public override unsafe void Main(TGarnetApi garnetApi, ref CustomPro // key will have an etag associated with it already but the transaction should not be able to see it. // if the transaction needs to see it, then it can send GET with cmd as GETWITHETAG - garnetApi.GET(key, out ArgSlice outval); + garnetApi.GET(key, out PinnedSpanByte outval); List valueToMessWith = outval.ToArray().ToList(); @@ -201,30 +205,55 @@ public override unsafe void Main(TGarnetApi garnetApi, ref CustomPro valueToMessWith.RemoveAt(valueToMessWith.Count - 1); } - RawStringInput input = new RawStringInput(RespCommand.SET); - input.header.cmd = RespCommand.SET; - // if we send a SET we must explictly ask it to retain etag, and use conditional set - input.header.SetWithEtagFlag(); + StringInput input = new StringInput(RespCommand.SETWITHETAG); fixed (byte* valuePtr = valueToMessWith.ToArray()) { - ArgSlice valForKey1 = new ArgSlice(valuePtr, valueToMessWith.Count); + PinnedSpanByte valForKey1 = PinnedSpanByte.FromPinnedPointer(valuePtr, valueToMessWith.Count); input.parseState.InitializeWithArgument(valForKey1); - // since we are setting with retain to etag, this change should be reflected in an etag update - garnetApi.SET_Conditional(key, ref input); + var etagOutput = new StringOutput(); + garnetApi.SET_ETagConditional(key, ref input, ref etagOutput); } - var keyToIncrment = GetNextArg(ref procInput, ref offset); - // for a non SET command the etag should be invisible and be updated automatically + // non-ETag commands are ETag-blind garnetApi.Increment(keyToIncrment, out long _, 1); } } - [AllureNUnit] + public sealed class CustomNotFoundFactory : CustomObjectFactory + { + public override CustomObjectBase Create(byte type) => null; + public override CustomObjectBase Deserialize(byte type, BinaryReader reader) => null; + } + + public sealed class CustomNotFoundFunctions : CustomObjectFunctions + { + public override void NotFound(ReadOnlySpan key, ref ObjectInput input, ref RespMemoryWriter writer) + { + writer.WriteBulkString(Encoding.UTF8.GetBytes($"Did not find: {Encoding.UTF8.GetString(key)}")); + } + } + + public sealed class CustomNotFoundStringFunctions : CustomRawStringFunctions + { + public override bool CopyUpdater(ReadOnlySpan key, ref StringInput input, ReadOnlySpan oldValue, Span newValue, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); + public override int GetInitialLength(ref StringInput input) => throw new NotImplementedException(); + public override int GetLength(ReadOnlySpan value, ref StringInput input) => throw new NotImplementedException(); + public override bool InitialUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); + public override bool InPlaceUpdater(ReadOnlySpan key, ref StringInput input, Span value, ref int valueLength, ref RespMemoryWriter writer, ref RMWInfo rmwInfo) => throw new NotImplementedException(); + + public override bool Reader(ReadOnlySpan key, ref StringInput input, ReadOnlySpan value, ref RespMemoryWriter writer, ref ReadInfo readInfo) => throw new NotImplementedException(); + + public override void NotFound(ReadOnlySpan key, ref StringInput input, ref RespMemoryWriter writer) + { + writer.WriteBulkString(Encoding.UTF8.GetBytes($"Did not find: {Encoding.UTF8.GetString(key)}")); + } + } + [TestFixture] - public class RespCustomCommandTests : AllureTestBase + public class RespCustomCommandTests : TestBase { GarnetServer server; private string _extTestDir1; @@ -550,7 +579,7 @@ public void CustomObjectCommandTest1() var result = db.Execute("MEMORY", "USAGE", mainkey); var actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - var expectedResponse = 272; + var expectedResponse = 304; ClassicAssert.AreEqual(expectedResponse, actualValue); string key2 = "mykey2"; @@ -562,7 +591,7 @@ public void CustomObjectCommandTest1() result = db.Execute("MEMORY", "USAGE", mainkey); actualValue = ResultType.Integer == result.Resp2Type ? Int32.Parse(result.ToString()) : -1; - expectedResponse = 408; + expectedResponse = 440; ClassicAssert.AreEqual(expectedResponse, actualValue); } @@ -825,6 +854,8 @@ public void CustomProcedureFreeBufferTest() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); byte[] value = new byte[10_000]; + for (int i = 0; i < value.Length; i++) + value[i] = (byte)(i % 251); // deterministic pattern for content validation db.StringSet(key, value); db.HashSet(hashKey, [new HashEntry(hashField, value)]); @@ -849,6 +880,8 @@ public void CustomTxnFreeBufferTest() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); byte[] value = new byte[10_000]; + for (int i = 0; i < value.Length; i++) + value[i] = (byte)(i % 251); // deterministic pattern for content validation db.StringSet(key, value); db.HashSet(hashKey, [new HashEntry(hashField, value)]); @@ -871,6 +904,8 @@ public void CustomProcedureOutOfOrderFreeBufferTest() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); byte[] value = new byte[10_000]; + for (int i = 0; i < value.Length; i++) + value[i] = (byte)(i % 251); // deterministic pattern for content validation db.StringSet(key, value); var result = db.Execute("OUTOFORDERFREE", key); @@ -977,14 +1012,15 @@ public void RegisterCustomCommandTest() [.. args]); // Test READWRITETX - string key = "readkey"; + string key1 = "readkey1"; + string key2 = "readkey2"; string value = "foovalue0"; - db.StringSet(key, value); + db.StringSet(key1, value); string writekey1 = "writekey1"; string writekey2 = "writekey2"; - var result = db.Execute("READWRITETX", key, writekey1, writekey2); + var result = db.Execute("READWRITETX", key1, writekey1, writekey2); ClassicAssert.AreEqual("SUCCESS", (string)result); // Read keys to verify transaction succeeded @@ -1000,32 +1036,32 @@ public void RegisterCustomCommandTest() string newValue2 = "foovalue2"; // This conditional set should pass (prefix matches) - result = db.Execute("SETIFPM", key, newValue1, "foo"); + result = db.Execute("SETIFPM", key1, newValue1, "foo"); ClassicAssert.AreEqual("OK", (string)result); - retValue = db.StringGet(key); + retValue = db.StringGet(key1); ClassicAssert.AreEqual(newValue1, retValue); // This conditional set should fail (prefix does not match) - result = db.Execute("SETIFPM", key, newValue2, "bar"); + result = db.Execute("SETIFPM", key1, newValue2, "bar"); ClassicAssert.AreEqual("OK", (string)result); - retValue = db.StringGet(key); + retValue = db.StringGet(key1); ClassicAssert.AreEqual(newValue1, retValue); // Test MYDICTSET string newKey1 = "newkey1"; string newKey2 = "newkey2"; - db.Execute("MYDICTSET", key, newKey1, newValue1); + db.Execute("MYDICTSET", key2, newKey1, newValue1); - var dictVal = db.Execute("MYDICTGET", key, newKey1); + var dictVal = db.Execute("MYDICTGET", key2, newKey1); ClassicAssert.AreEqual(newValue1, (string)dictVal); - db.Execute("MYDICTSET", key, newKey2, newValue2); + db.Execute("MYDICTSET", key2, newKey2, newValue2); // Test MYDICTGET - dictVal = db.Execute("MYDICTGET", key, newKey2); + dictVal = db.Execute("MYDICTGET", key2, newKey2); ClassicAssert.AreEqual(newValue2, (string)dictVal); } @@ -1446,21 +1482,12 @@ public void CustomTxnEtagInteractionTest() try { - db.Execute("SET", key1, value1, "WITHETAG"); - db.Execute("SET", key2, value2, "WITHETAG"); + db.Execute("SETWITHETAG", key1, value1); + db.Execute("SETWITHETAG", key2, value2); RedisResult result = db.Execute("RANDOPS", key1, key2); ClassicAssert.AreEqual("OK", result.ToString()); - - // check GETWITHETAG shows updated etag and expected values for both - RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key1); - ClassicAssert.AreEqual("2", res[0].ToString()); - ClassicAssert.IsTrue(res[1].ToString().All(c => c - 'a' >= 0 && c - 'a' < 26)); - - res = (RedisResult[])db.Execute("GETWITHETAG", key2); - ClassicAssert.AreEqual("2", res[0].ToString()); - ClassicAssert.AreEqual("18", res[1].ToString()); } catch (RedisServerException rse) { @@ -1484,29 +1511,60 @@ public void CustomProcEtagInteractionTest() try { - db.Execute("SET", key1, value1, "WITHETAG"); - db.Execute("SET", key2, value2, "WITHETAG"); + db.Execute("SETWITHETAG", key1, value1); + db.Execute("SETWITHETAG", key2, value2); // incr key2, and just get key1 RedisResult result = db.Execute("INCRGET", key2, key1); ClassicAssert.AreEqual(value1, result.ToString()); - - // check GETWITHETAG shows updated etag and expected values for both - RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key1); - // etag not updated for this - ClassicAssert.AreEqual("1", res[0].ToString()); - ClassicAssert.AreEqual(value1, res[1].ToString()); - - res = (RedisResult[])db.Execute("GETWITHETAG", key2); - // etag updated for this - ClassicAssert.AreEqual("2", res[0].ToString()); - ClassicAssert.AreEqual("257", res[1].ToString()); } catch (RedisServerException rse) { ClassicAssert.Fail(rse.Message); } } + + [Test] + public void CustomNotFoundResponseTest() + { + _ = server.Register.NewCommand("CUSTNF", CommandType.Read, new CustomNotFoundFactory(), new CustomNotFoundFunctions(), new RespCommandsInfo { Arity = 2 }); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(); + + var resp0 = (string)db.Execute("CUSTNF", "short string"); + ClassicAssert.AreEqual("Did not find: short string", resp0); + + var veryLongStringBytes = new byte[128 * 1024]; + for (var i = 0; i < veryLongStringBytes.Length; i++) + { + veryLongStringBytes[i] = (byte)('a' + (i % 26)); + } + + var resp1 = (string)db.Execute("CUSTNF", veryLongStringBytes); + ClassicAssert.AreEqual($"Did not find: {Encoding.UTF8.GetString(veryLongStringBytes)}", resp1); + } + + [Test] + public void CustomNotFoundStringResponseTest() + { + _ = server.Register.NewCommand("CUSTSNF", CommandType.Read, new CustomNotFoundStringFunctions(), new RespCommandsInfo { Arity = 2 }); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(); + + var resp0 = (string)db.Execute("CUSTSNF", "short string"); + ClassicAssert.AreEqual("Did not find: short string", resp0); + + var veryLongStringBytes = new byte[128 * 1024]; + for (var i = 0; i < veryLongStringBytes.Length; i++) + { + veryLongStringBytes[i] = (byte)('a' + (i % 26)); + } + + var resp1 = (string)db.Execute("CUSTSNF", veryLongStringBytes); + ClassicAssert.AreEqual($"Did not find: {Encoding.UTF8.GetString(veryLongStringBytes)}", resp1); + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespModuleTests.cs b/test/standalone/Garnet.test.scripting/RespModuleTests.cs similarity index 98% rename from test/Garnet.test/RespModuleTests.cs rename to test/standalone/Garnet.test.scripting/RespModuleTests.cs index 328c5e82cb1..c3b882d3da7 100644 --- a/test/Garnet.test/RespModuleTests.cs +++ b/test/standalone/Garnet.test.scripting/RespModuleTests.cs @@ -1,19 +1,17 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.IO; using System.Reflection; using System.Runtime.InteropServices; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespModuleTests : AllureTestBase + public class RespModuleTests : TestBase { GarnetServer server; private string testModuleDir; @@ -349,9 +347,8 @@ public void TestNoOpModule(bool loadFromDll) } [NonParallelizable] - [AllureNUnit] [TestFixture] - public class RespModuleAdditionalTests : AllureTestBase + public class RespModuleAdditionalTests : TestBase { private string testModuleDir; string binPath; diff --git a/test/Garnet.test/RespTransactionProcTests.cs b/test/standalone/Garnet.test.scripting/RespTransactionProcTests.cs similarity index 98% rename from test/Garnet.test/RespTransactionProcTests.cs rename to test/standalone/Garnet.test.scripting/RespTransactionProcTests.cs index d95585d2da8..534e7b8c9a6 100644 --- a/test/Garnet.test/RespTransactionProcTests.cs +++ b/test/standalone/Garnet.test.scripting/RespTransactionProcTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Threading; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -10,9 +9,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespTransactionProcTests : AllureTestBase + public class RespTransactionProcTests : TestBase { GarnetServer server; @@ -434,6 +432,7 @@ public void TransactionProcMSetPxTest() } [Test] + //[Repeat(10000)] public void TransactionProcMGetIfPMTest() { server.Register.NewTransactionProc("MSETPX", () => new MSetPxTxn()); @@ -478,9 +477,7 @@ public void TransactionProcMGetIfPMTest() // Set keys for (int i = 0; i < NumKeys; i++) - { args2[i + 1] = $"key{i}"; - } // Execute transaction var result2 = (string[])db.Execute("MGETIFPM", args2); @@ -488,11 +485,10 @@ public void TransactionProcMGetIfPMTest() // Verify results int expectedCount = NumKeys - 9; // only values with specified prefix ClassicAssert.AreEqual(2 * expectedCount, result2.Length); + // Verify that keys have the correct prefix for (int i = 0; i < expectedCount; i++) - { ClassicAssert.AreEqual(prefix, result2[2 * i + 1].Substring(0, prefix.Length)); - } } } } \ No newline at end of file diff --git a/test/standalone/Garnet.test.scripting/TestProjectSetup.cs b/test/standalone/Garnet.test.scripting/TestProjectSetup.cs new file mode 100644 index 00000000000..4aa2efb67b8 --- /dev/null +++ b/test/standalone/Garnet.test.scripting/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestScripting); + } +} \ No newline at end of file diff --git a/test/Garnet.test/TransactionTests.cs b/test/standalone/Garnet.test.scripting/TransactionTests.cs similarity index 98% rename from test/Garnet.test/TransactionTests.cs rename to test/standalone/Garnet.test.scripting/TransactionTests.cs index 676194b2337..2cfc6ca6f3a 100644 --- a/test/Garnet.test/TransactionTests.cs +++ b/test/standalone/Garnet.test.scripting/TransactionTests.cs @@ -4,7 +4,6 @@ using System; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,9 +11,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class TransactionTests : AllureTestBase + public class TransactionTests : TestBase { GarnetServer server; @@ -391,12 +389,12 @@ public async Task TxnCommandCoverage() client.Connect(); client.Execute("MULTI"); var result = await client.ExecuteAsync([.. command]).ConfigureAwait(false); - ClassicAssert.AreEqual("QUEUED", result, commandInfo.Name + " failed transaction coverage"); + ClassicAssert.AreEqual("QUEUED", result, commandInfo.Name + " failed transaction coverage inline"); client.Execute("DISCARD"); } - catch + catch (Exception ex) { - Assert.Fail($"{commandInfo.Name} failed transaction coverage"); + Assert.Fail($"{commandInfo.Name} failed transaction coverage with exception: {ex.Message}"); } } } @@ -444,7 +442,7 @@ public async Task WatchTestWithSetWithEtag() var lightClientRequest = TestUtils.CreateRequest(); var expectedResponse = ":1\r\n"; - var response = lightClientRequest.SendCommand("SET key1 value1 WITHETAG"); + var response = lightClientRequest.SendCommand("SETWITHETAG key1 value1"); TestUtils.AssertEqualUpToExpectedLength(expectedResponse, response); expectedResponse = "+OK\r\n"; @@ -464,7 +462,7 @@ public async Task WatchTestWithSetWithEtag() await Task.Run(() => { using var lightClientRequestCopy = TestUtils.CreateRequest(); - string command = "SET key1 value1_updated WITHETAG"; + string command = "SETWITHETAG key1 value1_updated"; lightClientRequestCopy.SendCommand(command); }).ConfigureAwait(false); @@ -478,11 +476,11 @@ await Task.Run(() => lightClientRequest.SendCommand("GET key1"); lightClientRequest.SendCommand("SET key2 value2"); // check that all the etag commands can be called inside a transaction - lightClientRequest.SendCommand("SET key3 value2 WITHETAG"); + lightClientRequest.SendCommand("SETWITHETAG key3 value2"); lightClientRequest.SendCommand("GETWITHETAG key3"); lightClientRequest.SendCommand("GETIFNOTMATCH key3 1"); lightClientRequest.SendCommand("SETIFMATCH key3 anotherVal 1"); - lightClientRequest.SendCommand("SET key3 arandomval WITHETAG"); + lightClientRequest.SendCommand("SETWITHETAG key3 arandomval"); response = lightClientRequest.SendCommand("EXEC"); diff --git a/test/standalone/Garnet.test.vectorset/Garnet.test.vectorset.csproj b/test/standalone/Garnet.test.vectorset/Garnet.test.vectorset.csproj new file mode 100644 index 00000000000..340c2ffce68 --- /dev/null +++ b/test/standalone/Garnet.test.vectorset/Garnet.test.vectorset.csproj @@ -0,0 +1,46 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + PreserveNewest + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + false + + diff --git a/test/Garnet.test/RespVectorSetTests.cs b/test/standalone/Garnet.test.vectorset/RespVectorSetTests.cs similarity index 89% rename from test/Garnet.test/RespVectorSetTests.cs rename to test/standalone/Garnet.test.vectorset/RespVectorSetTests.cs index b247f6a88a1..f2bdbf5f010 100644 --- a/test/Garnet.test/RespVectorSetTests.cs +++ b/test/standalone/Garnet.test.vectorset/RespVectorSetTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -9,7 +9,6 @@ using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -19,9 +18,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespVectorSetTests : AllureTestBase + public class RespVectorSetTests : TestBase { private const string DefaultAOFMemorySize = "2g"; // Very large because CI boxes have low IOPS, so try and flush to disk veeeeeery rarely @@ -66,28 +64,6 @@ public void DisabledWithFeatureFlag() } } - [Test] - public void OversizedRejected() - { - var options = GetOpts(server); - - var overflowSizeBytes = (int)(GarnetServerOptions.ParseSize(options.PageSize, out _) * 2); - - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var oversizedVectorData = Enumerable.Repeat(1, overflowSizeBytes).ToArray(); - var oversideAttribute = Enumerable.Repeat(2, overflowSizeBytes).ToArray(); - - var exc1 = ClassicAssert.Throws(() => db.Execute("VADD", ["foo", "XB8", oversizedVectorData, new byte[] { 0, 0, 0, 0 }, "XPREQ8"])); - ClassicAssert.AreEqual("ERR Vector exceed configured page size", exc1.Message); - - var basicVectorData = Enumerable.Repeat(3, 75).ToArray(); - - var exc2 = ClassicAssert.Throws(() => db.Execute("VADD", ["foo", "XB8", basicVectorData, new byte[] { 0, 0, 0, 1 }, "XPREQ8", "SETATTR", oversideAttribute])); - ClassicAssert.AreEqual("ERR Attribute exceed configured page size", exc2.Message); - } - [Test] public void WrongTypeForVectorSetOpsOnNonVectorSetKeys() { @@ -207,7 +183,7 @@ public void VADD() // Mismatch vector size for projection var exc3 = ClassicAssert.Throws(() => db.Execute("VADD", ["fizz", "REDUCE", "50", "VALUES", "5", "1.0", "2.0", "3.0", "4.0", "5.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32"])); - ClassicAssert.AreEqual("ERR Vector dimension mismatch - got 5 but set has 75", exc3.Message); + ClassicAssert.AreEqual("ERR REDUCE dimension must be <= vector dimensions", exc3.Message); } [Test] @@ -326,15 +302,15 @@ public void VADDErrors() // M out of range (Redis imposes M >= 4 and m <= 4096 var exc13 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "M", "1"])); - ClassicAssert.AreEqual("ERR invalid M", exc13.Message); + ClassicAssert.AreEqual("ERR M must be an integer between 4 and 4096", exc13.Message); var exc14 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "M", "10000"])); - ClassicAssert.AreEqual("ERR invalid M", exc14.Message); + ClassicAssert.AreEqual("ERR M must be an integer between 4 and 4096", exc14.Message); // Missing/bad option value var exc20 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "EF"])); ClassicAssert.AreEqual("ERR invalid option after element", exc20.Message); var exc21 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "EF", "0"])); - ClassicAssert.AreEqual("ERR invalid EF", exc21.Message); + ClassicAssert.AreEqual("ERR EF must be an integer between 1 and 1000000", exc21.Message); var exc22 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "SETATTR"])); ClassicAssert.AreEqual("ERR invalid option after element", exc22.Message); var exc23 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "1", "2.0", "bar", "M"])); @@ -387,6 +363,22 @@ public void VADDErrors() ClassicAssert.AreEqual("ERR invalid option after element", exc31.Message); var exc32 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "75", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "bar", "NOQUANT", "XDISTANCE_METRIC", "FOO"])); ClassicAssert.AreEqual("ERR invalid XDISTANCE_METRIC", exc32.Message); + + // Invalid vector type keyword (not FP32, VALUES, or XB8) + var exc40 = ClassicAssert.Throws(() => db.Execute("VADD", ["mykey", "GARBAGE", "data", "elem1"])); + ClassicAssert.AreEqual("ERR invalid vector specification", exc40.Message); + + // VALUES count exceeding MaxVectorDimensions (65536) must be rejected + var exc41 = ClassicAssert.Throws(() => db.Execute("VADD", ["foo", "VALUES", "100000", "1.0", "elem"])); + ClassicAssert.IsTrue(exc41.Message.Contains("maximum"), $"Expected dimension limit error, got: {exc41.Message}"); + + // EF exceeding MaxExplorationFactor (1,000,000) must be rejected + var exc42 = ClassicAssert.Throws(() => db.Execute("VADD", ["foo", "VALUES", "3", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "2000000000", "M", "32"])); + ClassicAssert.IsTrue(exc42.Message.Contains("EF must be an integer between"), $"Expected EF validation error, got: {exc42.Message}"); + + // REDUCE dim exceeding vector dimensions must be rejected + var exc43 = ClassicAssert.Throws(() => db.Execute("VADD", ["foo", "REDUCE", "100000", "VALUES", "3", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32"])); + ClassicAssert.IsTrue(exc43.Message.Contains("REDUCE dimension must be <= vector dimensions"), $"Expected REDUCE dimension limit error, got: {exc43.Message}"); var exc33 = ClassicAssert.Throws(() => db.Execute("VADD", [vectorSetKey, "VALUES", "75", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "bar", "NOQUANT", "XDISTANCE_METRIC", "XCOSINE_NORMALIZED"])); ClassicAssert.AreEqual("ERR Distance metric mismatch - got XCosine_Normalized but set has L2", exc33.Message); } @@ -841,6 +833,61 @@ public void VSIMWithAdvancedFilteringELEWithoutWithAttribs() ClassicAssert.AreEqual(2, res3.Length, "ELE + FILTER without WITHATTRIBS: arithmetic and comparison"); } + [Test] + public void VSIMErrors() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + _ = db.KeyDelete("foo"); + + // Add a vector so the key exists (needed for FILTER-EF test) + var res1 = db.Execute("VADD", ["foo", "VALUES", "3", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32", "SETATTR", "{\"year\":1980}"]); + ClassicAssert.AreEqual(1, (int)res1); + + // FILTER-EF exceeding MaxRetrieveCount must be rejected + var exc1 = ClassicAssert.Throws(() => db.Execute("VSIM", ["foo", "VALUES", "3", "0.0", "0.0", "0.0", "FILTER", ".year > 1950", "FILTER-EF", "999999999", "COUNT", "3", "WITHATTRIBS"])); + ClassicAssert.AreEqual("ERR FILTER-EF must be an integer between 0 and 100000000", exc1.Message); + + // COUNT exceeding MaxRetrieveCount must be rejected + var exc2 = ClassicAssert.Throws(() => db.Execute("VSIM", ["foo", "VALUES", "3", "0.0", "0.0", "0.0", "COUNT", "999999999"])); + ClassicAssert.AreEqual("ERR COUNT must be an integer between 0 and 100000000", exc2.Message); + + // VALUES count exceeding MaxVectorDimensions (65536) must be rejected + var exc3 = ClassicAssert.Throws(() => db.Execute("VSIM", ["foo", "VALUES", "100000", "1.0"])); + ClassicAssert.AreEqual("ERR vector exceeds maximum of 65536 dimensions", exc3.Message); + + // EF exceeding MaxExplorationFactor (1,000,000) must be rejected + var exc4 = ClassicAssert.Throws(() => db.Execute("VSIM", ["foo", "VALUES", "3", "0.0", "0.0", "0.0", "EF", "2000000000"])); + ClassicAssert.AreEqual("ERR EF must be an integer between 1 and 1000000", exc4.Message); + } + + [Test] + public void VSIMWithDefaultFilterEFOverflowDoesNotCrash() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + _ = db.KeyDelete("foo"); + + // Add a vector with attributes so FILTER can be used + var res1 = db.Execute("VADD", ["foo", "VALUES", "3", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32", "SETATTR", "{\"year\":1980}"]); + ClassicAssert.AreEqual(1, (int)res1); + + // Verify that a moderate COUNT with FILTER (no explicit FILTER-EF) works correctly. + // The default maxFilteringEffort = count*200. With count=1000, that's 200,000 which is safe. + // This validates the code path through the (long) cast fix without hitting resource limits. + var res = (byte[][])db.Execute("VSIM", ["foo", "VALUES", "3", "0.0", "0.0", "0.0", "FILTER", ".year > 1950", "COUNT", "1000", "WITHATTRIBS"]); + ClassicAssert.AreEqual(2, res.Length, "Should return 1 result (1 pair of id+attribute) for year > 1950"); + + // Verify that COUNT values which would overflow count*200 in int32 are rejected. + // 10,737,419 * 200 = 2,147,483,800 > int32.MaxValue. + // Our (long) cast prevents the overflow, but MaxRetrieveCount caps COUNT itself. + // Any COUNT above MaxRetrieveCount (~178M) is rejected at parse time. + var ex = Assert.Throws(() => db.Execute("VSIM", ["foo", "VALUES", "3", "0.0", "0.0", "0.0", "FILTER", ".year > 1950", "COUNT", "999999999", "WITHATTRIBS"])); + ClassicAssert.IsTrue(ex.Message.Contains("COUNT must be an integer between"), $"Expected COUNT validation error, got: {ex.Message}"); + } + private static byte[] SeedMoviesForAdvancedFiltering(IDatabase db) { _ = db.KeyDelete("movies"); @@ -884,17 +931,85 @@ public void DeleteVectorSet() } [Test] - public void InteterruptedVectorSetDelete_AfterMark() + public void FlushDB() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var s = redis.GetServers().Single(); + var db = redis.GetDatabase(); + +#if DEBUG + var preAddCreateCalls = server.Provider.StoreWrapper.DefaultDatabase.VectorManager.Service.CreateIndexCalls; +#endif + + var res1 = db.Execute("VADD", ["foo", "REDUCE", "3", "VALUES", "75", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32"]); + ClassicAssert.AreEqual(1, (int)res1); + + s.FlushDatabase(0); + +#if DEBUG + var finalCreateCalls = server.Provider.StoreWrapper.DefaultDatabase.VectorManager.Service.CreateIndexCalls; + var finalDropCalls = server.Provider.StoreWrapper.DefaultDatabase.VectorManager.Service.DropIndexCalls; + + // Check we actually dropped the index despite not touching the key explicitly + ClassicAssert.AreEqual(preAddCreateCalls + 1, finalCreateCalls); + ClassicAssert.AreEqual(finalDropCalls, finalCreateCalls); +#endif + + var res2 = db.KeyExists("foo"); + ClassicAssert.IsFalse(res2); + } + + [Test] + public async Task ExpirationAsync() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(); + + var res1 = await db.ExecuteAsync("VADD", ["foo", "REDUCE", "3", "VALUES", "75", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32"]).ConfigureAwait(false); + ClassicAssert.AreEqual(1, (int)res1); + +#if DEBUG + var preExpireDropCalls = server.Provider.StoreWrapper.DefaultDatabase.VectorManager.Service.DropIndexCalls; +#endif + + var res2 = await db.KeyExpireAsync("foo", TimeSpan.FromSeconds(0.5)).ConfigureAwait(false); + ClassicAssert.IsTrue(res2); + + // Wait for expiration to pass + await Task.Delay(TimeSpan.FromSeconds(2)).ConfigureAwait(false); + + // Force an expiration scan, check that at least one record was evicted + var res3 = (int[])await db.ExecuteAsync("EXPDELSCAN"); + ClassicAssert.AreEqual(1, res3[0]); + + var res4 = await db.KeyExistsAsync("foo").ConfigureAwait(false); + ClassicAssert.IsFalse(res4); + +#if DEBUG + var finalExpireDropCalls = server.Provider.StoreWrapper.DefaultDatabase.VectorManager.Service.DropIndexCalls; + + // Check that background cleanup was triggered, not just the key being removed + ClassicAssert.AreEqual(preExpireDropCalls + 1, finalExpireDropCalls); +#endif + } + + [Test] + public void InterruptedVectorSetDelete_BeforeMark() => InterruptedVectorSetDelete(ExceptionInjectionType.VectorSet_Interrupt_Delete_0); + [Test] - public void InterruptedVectorSetDelete_AfterZeroingOut() + public void InterruptedVectorSetDelete_DuringCleanup() => InterruptedVectorSetDelete(ExceptionInjectionType.VectorSet_Interrupt_Delete_1); [Test] - public void InterruptedVectorSetDelete_AfterDelete() + public void InterruptedVectorSetDelete_AfterCleanup() => InterruptedVectorSetDelete(ExceptionInjectionType.VectorSet_Interrupt_Delete_2); + [Test] + public void InterruptedVectorSetDelete_AfterMark() + => InterruptedVectorSetDelete(ExceptionInjectionType.VectorSet_Interrupt_Delete_3); + private void InterruptedVectorSetDelete(ExceptionInjectionType faultLocation) { #if !DEBUG @@ -910,11 +1025,14 @@ private void InterruptedVectorSetDelete(ExceptionInjectionType faultLocation) var res1 = db.Execute("VADD", [key, "REDUCE", "3", "VALUES", "75", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", "4.0", "1.0", "2.0", "3.0", new byte[] { 0, 0, 0, 0 }, "CAS", "NOQUANT", "EF", "16", "M", "32"]); ClassicAssert.AreEqual(1, (int)res1); - // TODO: we could use EXISTS here... except not all non-Vector Set commands understand Vector Sets, so that's a bit flaky ExceptionInjectionHelper.EnableException(faultLocation); try { - _ = ClassicAssert.Throws(() => db.KeyDelete(key)); + _ = db.KeyDelete(key); + } + catch + { + // Exception is possible (but not guarnateed) and legal } finally { @@ -1066,17 +1184,21 @@ private void InterruptedVectorSetDelete(ExceptionInjectionType faultLocation) } [Test] - public Task InterruptedVectorSetDelete_AfterMark_RecoveryAsync() + public Task InterruptedVectorSetDelete_BeforeMark_RecoveryAsync() => InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionType.VectorSet_Interrupt_Delete_0); [Test] - public Task InterruptedVectorSetDelete_AfterZeroingOut_RecoveryAsync() + public Task InteterruptedVectorSetDelete_DuringCleanup_RecoveryAsync() => InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionType.VectorSet_Interrupt_Delete_1); [Test] - public Task InterruptedVectorSetDelete_AfterDelete_RecoveryAsync() + public Task InteterruptedVectorSetDelete_AfterCleanup_RecoveryAsync() => InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionType.VectorSet_Interrupt_Delete_2); + [Test] + public Task InteterruptedVectorSetDelete_AfterMark_RecoveryAsync() + => InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionType.VectorSet_Interrupt_Delete_3); + private async Task InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionType faultLocation) { #if !DEBUG @@ -1096,7 +1218,11 @@ private async Task InterruptedVectorSetDeleteRecoveryAsync(ExceptionInjectionTyp ExceptionInjectionHelper.EnableException(faultLocation); try { - _ = ClassicAssert.Throws(() => db.KeyDelete(key)); + _ = db.KeyDelete(key); + } + catch + { + // Exception is possible (but not guarnateed) and legal } finally { @@ -1232,8 +1358,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (int* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 64, 1, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 64, 1, keyData); var iters = 0; for (var i = 0; i < batch.Count; i++) @@ -1248,8 +1374,8 @@ public unsafe void VectorReadBatchVariants() // Validate key batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(64, keyCopy.GetNamespaceInPayload()); - ClassicAssert.IsTrue(keyCopy.AsReadOnlySpan().SequenceEqual(MemoryMarshal.Cast(data.AsSpan().Slice(1, 1)))); + ClassicAssert.AreEqual(64, keyCopy.NamespaceBytes[0]); + ClassicAssert.IsTrue(keyCopy.KeyBytes.SequenceEqual(MemoryMarshal.Cast(data.AsSpan().Slice(1, 1)))); // Validate output doesn't throw batch.GetOutput(i, out _); @@ -1270,8 +1396,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (int* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 32, 7, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 32, 7, keyData); var iters = 0; for (var i = 0; i < batch.Count; i++) @@ -1286,10 +1412,10 @@ public unsafe void VectorReadBatchVariants() // Validate key batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(32, keyCopy.GetNamespaceInPayload()); + ClassicAssert.AreEqual(32, keyCopy.NamespaceBytes[0]); var offset = i * 2 + 1; - var keyCopyData = keyCopy.AsReadOnlySpan(); + var keyCopyData = keyCopy.KeyBytes; var expectedData = MemoryMarshal.Cast(data.AsSpan().Slice(offset, 1)); ClassicAssert.IsTrue(keyCopyData.SequenceEqual(expectedData)); @@ -1312,8 +1438,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (int* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 16, 7, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length * sizeof(int)); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 16, 7, keyData); var rand = new Random(2025_10_06_00); @@ -1329,10 +1455,10 @@ public unsafe void VectorReadBatchVariants() // Validate key batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(16, keyCopy.GetNamespaceInPayload()); + ClassicAssert.AreEqual(16, keyCopy.NamespaceBytes[0]); var offset = i * 2 + 1; - var keyCopyData = keyCopy.AsReadOnlySpan(); + var keyCopyData = keyCopy.KeyBytes; var expectedData = MemoryMarshal.Cast(data.AsSpan().Slice(offset, 1)); ClassicAssert.IsTrue(keyCopyData.SequenceEqual(expectedData)); @@ -1358,8 +1484,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (byte* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 8, 1, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 8, 1, keyData); var iters = 0; for (var i = 0; i < batch.Count; i++) @@ -1387,8 +1513,8 @@ public unsafe void VectorReadBatchVariants() }; batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(8, keyCopy.GetNamespaceInPayload()); - var keyCopyData = keyCopy.AsReadOnlySpan(); + ClassicAssert.AreEqual(8, keyCopy.NamespaceBytes[0]); + var keyCopyData = keyCopy.KeyBytes; var expectedData = data.AsSpan().Slice(expectedStart, expectedLength); ClassicAssert.IsTrue(expectedData.SequenceEqual(keyCopyData)); @@ -1465,8 +1591,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (byte* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 4, 8, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 4, 8, keyData); var iters = 0; for (var i = 0; i < batch.Count; i++) @@ -1508,8 +1634,8 @@ public unsafe void VectorReadBatchVariants() }; batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(4, keyCopy.GetNamespaceInPayload()); - var keyCopyData = keyCopy.AsReadOnlySpan(); + ClassicAssert.AreEqual(4, keyCopy.NamespaceBytes[0]); + var keyCopyData = keyCopy.KeyBytes; var expectedData = data.AsSpan().Slice(expectedStart, expectedLength); ClassicAssert.IsTrue(expectedData.SequenceEqual(keyCopyData)); @@ -1586,8 +1712,8 @@ public unsafe void VectorReadBatchVariants() var dataCopy = data.ToArray(); fixed (byte* dataPtr = data) { - var keyData = SpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); - using var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 4, 8, keyData); + var keyData = PinnedSpanByte.FromPinnedPointer((byte*)dataPtr, data.Length); + var batch = new VectorManager.VectorReadBatch(input.Callback, input.CallbackContext, 4, 8, keyData); var rand = new Random(2025_10_06_01); @@ -1630,8 +1756,8 @@ public unsafe void VectorReadBatchVariants() }; batch.GetKey(i, out var keyCopy); - ClassicAssert.AreEqual(4, keyCopy.GetNamespaceInPayload()); - var keyCopyData = keyCopy.AsReadOnlySpan(); + ClassicAssert.AreEqual(4, keyCopy.NamespaceBytes[0]); + var keyCopyData = keyCopy.KeyBytes; var expectedData = data.AsSpan().Slice(expectedStart, expectedLength); ClassicAssert.IsTrue(expectedData.SequenceEqual(keyCopyData)); @@ -1644,18 +1770,16 @@ public unsafe void VectorReadBatchVariants() } [Test] - public unsafe void MarkWithNamespace() + public unsafe void MakeVectorElementKey() { var data = new int[] { 4, 1234 }; var dataCopy = data.ToArray(); fixed (int* intPtr = data) { var bytePtr = (byte*)intPtr; - var span = VectorManager.MarkDiskANNKeyWithNamespace(8, (nint)(bytePtr + 4), 4); - ClassicAssert.AreEqual(8, span.GetNamespaceInPayload()); - ClassicAssert.AreEqual(1234, *(int*)span.ToPointer()); - - VectorManager.UnmarkDiskANNKey(span); + var span = VectorManager.MakeVectorElementKey(8, (nint)(bytePtr + 4), 4); + ClassicAssert.AreEqual(8, span.NamespaceBytes[0]); + ClassicAssert.AreEqual(1234, MemoryMarshal.Cast(span.KeyBytes)[0]); } ClassicAssert.IsTrue(dataCopy.SequenceEqual(data)); } @@ -2162,6 +2286,37 @@ public void VGETATTR() ClassicAssert.AreEqual(0, res4.Length); } + [Test] + public void VGETATTR_BinaryAttributes() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(); + + var vectorSetKey = "binattr"; + + // Attribute containing CR, LF, and CRLF sequences + var binaryAttr = new byte[] { + (byte)'{', (byte)'"', (byte)'k', (byte)'"', (byte)':', (byte)'"', + 0x0D, 0x0A, // CR LF + (byte)'"', (byte)'}', + }; + var elem1 = new byte[] { 0, 0, 0, 1 }; + var addRes1 = db.Execute("VADD", [vectorSetKey, "VALUES", "3", "1.0", "2.0", "3.0", elem1, "NOQUANT", "SETATTR", binaryAttr]); + ClassicAssert.AreEqual(1, (int)addRes1); + + var getRes1 = (byte[])db.Execute("VGETATTR", [vectorSetKey, elem1]); + ClassicAssert.IsTrue(binaryAttr.SequenceEqual(getRes1), "Binary attribute with CRLF round-trip mismatch"); + + // Attribute containing null bytes and high bytes + var binaryAttr2 = new byte[] { 0x00, 0xFF, 0x0D, 0x0A, 0x01, 0xFE }; + var elem2 = new byte[] { 0, 0, 0, 2 }; + var addRes2 = db.Execute("VADD", [vectorSetKey, "VALUES", "3", "4.0", "5.0", "6.0", elem2, "NOQUANT", "SETATTR", binaryAttr2]); + ClassicAssert.AreEqual(1, (int)addRes2); + + var getRes2 = (byte[])db.Execute("VGETATTR", [vectorSetKey, elem2]); + ClassicAssert.IsTrue(binaryAttr2.SequenceEqual(getRes2), "Binary attribute with null/high bytes round-trip mismatch"); + } + [Test] public void VREM() { diff --git a/test/standalone/Garnet.test.vectorset/TestProjectSetup.cs b/test/standalone/Garnet.test.vectorset/TestProjectSetup.cs new file mode 100644 index 00000000000..08611de295e --- /dev/null +++ b/test/standalone/Garnet.test.vectorset/TestProjectSetup.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using NUnit.Framework; + +namespace Garnet.test +{ + [SetUpFixture] + public class TestProjectSetup + { + [OneTimeSetUp] + public void SetPort() => TestUtils.SetTestPort(TestPortAssignment.GarnetTestVectorSet); + } +} \ No newline at end of file diff --git a/test/standalone/Garnet.test.vectorset/VectorCleanupVsResetRaceTests.cs b/test/standalone/Garnet.test.vectorset/VectorCleanupVsResetRaceTests.cs new file mode 100644 index 00000000000..cb1dd15075e --- /dev/null +++ b/test/standalone/Garnet.test.vectorset/VectorCleanupVsResetRaceTests.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Buffers.Binary; +using System.Runtime.CompilerServices; +using System.Threading; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; + +namespace Garnet.test +{ + /// + /// Regression test for a race between VectorManager's background + /// cleanup-task scan iterator (over the main string keyspace) and a + /// concurrent storeWrapper.Reset() (e.g. triggered by a replica + /// re-attach). The race originally manifested as an AVE in + /// SpanByteScanIterator.GetNext dereferencing a freed page. + /// + /// Production paths (cluster re-attach) wrap Reset with + /// VectorManager.PauseCleanupAsync / ResumeCleanup to + /// serialize the cleanup task (iterator + post-iterate RMWs) against + /// allocator teardown — Reset is only safe for concurrent SCAN iteration, + /// not for the cleanup task's RMWs on metadata records that follow the + /// iteration. This test mirrors that production pattern: add a vector set + /// on a single Garnet server, drop it (queues a cleanup scan), then hammer + /// Pause + Reset + Resume while the cleanup task runs. + /// + [TestFixture] + public class VectorCleanupVsResetRaceTests : TestBase + { + private global::Garnet.GarnetServer server; + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, enableAOF: true, enableVectorSetPreview: true); + server.Start(); + } + + [TearDown] + public void TearDown() + { + try { server.Dispose(); } catch { } + TestUtils.OnTearDown(); + } + + // The storeWrapper field is private on GarnetServer; everything below it + // (DefaultDatabase, VectorManager, Reset(int)) is public so we access them + // directly through the StoreWrapper reference returned here. + [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "storeWrapper")] + private static extern ref StoreWrapper GetStoreWrapper(global::Garnet.GarnetServer server); + + /// + /// Reproducer: drop a vector set (queues full-keyspace cleanup) and concurrently + /// hammer Pause+Reset+Resume — the production pattern used by cluster re-attach + /// (ReplicaDisklessSync / ReplicaDiskbasedSync). Without the Pause, Reset's + /// post-Phase-2 Initialize() would race with the cleanup task's RMWs on metadata + /// records (ClearDeleteInProgress / UpdateContextMetadata) and AVE — Reset is + /// only safe for concurrent SCAN iteration, not for arbitrary RMW/Read/Upsert. + /// + /// Verifies: + /// * Pause+Reset+Resume against an in-flight cleanup-task iteration does not AVE. + /// * cleanupGate correctly serializes cleanup-iteration with Reset. + /// + [Test] + [Repeat(5)] + public void DropVectorSetWhileResettingStore() + { + const int Vectors = 4_000; + const string Key = nameof(DropVectorSetWhileResettingStore); + + ref var storeWrapper = ref GetStoreWrapper(server); + ClassicAssert.IsNotNull(storeWrapper, "Could not access storeWrapper via UnsafeAccessor"); + + var vectorManager = storeWrapper.DefaultDatabase.VectorManager; + ClassicAssert.IsNotNull(vectorManager, "VectorManager not initialised — enableVectorSetPreview must be true"); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + + // Populate the vector set so the cleanup-task scan has lots of records to iterate. + var elem = new byte[4]; + var data = new byte[75]; + var rand = new Random(2026_05_01); + for (var i = 0; i < Vectors; i++) + { + BinaryPrimitives.WriteInt32LittleEndian(elem, i); + rand.NextBytes(data); + _ = db.Execute("VADD", [Key, "XB8", data, elem, "XPREQ8"]); + } + + // Drop the vector set. This calls VectorManager.CleanupDroppedIndex which writes to + // the cleanup channel; the background task then runs Iterate over the full keyspace + // and a series of RMWs to clear the in-progress-deletes metadata. + _ = db.KeyDelete(Key); + + // Race window: hammer Pause+Reset+Resume while the cleanup task iterates. + // Pause acquires VectorManager's cleanupGate; the cleanup-iteration body holds + // that gate from the start of the iterate through the post-iterate RMWs, so + // Pause waits for any in-flight iteration to fully finish before Reset proceeds. + var deadline = DateTime.UtcNow.AddSeconds(5); + int resets = 0; + while (DateTime.UtcNow < deadline) + { + vectorManager.PauseCleanupAsync().GetAwaiter().GetResult(); + try + { + try + { + storeWrapper.Reset(); + resets++; + } + catch (Exception ex) + { + // Reset itself can throw if the store is in an unexpected state — that's OK + // for our purposes; we care about whether the cleanup iteration AVEs. + TestContext.Progress.WriteLine($"[reset] threw: {ex.GetType().Name}: {ex.Message}"); + } + } + finally + { + vectorManager.ResumeCleanup(); + } + Thread.Sleep(1); + } + + TestContext.Progress.WriteLine($"[DropVectorSetWhileResettingStore] resets={resets}"); + // If we reach here the cleanup task did not AVE the host while Reset was hammering. + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/AofFinalizeDoubleReplayTxn.cs b/test/standalone/Garnet.test/AofFinalizeDoubleReplayTxn.cs similarity index 97% rename from test/Garnet.test/AofFinalizeDoubleReplayTxn.cs rename to test/standalone/Garnet.test/AofFinalizeDoubleReplayTxn.cs index dc9a149fac7..777a780046d 100644 --- a/test/Garnet.test/AofFinalizeDoubleReplayTxn.cs +++ b/test/standalone/Garnet.test/AofFinalizeDoubleReplayTxn.cs @@ -14,7 +14,7 @@ public class AofFinalizeDoubleReplayTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); return true; } public override void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) diff --git a/test/Garnet.test/CountingEventSlimTests.cs b/test/standalone/Garnet.test/CountingEventSlimTests.cs similarity index 95% rename from test/Garnet.test/CountingEventSlimTests.cs rename to test/standalone/Garnet.test/CountingEventSlimTests.cs index 0e86600b181..21416677d45 100644 --- a/test/Garnet.test/CountingEventSlimTests.cs +++ b/test/standalone/Garnet.test/CountingEventSlimTests.cs @@ -1,20 +1,18 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Concurrent; using System.Threading.Channels; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class CountingEventSlimTests : AllureTestBase + public class CountingEventSlimTests : TestBase { [Test] public void Basic() diff --git a/test/Garnet.test/CredentialManager.cs b/test/standalone/Garnet.test/CredentialManager.cs similarity index 100% rename from test/Garnet.test/CredentialManager.cs rename to test/standalone/Garnet.test/CredentialManager.cs diff --git a/test/Garnet.test/CustomRespCommandsDocs.json b/test/standalone/Garnet.test/CustomRespCommandsDocs.json similarity index 100% rename from test/Garnet.test/CustomRespCommandsDocs.json rename to test/standalone/Garnet.test/CustomRespCommandsDocs.json diff --git a/test/Garnet.test/CustomRespCommandsInfo.json b/test/standalone/Garnet.test/CustomRespCommandsInfo.json similarity index 100% rename from test/Garnet.test/CustomRespCommandsInfo.json rename to test/standalone/Garnet.test/CustomRespCommandsInfo.json diff --git a/test/Garnet.test/DeleteTxn.cs b/test/standalone/Garnet.test/DeleteTxn.cs similarity index 92% rename from test/Garnet.test/DeleteTxn.cs rename to test/standalone/Garnet.test/DeleteTxn.cs index 3c31b965fef..a77cb1216f6 100644 --- a/test/Garnet.test/DeleteTxn.cs +++ b/test/standalone/Garnet.test/DeleteTxn.cs @@ -19,7 +19,7 @@ sealed class DeleteTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, StoreType.All); return true; } @@ -27,7 +27,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p { var offset = 0; var key = GetNextArg(ref procInput.parseState, ref offset); - api.DELETE(key, StoreType.Main); + api.DELETE(key); WriteSimpleString(ref output, "SUCCESS"); } } diff --git a/test/Garnet.test/DocsTests.cs b/test/standalone/Garnet.test/DocsTests.cs similarity index 89% rename from test/Garnet.test/DocsTests.cs rename to test/standalone/Garnet.test/DocsTests.cs index 5bfa1a59488..3aedce86eab 100644 --- a/test/Garnet.test/DocsTests.cs +++ b/test/standalone/Garnet.test/DocsTests.cs @@ -1,16 +1,14 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Collections.Generic; using System.IO; using System.Text.RegularExpressions; -using Allure.NUnit; using NUnit.Framework; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public partial class DocsTests : AllureTestBase + public partial class DocsTests : TestBase { [GeneratedRegex(@"^\s*\|\s*\|\s*\[(?[^\]]+)\]\(.+?\)\s*\|\s*(?[➖])")] private static partial Regex CommandLinkAndMinusRegex(); diff --git a/test/Garnet.test/ExpiredKeyDeletionTests.cs b/test/standalone/Garnet.test/ExpiredKeyDeletionTests.cs similarity index 93% rename from test/Garnet.test/ExpiredKeyDeletionTests.cs rename to test/standalone/Garnet.test/ExpiredKeyDeletionTests.cs index dded993996f..3ee2ea7cfb8 100644 --- a/test/Garnet.test/ExpiredKeyDeletionTests.cs +++ b/test/standalone/Garnet.test/ExpiredKeyDeletionTests.cs @@ -3,16 +3,14 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - class ExpiredKeyDeletionTests : AllureTestBase + class ExpiredKeyDeletionTests : TestBase { private const int ExpiredKeyDeletionScanFrequencySecs = 10; @@ -102,11 +100,9 @@ private async Task TestExpiredKeyDeletionScanAsync(Func ex // Merge reviv stats across sessions server.Provider.StoreWrapper.store.DumpRevivificationStats(); - server.Provider.StoreWrapper.objectStore.DumpRevivificationStats(); // Check that revivification happened for expired record ClassicAssert.IsTrue(server.Provider.StoreWrapper.store.RevivificationManager.stats.successfulAdds > 0, "Active expiration did not revivify for main store as expected"); - ClassicAssert.IsTrue(server.Provider.StoreWrapper.objectStore.RevivificationManager.stats.successfulAdds > 0, "Active expiration did not revivify for obj store as expected"); // Post expired key deletion scan, expired records don't exist for sure. This can be fooled by passive expiration too, so check reviv metrics too CheckExistenceConditionOnAllKeys(db, tombstonedRecords, false, "All to be expired should no longer exist post gc"); @@ -126,22 +122,17 @@ private int PopulateStore(List keys, IDatabase db, (int, int) allowedExp for (int i = 0; i < keys.Count; i++) { int expirationOrScore = rnd.Next(allowedExpirationRange.Item1, allowedExpirationRange.Item2); - bool isMainStore = rnd.Next(0, 2) == 0; + bool addString = rnd.Next(0, 2) == 0; bool hasExpiration = rnd.Next(0, 2) == 0; - if (isMainStore) - { + if (addString) db.StringSet(keys[i], Guid.NewGuid().ToString()); - } else - { db.SortedSetAdd(keys[i], Guid.NewGuid().ToString(), expirationOrScore); - } if (hasExpiration || forceExpirationaddition) { if (expirationOrScore < ExpiredKeyDeletionScanFrequencySecs) totalKeysThatWillExpire++; - ClassicAssert.IsTrue(db.KeyExpire(keys[i], TimeSpan.FromSeconds(expirationOrScore))); } } diff --git a/test/Garnet.test/Extensions/BulkIncrementBy.cs b/test/standalone/Garnet.test/Extensions/BulkIncrementBy.cs similarity index 88% rename from test/Garnet.test/Extensions/BulkIncrementBy.cs rename to test/standalone/Garnet.test/Extensions/BulkIncrementBy.cs index 9f358d5a905..361bfe2927f 100644 --- a/test/Garnet.test/Extensions/BulkIncrementBy.cs +++ b/test/standalone/Garnet.test/Extensions/BulkIncrementBy.cs @@ -9,8 +9,9 @@ namespace Garnet { sealed class BulkIncrementBy : CustomTransactionProcedure { - // BULKINCRBY 2 a 10 [b 15] [c 25] ... + // BULKINCRBY k1 incrby1 [k2 incrby2 [k3 incrby3 ...]] public static readonly RespCommandsInfo CommandInfo = new() { Arity = -4 }; + public static readonly string Name = "BULKINCRBY"; public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { @@ -22,8 +23,8 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce for (var i = 0; i < count; i++) { - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, isObject: false); - GetNextArg(ref procInput, ref offset); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, storeType: StoreType.Main); + _ = GetNextArg(ref procInput, ref offset); } return true; diff --git a/test/Garnet.test/Extensions/BulkRead.cs b/test/standalone/Garnet.test/Extensions/BulkRead.cs similarity index 88% rename from test/Garnet.test/Extensions/BulkRead.cs rename to test/standalone/Garnet.test/Extensions/BulkRead.cs index 946dd8d3fcc..f6072a79921 100644 --- a/test/Garnet.test/Extensions/BulkRead.cs +++ b/test/standalone/Garnet.test/Extensions/BulkRead.cs @@ -9,8 +9,9 @@ namespace Garnet { sealed class BulkRead : CustomTransactionProcedure { - // BULKREAD 3 a [b] [c] + // BULKREAD a [b] [c] public static readonly RespCommandsInfo CommandInfo = new() { Arity = -3 }; + public static readonly string Name = "BULKREAD"; public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { @@ -21,7 +22,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce return false; for (var i = 0; i < count; i++) - AddKey(GetNextArg(ref procInput, ref offset), LockType.Shared, isObject: false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Shared, storeType: StoreType.Main); return true; } @@ -36,7 +37,7 @@ public override unsafe void Main(TGarnetApi api, ref CustomProcedure return; } - var result = new ArgSlice[count]; + var result = new PinnedSpanByte[count]; for (var i = 0; i < count; i++) { diff --git a/test/Garnet.test/Extensions/ProcCustomCmd.cs b/test/standalone/Garnet.test/Extensions/ProcCustomCmd.cs similarity index 92% rename from test/Garnet.test/Extensions/ProcCustomCmd.cs rename to test/standalone/Garnet.test/Extensions/ProcCustomCmd.cs index 1fc59947269..7a807b0c270 100644 --- a/test/Garnet.test/Extensions/ProcCustomCmd.cs +++ b/test/standalone/Garnet.test/Extensions/ProcCustomCmd.cs @@ -3,6 +3,7 @@ using Garnet.common; using Garnet.server; +using Tsavorite.core; namespace Garnet { @@ -13,7 +14,7 @@ public override unsafe bool Execute(TGarnetApi garnetApi, ref Custom var offset = 0; var key = GetNextArg(ref procInput, ref offset); - var args = new ArgSlice[2]; + var args = new PinnedSpanByte[2]; args[0] = GetNextArg(ref procInput, ref offset); // value to set args[1] = GetNextArg(ref procInput, ref offset); // prefix to match diff --git a/test/Garnet.test/Extensions/RateLimiterTxn.cs b/test/standalone/Garnet.test/Extensions/RateLimiterTxn.cs similarity index 90% rename from test/Garnet.test/Extensions/RateLimiterTxn.cs rename to test/standalone/Garnet.test/Extensions/RateLimiterTxn.cs index e1fdc0816b9..757fc4a84d9 100644 --- a/test/Garnet.test/Extensions/RateLimiterTxn.cs +++ b/test/standalone/Garnet.test/Extensions/RateLimiterTxn.cs @@ -14,7 +14,7 @@ sealed class RateLimiterTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, true); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Object); return true; } @@ -53,8 +53,8 @@ public override unsafe void Main(TGarnetApi api, ref CustomProcedure var timeInMicroSecondBytes = Encoding.ASCII.GetBytes(timeInMicroSecond.ToString()); fixed (byte* timeInMicroSecondBytesPtr = timeInMicroSecondBytes) { - api.SortedSetAdd(key, new ArgSlice(unixTimeInMilliSecondPtr, unixTimeInMilliSecondBytes.Length), new ArgSlice(timeInMicroSecondBytesPtr, timeInMicroSecondBytes.Length), out var _); - api.EXPIRE(key, TimeSpan.FromMilliseconds(slidingWindowInMilliSeconds), out var _, StoreType.Object); + api.SortedSetAdd(key, PinnedSpanByte.FromPinnedPointer(unixTimeInMilliSecondPtr, unixTimeInMilliSecondBytes.Length), PinnedSpanByte.FromPinnedPointer(timeInMicroSecondBytesPtr, timeInMicroSecondBytes.Length), out var _); + api.EXPIRE(key, TimeSpan.FromMilliseconds(slidingWindowInMilliSeconds), out _); } } diff --git a/test/Garnet.test/Extensions/SortedSetCountTxn.cs b/test/standalone/Garnet.test/Extensions/SortedSetCountTxn.cs similarity index 97% rename from test/Garnet.test/Extensions/SortedSetCountTxn.cs rename to test/standalone/Garnet.test/Extensions/SortedSetCountTxn.cs index 023a9653640..199d922b84e 100644 --- a/test/Garnet.test/Extensions/SortedSetCountTxn.cs +++ b/test/standalone/Garnet.test/Extensions/SortedSetCountTxn.cs @@ -12,7 +12,7 @@ sealed class SortedSetCountTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput input) { int offset = 0; - AddKey(GetNextArg(ref input, ref offset), LockType.Shared, true); + AddKey(GetNextArg(ref input, ref offset), LockType.Shared, StoreType.Object); return true; } diff --git a/test/Garnet.test/Extensions/TxnCustomCmd.cs b/test/standalone/Garnet.test/Extensions/TxnCustomCmd.cs similarity index 89% rename from test/Garnet.test/Extensions/TxnCustomCmd.cs rename to test/standalone/Garnet.test/Extensions/TxnCustomCmd.cs index 989852c2ba5..ce0ad587348 100644 --- a/test/Garnet.test/Extensions/TxnCustomCmd.cs +++ b/test/standalone/Garnet.test/Extensions/TxnCustomCmd.cs @@ -18,10 +18,10 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var mainStoreKey = GetNextArg(ref procInput, ref offset); _ = GetNextArg(ref procInput, ref offset); // mainStoreValue - AddKey(mainStoreKey, LockType.Exclusive, false); + AddKey(mainStoreKey, LockType.Exclusive, StoreType.Main); var myDictKey = GetNextArg(ref procInput, ref offset); - AddKey(myDictKey, LockType.Exclusive, true); + AddKey(myDictKey, LockType.Exclusive, StoreType.Object); if (!ParseCustomObjectCommand("MYDICTSET", out customObjectCommand)) return false; @@ -42,7 +42,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var myDictField = GetNextArg(ref procInput, ref offset); var myDictValue = GetNextArg(ref procInput, ref offset); - var args = new ArgSlice[2]; + var args = new PinnedSpanByte[2]; args[0] = myDictField; args[1] = myDictValue; diff --git a/test/Garnet.test/FuzzTargetTests.cs b/test/standalone/Garnet.test/FuzzTargetTests.cs similarity index 92% rename from test/Garnet.test/FuzzTargetTests.cs rename to test/standalone/Garnet.test/FuzzTargetTests.cs index 4e6de372938..a82386d19f1 100644 --- a/test/Garnet.test/FuzzTargetTests.cs +++ b/test/standalone/Garnet.test/FuzzTargetTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Reflection; using System.Text; -using Allure.NUnit; using Garnet.fuzz; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -14,9 +13,8 @@ namespace Garnet.test /// /// Tests that assert the basics of Garnet.fuzz still work, so they aren't broken between fuzzing runs. /// - [AllureNUnit] [TestFixture] - public class FuzzTargetTests : AllureTestBase + public class FuzzTargetTests : TestBase { [TearDown] public void TearDown() diff --git a/test/standalone/Garnet.test/Garnet.test.csproj b/test/standalone/Garnet.test/Garnet.test.csproj new file mode 100644 index 00000000000..3d0dcfc4ee3 --- /dev/null +++ b/test/standalone/Garnet.test/Garnet.test.csproj @@ -0,0 +1,88 @@ + + + + true + ../../../Garnet.snk + false + + + + 1701;1702;1591 + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + + false + $(DefineConstants);TEST_PROJECT + + + diff --git a/test/Garnet.test/GarnetClientTests.cs b/test/standalone/Garnet.test/GarnetClientTests.cs similarity index 99% rename from test/Garnet.test/GarnetClientTests.cs rename to test/standalone/Garnet.test/GarnetClientTests.cs index 24f0e1d0a00..8f63ddb1ea3 100644 --- a/test/Garnet.test/GarnetClientTests.cs +++ b/test/standalone/Garnet.test/GarnetClientTests.cs @@ -7,16 +7,14 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class GarnetClientTests : AllureTestBase + public class GarnetClientTests : TestBase { readonly string[,] worldcities = new string[,] { diff --git a/test/Garnet.test/GarnetServerConfigTests.cs b/test/standalone/Garnet.test/GarnetServerConfigTests.cs similarity index 77% rename from test/Garnet.test/GarnetServerConfigTests.cs rename to test/standalone/Garnet.test/GarnetServerConfigTests.cs index 524991d6336..071e4f8e11c 100644 --- a/test/Garnet.test/GarnetServerConfigTests.cs +++ b/test/standalone/Garnet.test/GarnetServerConfigTests.cs @@ -10,7 +10,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; -using Allure.NUnit; using CommandLine; using Garnet.common; using Garnet.server; @@ -22,9 +21,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture, NonParallelizable] - public class GarnetServerConfigTests : AllureTestBase + public class GarnetServerConfigTests : TestBase { [Test] public void DefaultConfigurationOptionsCoverage() @@ -116,26 +114,25 @@ public void ImportExportConfigLocal() parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(null, out options, out invalidOptions, out var optionsJson, out exitGracefully, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.AreEqual("32m", options.PageSize); - ClassicAssert.AreEqual("16g", options.MemorySize); + ClassicAssert.AreEqual("16m", options.PageSize); + ClassicAssert.AreEqual("16g", options.LogMemorySize); var nonDefaultOptions = JsonSerializer.Deserialize>(optionsJson); ClassicAssert.IsEmpty(nonDefaultOptions); // No import path, include command line args, export to file // Check values from command line override values from defaults.conf - static string GetFullExtensionBinPath(string testProjectName) => Path.GetFullPath(testProjectName, TestUtils.RootTestsProjectPath); - var binPaths = new[] { GetFullExtensionBinPath("Garnet.test"), GetFullExtensionBinPath("Garnet.test.cluster") }; + static string GetFullExtensionBinPath(string relativePath) => Path.GetFullPath(relativePath, TestUtils.RootTestsProjectPath); + var binPaths = new[] { GetFullExtensionBinPath(Path.Combine("standalone", "Garnet.test")), GetFullExtensionBinPath(Path.Combine("cluster", "Garnet.test.cluster")) }; var modules = new[] { Assembly.GetExecutingAssembly().Location }; - var args = new[] { "--config-export-path", configPath, "-p", "4m", "-m", "128m", "-s", "2g", "--index", "128m", "--recover", "--port", "53", "--reviv-obj-bin-record-count", "2", "--reviv-fraction", "0.5", "--reviv-bin-record-counts", "1,2,3", "--extension-bin-paths", string.Join(',', binPaths), "--loadmodulecs", string.Join(',', modules) }; + var args = new[] { "--config-export-path", configPath, "-p", "8m", "-m", "128m", "-s", "2g", "--index", "128m", "--recover", "--port", "53", "--reviv-fraction", "0.5", "--reviv-bin-record-counts", "1,2,3", "--extension-bin-paths", string.Join(',', binPaths), "--loadmodulecs", string.Join(',', modules) }; parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out optionsJson, out exitGracefully, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.AreEqual("4m", options.PageSize); - ClassicAssert.AreEqual("128m", options.MemorySize); + ClassicAssert.AreEqual("8m", options.PageSize); + ClassicAssert.AreEqual("128m", options.LogMemorySize); ClassicAssert.AreEqual("2g", options.SegmentSize); ClassicAssert.AreEqual(53, options.Port); - ClassicAssert.AreEqual(2, options.RevivObjBinRecordCount); ClassicAssert.AreEqual(0.5, options.RevivifiableFraction); CollectionAssert.AreEqual(new[] { 1, 2, 3 }, options.RevivBinRecordCounts); ClassicAssert.IsTrue(options.Recover); @@ -145,9 +142,9 @@ public void ImportExportConfigLocal() // Validate non-default configuration options nonDefaultOptions = JsonSerializer.Deserialize>(optionsJson); - ClassicAssert.AreEqual(10, nonDefaultOptions.Count); + ClassicAssert.AreEqual(9, nonDefaultOptions.Count); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.PageSize))); - ClassicAssert.AreEqual("4m", ((JsonElement)nonDefaultOptions[nameof(Options.PageSize)]).GetString()); + ClassicAssert.AreEqual("8m", ((JsonElement)nonDefaultOptions[nameof(Options.PageSize)]).GetString()); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.Port))); ClassicAssert.AreEqual(53, ((JsonElement)nonDefaultOptions[nameof(Options.Port)]).GetInt32()); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.RevivifiableFraction))); @@ -169,27 +166,27 @@ public void ImportExportConfigLocal() parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out optionsJson, out exitGracefully, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "4m"); - ClassicAssert.IsTrue(options.MemorySize == "128m"); + ClassicAssert.IsTrue(options.PageSize == "8m"); + ClassicAssert.IsTrue(options.LogMemorySize == "128m"); CollectionAssert.AreEqual(new[] { 1, 2, 3 }, options.RevivBinRecordCounts); CollectionAssert.AreEqual(binPaths, options.ExtensionBinPaths); CollectionAssert.AreEqual(modules, options.LoadModuleCS); // Validate non-default configuration options nonDefaultOptions = JsonSerializer.Deserialize>(optionsJson); - ClassicAssert.AreEqual(10, nonDefaultOptions.Count); + ClassicAssert.AreEqual(9, nonDefaultOptions.Count); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.PageSize))); - ClassicAssert.AreEqual("4m", ((JsonElement)nonDefaultOptions[nameof(Options.PageSize)]).GetString()); + ClassicAssert.AreEqual("8m", ((JsonElement)nonDefaultOptions[nameof(Options.PageSize)]).GetString()); // Import from previous export command, include command line args, export to file // Check values from import path override values from default.conf, and values from command line override values from default.conf and import path - binPaths = [GetFullExtensionBinPath("Garnet.test")]; + binPaths = [GetFullExtensionBinPath(Path.Combine("standalone", "Garnet.test"))]; args = ["--config-import-path", configPath, "-p", "12m", "-s", "1g", "--recover", "false", "--index", "256m", "--port", "0", "--no-obj", "--aof", "--reviv-bin-record-counts", "4,5", "--extension-bin-paths", string.Join(',', binPaths)]; parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out optionsJson, out exitGracefully, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); ClassicAssert.AreEqual("12m", options.PageSize); - ClassicAssert.AreEqual("128m", options.MemorySize); + ClassicAssert.AreEqual("128m", options.LogMemorySize); ClassicAssert.AreEqual("1g", options.SegmentSize); ClassicAssert.AreEqual(0, options.Port); ClassicAssert.IsFalse(options.Recover); @@ -200,13 +197,13 @@ public void ImportExportConfigLocal() // Validate non-default configuration options nonDefaultOptions = JsonSerializer.Deserialize>(optionsJson); - ClassicAssert.AreEqual(11, nonDefaultOptions.Count); + ClassicAssert.AreEqual(10, nonDefaultOptions.Count); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.PageSize))); ClassicAssert.AreEqual("12m", ((JsonElement)nonDefaultOptions[nameof(Options.PageSize)]).GetString()); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.Port))); ClassicAssert.AreEqual(0, ((JsonElement)nonDefaultOptions[nameof(Options.Port)]).GetInt32()); - ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.IndexSize))); - ClassicAssert.AreEqual("256m", ((JsonElement)nonDefaultOptions[nameof(Options.IndexSize)]).GetString()); + ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.IndexMemorySize))); + ClassicAssert.AreEqual("256m", ((JsonElement)nonDefaultOptions[nameof(Options.IndexMemorySize)]).GetString()); ClassicAssert.IsTrue(nonDefaultOptions.ContainsKey(nameof(Options.RevivBinRecordCounts))); ClassicAssert.AreEqual(new[] { 4, 5 }, ((JsonElement)nonDefaultOptions[nameof(Options.RevivBinRecordCounts)]).EnumerateArray() @@ -224,7 +221,7 @@ public void ImportExportConfigLocal() ClassicAssert.IsNull(options); ClassicAssert.AreEqual(7, invalidOptions.Count); ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.Address))); - ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.MemorySize))); + ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.LogMemorySize))); ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.Port))); ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.MutablePercent))); ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.AclFile))); @@ -254,7 +251,7 @@ public void ImportExportRedisConfigLocal() ClassicAssert.AreEqual(ConnectionProtectionOption.Local, options.EnableDebugCommand); ClassicAssert.AreEqual(ConnectionProtectionOption.Yes, options.EnableModuleCommand); ClassicAssert.AreEqual(6379, options.Port); - ClassicAssert.AreEqual("20gb", options.MemorySize); + ClassicAssert.AreEqual("20gb", options.LogMemorySize); ClassicAssert.AreEqual("./garnet-log", options.FileLogger); ClassicAssert.AreEqual("./", options.CheckpointDir); ClassicAssert.IsTrue(options.EnableCluster); @@ -278,7 +275,7 @@ public void ImportExportRedisConfigLocal() ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); ClassicAssert.AreEqual("12m", options.PageSize); - ClassicAssert.AreEqual("20gb", options.MemorySize); + ClassicAssert.AreEqual("20gb", options.LogMemorySize); ClassicAssert.AreEqual("1g", options.SegmentSize); ClassicAssert.AreEqual(6, options.ThreadPoolMinThreads); ClassicAssert.AreEqual(10, options.ReplicaSyncDelayMs); @@ -313,18 +310,18 @@ public void ImportExportConfigAzure() var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(null, out var options, out var invalidOptions, out _, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "32m"); - ClassicAssert.IsTrue(options.MemorySize == "16g"); + ClassicAssert.IsTrue(options.PageSize == "16m"); + ClassicAssert.IsTrue(options.LogMemorySize == "16g"); ClassicAssert.IsNull(options.AzureStorageServiceUri); ClassicAssert.IsNull(options.AzureStorageManagedIdentity); ClassicAssert.AreNotEqual(DeviceType.AzureStorage, options.GetDeviceType()); - var args = new[] { "--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-export", "true", "--config-export-path", configPath, "-p", "4m", "-m", "128m", "--storage-service-uri", "https://demo.blob.core.windows.net", "--storage-managed-identity", "demo" }; + var args = new[] { "--storage-string", AzureEmulatedStorageString, "--use-azure-storage-for-config-export", "true", "--config-export-path", configPath, "-p", "8m", "-m", "128m", "--storage-service-uri", "https://demo.blob.core.windows.net", "--storage-managed-identity", "demo" }; parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out _, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "4m"); - ClassicAssert.IsTrue(options.MemorySize == "128m"); + ClassicAssert.IsTrue(options.PageSize == "8m"); + ClassicAssert.IsTrue(options.LogMemorySize == "128m"); ClassicAssert.IsTrue(options.AzureStorageServiceUri == "https://demo.blob.core.windows.net"); ClassicAssert.IsTrue(options.AzureStorageManagedIdentity == "demo"); @@ -332,8 +329,8 @@ public void ImportExportConfigAzure() parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out _, out _, silentMode: true); ClassicAssert.IsTrue(parseSuccessful); ClassicAssert.AreEqual(invalidOptions.Count, 0); - ClassicAssert.IsTrue(options.PageSize == "4m"); - ClassicAssert.IsTrue(options.MemorySize == "128m"); + ClassicAssert.IsTrue(options.PageSize == "8m"); + ClassicAssert.IsTrue(options.LogMemorySize == "128m"); ClassicAssert.IsTrue(options.AzureStorageServiceUri == "https://demo.blob.core.windows.net"); ClassicAssert.IsTrue(options.AzureStorageManagedIdentity == "demo"); @@ -940,123 +937,6 @@ public void ClusterReplicaResumeWithData() } } - [Test] - public void EnableVectorSetPreview() - { - // Command line args - { - // Default accepted - { - var args = Array.Empty(); - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsFalse(options.EnableVectorSetPreview); - } - - // Switch is accepted - { - var args = new[] { "--enable-vector-set-preview" }; - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsTrue(options.EnableVectorSetPreview); - } - } - - // JSON args - { - // Default accepted - { - const string JSON = @"{ }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsFalse(options.EnableVectorSetPreview); - } - - // False is accepted - { - const string JSON = @"{ ""EnableVectorSetPreview"": false }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsFalse(options.EnableVectorSetPreview); - } - - // True is accepted - { - const string JSON = @"{ ""EnableVectorSetPreview"": true }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsTrue(options.EnableVectorSetPreview); - } - - // Invalid rejected - { - const string JSON = @"{ ""EnableVectorSetPreview"": ""foo"" }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); - ClassicAssert.IsFalse(parseSuccessful); - } - } - } - - [Test] - public void MinimumPageSizeWithVectorSetPreview() - { - // Command line args - { - // Allow exactly minimum - { - var args = new[] { "--enable-vector-set-preview", "--page", "16k" }; - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsTrue(options.EnableVectorSetPreview); - ClassicAssert.AreEqual("16k", options.PageSize); - } - - // Allow lower than minimum if preview not enabled - { - var args = new[] { "--page", "1k" }; - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsFalse(options.EnableVectorSetPreview); - ClassicAssert.AreEqual("1k", options.PageSize); - } - - // Reject too small - { - var args = new[] { "--enable-vector-set-preview", "--page", "4k" }; - var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out _, out _, out _); - ClassicAssert.IsFalse(parseSuccessful); - } - } - - // JSON args - { - // Allow exactly minimum - { - const string JSON = @"{ ""EnableVectorSetPreview"": true, ""PageSize"": ""16k"" }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsTrue(options.EnableVectorSetPreview); - ClassicAssert.AreEqual("16k", options.PageSize); - } - - // Allow lower than minimum if preview not enabled - { - const string JSON = @"{ ""EnableVectorSetPreview"": false, ""PageSize"": ""1k"" }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out _, out _); - ClassicAssert.IsTrue(parseSuccessful); - ClassicAssert.IsFalse(options.EnableVectorSetPreview); - ClassicAssert.AreEqual("1k", options.PageSize); - } - - // Reject too small - { - const string JSON = @"{ ""EnableVectorSetPreview"": true, ""PageSize"": ""4k"" }"; - var parseSuccessful = TryParseGarnetConfOptions(JSON, out _, out _, out _); - ClassicAssert.IsFalse(parseSuccessful); - } - } - } - /// /// Import a garnet.conf file with the given contents /// @@ -1313,6 +1193,123 @@ public void RevivificationFlagOrderingIndependence() } } + [Test] + public void EnableVectorSetPreview() + { + // Command line args + { + // Default accepted + { + var args = Array.Empty(); + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsFalse(options.EnableVectorSetPreview); + } + + // Switch is accepted + { + var args = new[] { "--enable-vector-set-preview" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsTrue(options.EnableVectorSetPreview); + } + } + + // JSON args + { + // Default accepted + { + const string JSON = @"{ }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsFalse(options.EnableVectorSetPreview); + } + + // False is accepted + { + const string JSON = @"{ ""EnableVectorSetPreview"": false }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsFalse(options.EnableVectorSetPreview); + } + + // True is accepted + { + const string JSON = @"{ ""EnableVectorSetPreview"": true }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsTrue(options.EnableVectorSetPreview); + } + + // Invalid rejected + { + const string JSON = @"{ ""EnableVectorSetPreview"": ""foo"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out var exitGracefully); + ClassicAssert.IsFalse(parseSuccessful); + } + } + } + + [Test] + public void MinimumPageSizeWithVectorSetPreview() + { + // Command line args + { + // Allow exactly minimum + { + var args = new[] { "--enable-vector-set-preview", "--page", "16k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsTrue(options.EnableVectorSetPreview); + ClassicAssert.AreEqual("16k", options.PageSize); + } + + // Allow lower than minimum if preview not enabled + { + var args = new[] { "--page", "1k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out _, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsFalse(options.EnableVectorSetPreview); + ClassicAssert.AreEqual("1k", options.PageSize); + } + + // Reject too small + { + var args = new[] { "--enable-vector-set-preview", "--page", "4k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out _, out _, out _); + ClassicAssert.IsFalse(parseSuccessful); + } + } + + // JSON args + { + // Allow exactly minimum + { + const string JSON = @"{ ""EnableVectorSetPreview"": true, ""PageSize"": ""16k"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsTrue(options.EnableVectorSetPreview); + ClassicAssert.AreEqual("16k", options.PageSize); + } + + // Allow lower than minimum if preview not enabled + { + const string JSON = @"{ ""EnableVectorSetPreview"": false, ""PageSize"": ""1k"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out _, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.IsFalse(options.EnableVectorSetPreview); + ClassicAssert.AreEqual("1k", options.PageSize); + } + + // Reject too small + { + const string JSON = @"{ ""EnableVectorSetPreview"": true, ""PageSize"": ""4k"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out _, out _, out _); + ClassicAssert.IsFalse(parseSuccessful); + } + } + } + [Test] public void AofSizeLimitWithoutAofEnabled() { @@ -1331,5 +1328,330 @@ public void AofSizeLimitWithoutAofEnabled() ClassicAssert.AreEqual(0, invalidOptions.Count); Assert.DoesNotThrow(() => options.GetServerOptions()); } + + [Test] + public void AofSegmentSizeFlowsToTsavoriteLogSettings() + { + // Default AofSegmentSize from defaults.conf should be 1g and applied to TsavoriteLogSettings. + var args = new[] { "--aof" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual("1g", options.AofSegmentSize); + + var serverOptions = options.GetServerOptions(); + serverOptions.GetAofSettings(0, out var logSettings); + try + { + ClassicAssert.AreEqual(1, logSettings.Length); + ClassicAssert.AreEqual(1L << 30, logSettings[0].SegmentSize); + } + finally + { + foreach (var s in logSettings) + { + s.LogDevice?.Dispose(); + s.LogCommitManager?.Dispose(); + } + } + + // Configured AofSegmentSize should override default and propagate to TsavoriteLogSettings.SegmentSize. + args = ["--aof", "--aof-segment-size", "64m"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual("64m", options.AofSegmentSize); + + serverOptions = options.GetServerOptions(); + serverOptions.GetAofSettings(0, out logSettings); + try + { + ClassicAssert.AreEqual(1, logSettings.Length); + ClassicAssert.AreEqual(1L << 26, logSettings[0].SegmentSize); + } + finally + { + foreach (var s in logSettings) + { + s.LogDevice?.Dispose(); + s.LogCommitManager?.Dispose(); + } + } + + // AofPageSize > AofSegmentSize should throw. + args = ["--aof", "--aof-page-size", "8m", "--aof-segment-size", "4m"]; + parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out options, out invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + serverOptions = options.GetServerOptions(); + var ex = Assert.Throws(() => serverOptions.GetAofSettings(0, out _)); + ClassicAssert.IsTrue(ex.Message.Contains("AOF Page size cannot be more than the AOF segment size.")); + } + + [Test] + public void ValueOverflowThresholdParsing() + { + // Default value from defaults.conf is "16k" + { + var args = Array.Empty(); + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual("16k", options.ValueOverflowThreshold); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual("16k", serverOptions.ValueOverflowThreshold); + ClassicAssert.AreEqual(16384, serverOptions.ValueOverflowThresholdBytes()); + } + + // Various valid memory size strings (CLI). Use a 1g page size so the upper-bound case (256m) passes the cross-property fit check. + foreach (var (input, expectedBytes) in new[] { ("64", 64), ("1k", 1024), ("4k", 4096), ("1m", 1048576), ("256m", 1 << 28) }) + { + var args = new[] { "--page", "1g", "--value-overflow-threshold", input }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful, $"CLI parsing failed for '{input}'"); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(input, options.ValueOverflowThreshold); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(expectedBytes, serverOptions.ValueOverflowThresholdBytes(), $"Expected {expectedBytes} bytes for '{input}'"); + } + + // JSON parsing of a valid memory size string + { + const string JSON = @"{ ""ValueOverflowThreshold"": ""1m"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual("1m", options.ValueOverflowThreshold); + ClassicAssert.AreEqual(1048576, options.GetServerOptions().ValueOverflowThresholdBytes()); + } + } + + [Test] + public void ValueOverflowThresholdValidation() + { + // Reject invalid format (CLI) + { + var args = new[] { "--value-overflow-threshold", "abc" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsFalse(parseSuccessful); + ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.ValueOverflowThreshold))); + } + + // Regression: the previous regex used a `[K|k|M|m|G|g]` character class which treated the pipe as a literal, + // so inputs like '4|' were silently accepted. The tightened regex rejects them. + { + var args = new[] { "--value-overflow-threshold", "4|" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out _, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsFalse(parseSuccessful); + ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.ValueOverflowThreshold))); + } + + // Reject below minimum (32 bytes < 64 bytes minimum) — enforced at server-options time. + { + var args = new[] { "--value-overflow-threshold", "32" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + var ex = Assert.Throws(() => serverOptions.ValueOverflowThresholdBytes()); + ClassicAssert.IsTrue(ex.Message.Contains(nameof(serverOptions.ValueOverflowThreshold))); + } + + // Reject above maximum (512m > 256m maximum) — enforced at server-options time. + { + var args = new[] { "--value-overflow-threshold", "512m" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + var ex = Assert.Throws(() => serverOptions.ValueOverflowThresholdBytes()); + ClassicAssert.IsTrue(ex.Message.Contains(nameof(serverOptions.ValueOverflowThreshold))); + } + + // Accept exactly the minimum (64 bytes) + { + var args = new[] { "--value-overflow-threshold", "64" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(64, options.GetServerOptions().ValueOverflowThresholdBytes()); + } + + // Accept exactly the maximum (256m = 1<<28). Use a 1g PageSize so the cross-property fit check passes. + { + var args = new[] { "--page", "1g", "--value-overflow-threshold", "256m" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + ClassicAssert.AreEqual(1 << 28, options.GetServerOptions().ValueOverflowThresholdBytes()); + } + + // JSON: reject invalid format + { + const string JSON = @"{ ""ValueOverflowThreshold"": ""xyz"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out _, out var invalidOptions, out _); + ClassicAssert.IsFalse(parseSuccessful); + ClassicAssert.IsTrue(invalidOptions.Contains(nameof(Options.ValueOverflowThreshold))); + } + + // JSON: reject below minimum (enforced at server-options time) + { + const string JSON = @"{ ""ValueOverflowThreshold"": ""32"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + Assert.Throws(() => serverOptions.ValueOverflowThresholdBytes()); + } + + // JSON: reject above maximum (enforced at server-options time) + { + const string JSON = @"{ ""ValueOverflowThreshold"": ""512m"" }"; + var parseSuccessful = TryParseGarnetConfOptions(JSON, out var options, out var invalidOptions, out _); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + Assert.Throws(() => serverOptions.ValueOverflowThresholdBytes()); + } + } + + [Test] + public void ValueOverflowThresholdMustFitOnPage() + { + // ValueOverflowThreshold equal to PageSize is clamped down to PageSize/2 (next power of 2 down). + { + var args = new[] { "--page", "4k", "--value-overflow-threshold", "4k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(2048, serverOptions.ValueOverflowThresholdBytes()); + } + + // ValueOverflowThreshold greater than PageSize is clamped down to PageSize/2. + { + var args = new[] { "--page", "1k", "--value-overflow-threshold", "4k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(512, serverOptions.ValueOverflowThresholdBytes()); + } + + // ValueOverflowThreshold strictly less than PageSize (next power-of-2 down) is returned as-is. + { + var args = new[] { "--page", "8k", "--value-overflow-threshold", "4k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(4096, serverOptions.ValueOverflowThresholdBytes()); + } + + // Effective comparison uses post-rounding (previous power of 2): "5k" rounds to 4k, "8k" stays 8k -> ok. + { + var args = new[] { "--page", "8k", "--value-overflow-threshold", "5k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + // Raw bytes 5120 are returned by the helper; Tsavorite rounds to 4096 internally. + ClassicAssert.AreEqual(5120, serverOptions.ValueOverflowThresholdBytes()); + } + } + + [Test] + public void MinimumPageSize() + { + // PageSize below 512 bytes must be rejected at server-options consumption time. + // 256B is a valid memory-size string at parse time but is rejected by PageSizeBits(). + { + var args = new[] { "--page", "256" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + var ex = Assert.Throws(() => serverOptions.PageSizeBits()); + ClassicAssert.IsTrue(ex.Message.Contains(nameof(serverOptions.PageSize))); + ClassicAssert.IsTrue(ex.Message.Contains("512")); + } + + // 384B rounds down to 256B (previous power of 2) and is rejected. + { + var args = new[] { "--page", "384" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + Assert.Throws(() => serverOptions.PageSizeBits()); + } + + // Exactly 512B is accepted. + { + var args = new[] { "--page", "512" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(9, serverOptions.PageSizeBits()); + } + + // 1k is accepted. + { + var args = new[] { "--page", "1k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(10, serverOptions.PageSizeBits()); + } + } + + [Test] + public void MinimumReadCachePageSize() + { + // ReadCachePageSize below 512 bytes must be rejected. + // 256B is a valid memory-size string at parse time but is rejected by ReadCachePageSizeBits(). + { + var args = new[] { "--readcache-page", "256" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + var ex = Assert.Throws(() => serverOptions.ReadCachePageSizeBits()); + ClassicAssert.IsTrue(ex.Message.Contains(nameof(serverOptions.ReadCachePageSize))); + ClassicAssert.IsTrue(ex.Message.Contains("512")); + } + + // 384B rounds down to 256B (previous power of 2) and is rejected. + { + var args = new[] { "--readcache-page", "384" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + Assert.Throws(() => serverOptions.ReadCachePageSizeBits()); + } + + // Exactly 512B is accepted. + { + var args = new[] { "--readcache-page", "512" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(9, serverOptions.ReadCachePageSizeBits()); + } + + // 1k is accepted. + { + var args = new[] { "--readcache-page", "1k" }; + var parseSuccessful = ServerSettingsManager.TryParseCommandLineArguments(args, out var options, out var invalidOptions, out _, out _, silentMode: true); + ClassicAssert.IsTrue(parseSuccessful); + ClassicAssert.AreEqual(0, invalidOptions.Count); + var serverOptions = options.GetServerOptions(); + ClassicAssert.AreEqual(10, serverOptions.ReadCachePageSizeBits()); + } + } } } \ No newline at end of file diff --git a/test/Garnet.test/NUnitLoggerProvider.cs b/test/standalone/Garnet.test/NUnitLoggerProvider.cs similarity index 64% rename from test/Garnet.test/NUnitLoggerProvider.cs rename to test/standalone/Garnet.test/NUnitLoggerProvider.cs index 8d223958330..401bad77b8e 100644 --- a/test/Garnet.test/NUnitLoggerProvider.cs +++ b/test/standalone/Garnet.test/NUnitLoggerProvider.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Garnet.common; using Microsoft.Extensions.Logging; @@ -18,6 +19,11 @@ public class NUnitLoggerProvider : ILoggerProvider private readonly bool matchLevel; private readonly LogLevel logLevel; + /// + /// Array of enabled test logging flag types + /// + public bool[] GarnetTestLoggingEvents = [.. Enum.GetValues().Select(_ => false)]; + static readonly string[] lvl = [ "trce", @@ -38,13 +44,14 @@ public NUnitLoggerProvider(TextWriter textWriter, string scope = "", HashSet new NUnitLogger(categoryName, textWriter, scope, skipCmd, recvOnly: recvOnly, matchLevel: matchLevel, logLevel: logLevel); + public ILogger CreateLogger(string categoryName) => new NUnitLogger(this, categoryName, textWriter, scope, skipCmd, recvOnly: recvOnly, matchLevel: matchLevel, logLevel: logLevel); public void Dispose() { } private class NUnitLogger : ILogger { + private readonly NUnitLoggerProvider provider; private readonly string categoryName; private readonly TextWriter textWriter; private readonly string scope; @@ -53,8 +60,9 @@ private class NUnitLogger : ILogger private readonly bool matchLevel; private readonly LogLevel logLevel; - public NUnitLogger(string categoryName, TextWriter textWriter, string scope, HashSet skipCmd = null, bool recvOnly = false, bool matchLevel = false, LogLevel logLevel = LogLevel.None) + public NUnitLogger(NUnitLoggerProvider provider, string categoryName, TextWriter textWriter, string scope, HashSet skipCmd = null, bool recvOnly = false, bool matchLevel = false, LogLevel logLevel = LogLevel.None) { + this.provider = provider; this.categoryName = categoryName; this.textWriter = textWriter; this.scope = scope; @@ -77,11 +85,26 @@ public void Log( Exception exception, Func formatter) { - if ((matchLevel && logLevel == this.logLevel) || !matchLevel) + if (state is GarnetTestLoggingEvent _state) + { + if (provider.GarnetTestLoggingEvents[(int)_state.Type]) + { + var msg = string.Format("[{0:d1}.{1}.({2})] |{3}| <{4}> {5} ^{6}^", + eventId.Id, + LogFormatter.FormatTime(DateTime.UtcNow), + GetLevelStr(logLevel), + scope, + categoryName, + exception, + formatter(state, exception)); + textWriter.Write(msg); + } + } + else if ((matchLevel && logLevel == this.logLevel) || !matchLevel) { - var msg = string.Format("[{0:D3}.{1}.({2})] |{3}| <{4}> {5} ^{6}^", + var msg = string.Format("[{0:d1}.{1}.({2})] |{3}| <{4}> {5} ^{6}^", eventId.Id, - LogFormatter.FormatDate(DateTime.UtcNow), + LogFormatter.FormatTime(DateTime.UtcNow), GetLevelStr(logLevel), scope, categoryName, diff --git a/test/Garnet.test/NetworkTests.cs b/test/standalone/Garnet.test/NetworkTests.cs similarity index 98% rename from test/Garnet.test/NetworkTests.cs rename to test/standalone/Garnet.test/NetworkTests.cs index 2a4cd4b6920..9dcc854b723 100644 --- a/test/Garnet.test/NetworkTests.cs +++ b/test/standalone/Garnet.test/NetworkTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. #if DEBUG @@ -7,7 +7,6 @@ using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -15,9 +14,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class NetworkTests : AllureTestBase + public class NetworkTests : TestBase { GarnetServer server; @@ -172,7 +170,7 @@ public void DisposeCallsDisposeImplWithoutSaeaBackup() // with the shared server from SetUp, and so TearDown doesn't hang. var testDir = TestUtils.MethodTestDir + "_injection"; TestUtils.DeleteDirectory(testDir, wait: true); - var endpoint = new IPEndPoint(IPAddress.Loopback, TestUtils.TestPort + 1000); + var endpoint = new IPEndPoint(IPAddress.Loopback, (int)TestPortAssignment.GarnetTestAlternate); var testServer = TestUtils.CreateGarnetServer(testDir, enableTLS: true, endpoints: [endpoint]); testServer.Start(); diff --git a/test/Garnet.test/NumUtils.cs b/test/standalone/Garnet.test/NumUtils.cs similarity index 100% rename from test/Garnet.test/NumUtils.cs rename to test/standalone/Garnet.test/NumUtils.cs diff --git a/test/Garnet.test/ObjectExpiryTxn.cs b/test/standalone/Garnet.test/ObjectExpiryTxn.cs similarity index 91% rename from test/Garnet.test/ObjectExpiryTxn.cs rename to test/standalone/Garnet.test/ObjectExpiryTxn.cs index 090321f0964..7a69716028f 100644 --- a/test/Garnet.test/ObjectExpiryTxn.cs +++ b/test/standalone/Garnet.test/ObjectExpiryTxn.cs @@ -19,7 +19,7 @@ sealed class ObjectExpiryTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, true); + AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, StoreType.Object); return true; } @@ -29,7 +29,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var key = GetNextArg(ref procInput.parseState, ref offset); var expiryMs = GetNextArg(ref procInput.parseState, ref offset); - api.EXPIRE(key, expiryMs, out _, StoreType.Object); + api.EXPIRE(key, expiryMs, out _); WriteSimpleString(ref output, "SUCCESS"); } } diff --git a/test/Garnet.test/ReqGen.cs b/test/standalone/Garnet.test/ReqGen.cs similarity index 100% rename from test/Garnet.test/ReqGen.cs rename to test/standalone/Garnet.test/ReqGen.cs diff --git a/test/Garnet.test/Resp/RespParseFuzzRegressionTests.cs b/test/standalone/Garnet.test/Resp/RespParseFuzzRegressionTests.cs similarity index 97% rename from test/Garnet.test/Resp/RespParseFuzzRegressionTests.cs rename to test/standalone/Garnet.test/Resp/RespParseFuzzRegressionTests.cs index f8d82443db5..daae8710f50 100644 --- a/test/Garnet.test/Resp/RespParseFuzzRegressionTests.cs +++ b/test/standalone/Garnet.test/Resp/RespParseFuzzRegressionTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; -using Allure.NUnit; using Garnet.common.Parsing; using Garnet.server; using NUnit.Framework; @@ -15,10 +14,9 @@ namespace Garnet.test.Resp /// /// Accordingly these are a bit of a grab bag, but they need to go somewhere. /// - [AllureNUnit] [TestFixture] [Category("FUZZING")] - public class RespParseFuzzRegressionTests : AllureTestBase + public class RespParseFuzzRegressionTests : TestBase { [Test] public void MakeUpperCaseAccessViolation() diff --git a/test/Garnet.test/Resp/RespReadUtilsTests.cs b/test/standalone/Garnet.test/Resp/RespReadUtilsTests.cs similarity index 78% rename from test/Garnet.test/Resp/RespReadUtilsTests.cs rename to test/standalone/Garnet.test/Resp/RespReadUtilsTests.cs index 043a05ed450..b1aae7e529f 100644 --- a/test/Garnet.test/Resp/RespReadUtilsTests.cs +++ b/test/standalone/Garnet.test/Resp/RespReadUtilsTests.cs @@ -1,8 +1,7 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System.Text; -using Allure.NUnit; using Garnet.common; using Garnet.common.Parsing; using NUnit.Framework; @@ -13,9 +12,8 @@ namespace Garnet.test.Resp /// /// Tests for RespReadUtils parsing functions. /// - [AllureNUnit] [TestFixture] - unsafe class RespReadUtilsTests : AllureTestBase + unsafe class RespReadUtilsTests : TestBase { /// /// Tests that ReadLengthHeader successfully parses valid numbers. @@ -291,5 +289,96 @@ public static unsafe void ReadBoolWithLengthHeaderTest(string text, bool expecte ClassicAssert.IsTrue(start == end); } } + + /// + /// Tests that GetSerializedRecordSpan correctly parses a valid length-prefixed record. + /// + [Test] + public static unsafe void GetSerializedRecordSpanValidTest() + { + // Layout: [int32 length = 5][5 bytes of data] + var data = new byte[sizeof(int) + 5]; + fixed (byte* ptr = data) + { + *(int*)ptr = 5; + ptr[4] = 0xAA; + ptr[5] = 0xBB; + ptr[6] = 0xCC; + ptr[7] = 0xDD; + ptr[8] = 0xEE; + + var start = ptr; + var end = ptr + data.Length; + var success = RespReadUtils.GetSerializedRecordSpan(out var recordSpan, ref start, end); + + ClassicAssert.IsTrue(success); + ClassicAssert.AreEqual(5, recordSpan.Length); + ClassicAssert.AreEqual(0xAA, recordSpan.ReadOnlySpan[0]); + ClassicAssert.AreEqual(0xEE, recordSpan.ReadOnlySpan[4]); + ClassicAssert.IsTrue(start == end); + } + } + + /// + /// Tests that GetSerializedRecordSpan rejects a record whose declared length exceeds the payload boundary. + /// + [Test] + public static unsafe void GetSerializedRecordSpanOverflowLengthTest() + { + // Layout: [int32 length = 1000][only 3 bytes of actual data] + var data = new byte[sizeof(int) + 3]; + fixed (byte* ptr = data) + { + *(int*)ptr = 1000; + ptr[4] = 0x01; + ptr[5] = 0x02; + ptr[6] = 0x03; + + var start = ptr; + var end = ptr + data.Length; + var success = RespReadUtils.GetSerializedRecordSpan(out var recordSpan, ref start, end); + + ClassicAssert.IsFalse(success); + ClassicAssert.AreEqual(0, recordSpan.Length); + } + } + + /// + /// Tests that GetSerializedRecordSpan rejects a negative record length. + /// + [Test] + public static unsafe void GetSerializedRecordSpanNegativeLengthTest() + { + var data = new byte[sizeof(int) + 10]; + fixed (byte* ptr = data) + { + *(int*)ptr = -1; + + var start = ptr; + var end = ptr + data.Length; + var success = RespReadUtils.GetSerializedRecordSpan(out var recordSpan, ref start, end); + + ClassicAssert.IsFalse(success); + ClassicAssert.AreEqual(0, recordSpan.Length); + } + } + + /// + /// Tests that GetSerializedRecordSpan rejects when there's not enough data for the length prefix itself. + /// + [Test] + public static unsafe void GetSerializedRecordSpanInsufficientHeaderTest() + { + var data = new byte[2]; // Less than sizeof(int) + fixed (byte* ptr = data) + { + var start = ptr; + var end = ptr + data.Length; + var success = RespReadUtils.GetSerializedRecordSpan(out var recordSpan, ref start, end); + + ClassicAssert.IsFalse(success); + ClassicAssert.AreEqual(0, recordSpan.Length); + } + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespAdminCommandsTests.cs b/test/standalone/Garnet.test/RespAdminCommandsTests.cs similarity index 96% rename from test/Garnet.test/RespAdminCommandsTests.cs rename to test/standalone/Garnet.test/RespAdminCommandsTests.cs index 3d1f801b2b9..ca21cb107cc 100644 --- a/test/Garnet.test/RespAdminCommandsTests.cs +++ b/test/standalone/Garnet.test/RespAdminCommandsTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -15,9 +14,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespAdminCommandsTests : AllureTestBase + public class RespAdminCommandsTests : TestBase { GarnetServer server; @@ -331,15 +329,18 @@ static void ValidateServerData(IDatabase db, string strKey, string strValue, str [Test] [TestCase(63, 15, 1)] - [TestCase(63, 2, 1)] + [TestCase(63, 4, 1)] [TestCase(16, 16, 1)] [TestCase(5, 64, 1)] + //[Repeat(3000)] public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemorySize, int pageSize) { - string sizeToString(int size) => size + "k"; + static string sizeToString(int size) => size + "k"; server.Dispose(); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, memorySize: sizeToString(memorySize), pageSize: sizeToString(pageSize)); + var pageCount = recoveryMemorySize / pageSize; + var totalMemorySize = recoveryMemorySize + 64; // Add in some for heap + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, lowMemory: true, memorySize: sizeToString(totalMemorySize), pageCount: pageCount, pageSize: sizeToString(pageSize)); server.Start(); var ldata = new RedisValue[] { "a", "b", "c", "d" }; @@ -347,28 +348,29 @@ public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemoryS using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { var db = redis.GetDatabase(0); - for (int i = 0; i < 3000; i++) - db.ListLeftPush($"SeSaveRecoverTestKey{i:0000}", ldata); + for (var i = 0; i < 3000; i++) + _ = db.ListLeftPush($"SeSaveRecoverTestKey{i:0000}", ldata); - for (int i = 0; i < 3000; i++) + for (var i = 0; i < 3000; i++) ClassicAssert.AreEqual(ldataArr, db.ListRange($"SeSaveRecoverTestKey{i:0000}"), $"key {i:0000}"); // Issue and wait for DB save var server = redis.GetServer(TestUtils.EndPoint); server.Save(SaveType.BackgroundSave); - while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); + while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) + Thread.Sleep(10); } server.Dispose(false); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, memorySize: sizeToString(recoveryMemorySize), pageSize: sizeToString(pageSize), objectStoreHeapMemorySize: "64k"); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, memorySize: sizeToString(totalMemorySize /* will add 'k' */), pageCount: pageCount, pageSize: sizeToString(pageSize)); server.Start(); - ClassicAssert.LessOrEqual(server.Provider.StoreWrapper.objectStore.MaxAllocatedPageCount, (recoveryMemorySize / pageSize) + 1); + ClassicAssert.LessOrEqual(server.Provider.StoreWrapper.store.HighWaterAllocatedPageCount, pageCount); using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) { var db = redis.GetDatabase(0); for (var i = 3000; i < 3100; i++) - db.ListLeftPush($"SeSaveRecoverTestKey{i:0000}", ldata); + _ = db.ListLeftPush($"SeSaveRecoverTestKey{i:0000}", ldata); for (var i = 0; i < 3100; i++) ClassicAssert.AreEqual(ldataArr, db.ListRange($"SeSaveRecoverTestKey{i:0000}"), $"key {i:0000}"); @@ -378,7 +380,7 @@ public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemoryS [Test] [TestCase("63k", "15k")] [TestCase("63k", "3k")] - [TestCase("63k", "1k")] + [TestCase("63k", "2k")] [TestCase("8k", "5k")] [TestCase("16k", "16k")] [TestCase("5k", "8k")] @@ -395,9 +397,7 @@ public void SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemo { var db = redis.GetDatabase(0); for (int i = 0; i < 1000; i++) - { db.StringSet($"SeSaveRecoverTestKey{i:0000}", $"SeSaveRecoverTestValue"); - } for (int i = 0; i < 1000; i++) { @@ -413,9 +413,7 @@ public void SeSaveRecoverMultipleKeysTest(string memorySize, string recoveryMemo while (server.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) Thread.Sleep(10); for (int i = 1000; i < 2000; i++) - { db.StringSet($"SeSaveRecoverTestKey{i:0000}", $"SeSaveRecoverTestValue"); - } for (int i = 1000; i < 2000; i++) { diff --git a/test/Garnet.test/RespCommandStatsTests.cs b/test/standalone/Garnet.test/RespCommandStatsTests.cs similarity index 99% rename from test/Garnet.test/RespCommandStatsTests.cs rename to test/standalone/Garnet.test/RespCommandStatsTests.cs index 8cdebe5a083..86e4fd6c196 100644 --- a/test/Garnet.test/RespCommandStatsTests.cs +++ b/test/standalone/Garnet.test/RespCommandStatsTests.cs @@ -4,16 +4,14 @@ using System; using System.Linq; using System.Threading; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespCommandStatsTests : AllureTestBase + public class RespCommandStatsTests : TestBase { GarnetServer server; diff --git a/test/Garnet.test/RespCommandTests.cs b/test/standalone/Garnet.test/RespCommandTests.cs similarity index 99% rename from test/Garnet.test/RespCommandTests.cs rename to test/standalone/Garnet.test/RespCommandTests.cs index 33fdc22e754..6a633294a5f 100644 --- a/test/Garnet.test/RespCommandTests.cs +++ b/test/standalone/Garnet.test/RespCommandTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; @@ -7,7 +7,6 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -19,9 +18,8 @@ namespace Garnet.test /// /// This test class tests the RESP COMMAND and COMMAND INFO commands /// - [AllureNUnit] [TestFixture] - public class RespCommandTests : AllureTestBase + public class RespCommandTests : TestBase { GarnetServer server; private string extTestDir; @@ -51,7 +49,9 @@ public class RespCommandTests : AllureTestBase RespCommand.BITOP_NOT, RespCommand.BITOP_DIFF, RespCommand.INVALID, - RespCommand.DELIFEXPIM + RespCommand.DELIFEXPIM, + RespCommand.RIPROMOTE, + RespCommand.RIRESTORE ]; [SetUp] diff --git a/test/standalone/Garnet.test/RespConfigTests.cs b/test/standalone/Garnet.test/RespConfigTests.cs new file mode 100644 index 00000000000..992294cf82c --- /dev/null +++ b/test/standalone/Garnet.test/RespConfigTests.cs @@ -0,0 +1,742 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Linq; +using System.Threading; +using Garnet.common; +using Garnet.server; +using NUnit.Framework; +using NUnit.Framework.Legacy; +using StackExchange.Redis; +using Tsavorite.core; +using static Tsavorite.core.Utility; + +namespace Garnet.test +{ + /// + /// Test dynamically changing server configuration using CONFIG SET command. + /// + [TestFixture(RevivificationMode.NoReviv)] + [TestFixture(RevivificationMode.UseReviv)] + public class RespConfigTests : TestBase + { + GarnetServer server; + private readonly string memorySizeStr = "17g"; + private readonly string indexSizeStr = "64m"; + private readonly string pageSizeStr = "32m"; + private readonly bool useReviv; + + // The HLOG will always have at least two pages allocated. + internal const int MinLogAllocatedPageCount = 2; + + public RespConfigTests(RevivificationMode revivMode) + { + this.useReviv = revivMode == RevivificationMode.UseReviv; + } + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, + memorySize: memorySizeStr, + indexSize: indexSizeStr, + pageSize: pageSizeStr, + useReviv: useReviv); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + /// + /// This test verifies that dynamically changing the memory size configuration using CONFIG SET memory + /// incurs the expected changes in Garnet server metrics, as well as verifies error handling for incorrect inputs. + /// + /// Memory size smaller than the initial size + /// Memory size larger than the initial size (within buffer bounds) + /// Memory size larger than the buffer size + /// Malformed memory size string + /// Initial memory size for main log is 32GB + [Test] + [TestCase("16g", "32g", "64g", "g4")] + [TestCase("9gB", "28GB", "33G", "2gBB")] + [TestCase("128m", "256m", "256GB", "3bm")] + [TestCase("500m", "1500M", "128GB", "44d")] + public void ConfigSetMemorySizeTest(string smallerSize, string largerSize, string largerThanBufferSize, string malformedSize) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var option = "memory"; + var metricName = "Log.AllocatedPageCount"; + var metricType = InfoMetricsType.STORE; + var initMemorySize = memorySizeStr; + + var store = server.Provider.StoreWrapper.store; + var tracker = store.Log.LogSizeTracker; + var currMemorySize = ServerOptions.ParseSize(initMemorySize, out _); + var pageSize = ServerOptions.ParseSize(pageSizeStr, out _); + + var bufferSizeInBytes = ServerOptions.NextPowerOf2(currMemorySize); + Assert.That(bufferSizeInBytes / pageSize, Is.EqualTo(store.Log.BufferSize)); + + // expectedMaxAPC does not change after being set in AllocatorBase initialization + var expectedMaxAPC = (int)(RoundUp(currMemorySize, pageSize) / pageSize); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Check initial AllocatedPageCount before any changes + var metrics = server.Metrics.GetInfoMetrics(metricType); + var miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out var allocatedPageCount)); + long expectedAPC = MinLogAllocatedPageCount; + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Try to set memory size to the same value as current + var result = db.Execute("CONFIG", "SET", option, initMemorySize); + ClassicAssert.AreEqual("OK", result.ToString()); + + // miAPC should remain unchanged + metrics = server.Metrics.GetInfoMetrics(metricType); + miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out allocatedPageCount)); + // expectedAPC remains unchanged because we didn't add records + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Try to set memory size to a smaller value than current + result = db.Execute("CONFIG", "SET", option, smallerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Check that miAPC has changed accordingly + currMemorySize = ServerOptions.ParseSize(smallerSize, out _); + metrics = server.Metrics.GetInfoMetrics(metricType); + miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out allocatedPageCount)); + // expectedAPC remains unchanged because we didn't add records + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Try to set memory size to a larger value than current + result = db.Execute("CONFIG", "SET", option, largerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Check that miAPC has changed accordingly + currMemorySize = ServerOptions.ParseSize(largerSize, out _); + metrics = server.Metrics.GetInfoMetrics(metricType); + miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out allocatedPageCount)); + // expectedAPC remains unchanged because we didn't add records + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Try to set memory size larger than the buffer size - this should fail + _ = Assert.Throws(() => db.Execute("CONFIG", "SET", option, largerThanBufferSize), + string.Format(CmdStrings.GenericErrMemorySizeGreaterThanBuffer, option)); + + // Page counts should remain unchanged + metrics = server.Metrics.GetInfoMetrics(metricType); + miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out allocatedPageCount)); + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + + // Try to set memory size with a malformed size input - this should fail + _ = Assert.Throws(() => db.Execute("CONFIG", "SET", option, malformedSize), + string.Format(CmdStrings.GenericErrIncorrectSizeFormat, option)); + + // Page counts should remain unchanged + metrics = server.Metrics.GetInfoMetrics(metricType); + miAPC = metrics.FirstOrDefault(mi => mi.Name == metricName); + ClassicAssert.IsNotNull(miAPC); + ClassicAssert.IsTrue(long.TryParse(miAPC.Value, out allocatedPageCount)); + ClassicAssert.AreEqual(expectedAPC, allocatedPageCount); + Assert.That(tracker.logAccessor.allocatorBase.MaxAllocatedPageCount, Is.EqualTo(expectedMaxAPC)); + } + + /// + /// This test verifies that dynamically changing the index size configuration using CONFIG SET index / obj-index + /// incurs the expected changes in Garnet server metrics, as well as verifies error handling for incorrect inputs. + /// + /// Index size smaller than the initial size + /// Index size larger than the initial size + /// Illegal index size (not a power of 2) + /// Malformed index size string + /// Initial index size for main log is 1MB + [Test] + [TestCase("32m", "128m", "63m", "8d")] + [TestCase("16mB", "256MB", "23m", "g8")] + [TestCase("2m", "512m", "28m", "m9")] + [TestCase("4Mb", "1024mB", "129MB", "0.3gb")] + public void ConfigSetIndexSizeTest(string smallerSize, string largerSize, string illegalSize, string malformedSize) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var metricType = InfoMetricsType.STORE; + var option = "index"; + var initIndexSize = indexSizeStr; + + // Check initial index size before any changes + var currIndexSize = ServerOptions.ParseSize(initIndexSize, out _); + var metrics = server.Metrics.GetInfoMetrics(metricType); + var miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexMemorySize"); + ClassicAssert.IsNotNull(miIndexSize); + ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out var actualIndexSize)); + var expectedIndexSize = currIndexSize / 64; + ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); + + // Try to set index size to the same value as current + var result = db.Execute("CONFIG", "SET", option, initIndexSize); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Index size should remain unchanged + metrics = server.Metrics.GetInfoMetrics(metricType); + miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexMemorySize"); + ClassicAssert.IsNotNull(miIndexSize); + ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out actualIndexSize)); + ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); + + // Try to set index size to a larger value than current + result = db.Execute("CONFIG", "SET", option, largerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Check that index size has changed accordingly + currIndexSize = ServerOptions.ParseSize(largerSize, out _); + metrics = server.Metrics.GetInfoMetrics(metricType); + miIndexSize = metrics.FirstOrDefault(mi => mi.Name == "IndexMemorySize"); + ClassicAssert.IsNotNull(miIndexSize); + ClassicAssert.IsTrue(long.TryParse(miIndexSize.Value, out actualIndexSize)); + expectedIndexSize = currIndexSize / 64; + ClassicAssert.AreEqual(expectedIndexSize, actualIndexSize); + + // Try to set index size to a smaller value than current - this should fail + _ = Assert.Throws(() => db.Execute("CONFIG", "SET", option, smallerSize), + string.Format(CmdStrings.GenericErrIndexSizeSmallerThanCurrent, option)); + + // Try to set index size to a value that is not a power of two - this should fail + _ = Assert.Throws(() => db.Execute("CONFIG", "SET", option, illegalSize), + string.Format(CmdStrings.GenericErrIndexSizePowerOfTwo, option)); + + // Try to set index size with a malformed size input - this should fail + _ = Assert.Throws(() => db.Execute("CONFIG", "SET", option, malformedSize), + string.Format(CmdStrings.GenericErrIncorrectSizeFormat, option)); + } + } + + + + /// + /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET memory. + /// + [TestFixture(RevivificationMode.NoReviv)] + [TestFixture(RevivificationMode.UseReviv)] + public class RespConfigUtilizationTests : TestBase + { + GarnetServer server; + private readonly string memorySize = "3m"; + private readonly string indexSize = "1m"; + private readonly string pageSize = "1024"; + private readonly bool useReviv; + + public RespConfigUtilizationTests(RevivificationMode revivMode) + { + this.useReviv = revivMode == RevivificationMode.UseReviv; + } + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(null, + memorySize: memorySize, + indexSize: indexSize, + pageSize: pageSize, + useReviv: useReviv); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + /// + /// This test verifies that dynamically changing the memory size configuration using CONFIG SET + /// incurs the expected shifts in the head and tail addresses of the store. + /// + /// Memory size smaller than the initial size + /// Memory size larger than the initial size (within buffer bounds) + [Test] + [TestCase("1m", "4m")] + [TestCase("1024k", "4000k")] + [TestCase("4k", "8k")] + [TestCase("8k", "64k")] + public void ConfigSetInlineMemorySizeUtilizationTest(string smallerSize, string largerSize) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + var option = "memory"; + var currMemorySize = TestUtils.GetEffectiveMemorySize(memorySize, pageSize, out var parsedPageSize); + var initialMemorySize = currMemorySize; + + var store = server.Provider.StoreWrapper.store; + var tracker = store.Log.LogSizeTracker; + Assert.That(tracker.TargetSize, Is.EqualTo(currMemorySize)); + + using var trimCompleteEvent = new ManualResetEventSlim(false); + tracker.PostMemoryTrim = (allocatedPageCount, headAddress) => { trimCompleteEvent.Set(); }; + + var garnetServer = redis.GetServer(TestUtils.EndPoint); + var info = TestUtils.GetStoreAddressInfo(garnetServer); + ClassicAssert.AreEqual(PageHeader.Size, info.TailAddress); + + var i = 0; + var val = new RedisValue(new string('x', 512 - 32)); + + // Insert records until head address moves + var prevHead = info.HeadAddress; + var prevTail = info.TailAddress; + while (info.HeadAddress == prevHead) + { + var key = $"key{i++:00000}"; + _ = db.StringSet(key, val); + + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + info = TestUtils.GetStoreAddressInfo(garnetServer); + } + + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + + // Verify that records were inserted up to the configured memory size limit. + // We may have overflowed by multiple pages. + Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(tracker.TargetDeltaRange.high)); + + //////////////////////////////////////////////////////// + // Try to set memory size to a smaller value than current + currMemorySize = TestUtils.GetEffectiveMemorySize(smallerSize, pageSize, out _); + Assert.That(currMemorySize, Is.LessThan(initialMemorySize)); + var result = db.Execute("CONFIG", "SET", option, smallerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + Assert.That(tracker.TargetSize, Is.EqualTo(currMemorySize)); + + // Insert records until head address moves + // Precondition: We have too much in memory for the smallSize and must evict. + Assert.That(prevTail - prevHead, Is.GreaterThan(tracker.TargetDeltaRange.high)); + + // Wait for the logSizeTracker to stabilize. + Assert.That(trimCompleteEvent.Wait(TimeSpan.FromSeconds(3 * 3 * LogSizeTracker.ResizeTaskDelaySeconds)), + "Timeout occurred. Resizing did not happen within the specified time."); + + info = TestUtils.GetStoreAddressInfo(garnetServer); + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + + // Verify that records were inserted up to the configured memory size limit. + // We may have overflowed by multiple pages. + Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(tracker.TargetDeltaRange.high)); + + //////////////////////////////////////////////////////// + // Try to set memory size to a larger value than current + currMemorySize = TestUtils.GetEffectiveMemorySize(largerSize, pageSize, out _); + result = db.Execute("CONFIG", "SET", option, largerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + Assert.That(tracker.TargetSize, Is.EqualTo(currMemorySize)); + + // Continue to insert records until new memory capacity is reached + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + while (info.HeadAddress == prevHead) + { + var key = $"key{i++:00000}"; + _ = db.StringSet(key, val); + + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + info = TestUtils.GetStoreAddressInfo(garnetServer); + } + + // Verify that memory is fully utilized and within memory bounds + Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(tracker.TargetDeltaRange.high)); + } + + /// + /// This test verifies recovery behavior after dynamically changing the memory size configuration using CONFIG SET memory. + /// The test fills the store to a larger capacity than the initial memory size, then verifies that recovering with the + /// smaller initial memory size retains the last inserted keys in the expected initial capacity. + /// + /// Memory size larger than the initial size (within buffer bounds) + [Test] + [TestCase("4m")] + public void ConfigSetMemorySizeRecoveryTest(string largerSize) + { + var option = "memory"; + var initMemorySize = memorySize; + + var currMemorySize = TestUtils.GetEffectiveMemorySize(initMemorySize, pageSize, out var parsedPageSize); + + var store = server.Provider.StoreWrapper.store; + var tracker = store.Log.LogSizeTracker; + Assert.That(tracker.TargetSize, Is.EqualTo(currMemorySize)); + Assert.That(store.hlogBase.MaxAllocatedPageCount, Is.EqualTo((int)(currMemorySize / parsedPageSize))); + + int lastIdxSecondRound; + int keysInsertedFirstRound; + + // These are outside the individual blocks for debugging + var lastIdxFirstRound = -1; + var allocatedPagesFirstRound = -1; + var allocatedPagesSecondRound = -1; + long highTarget1 = -1, lowTarget1 = -1; + int maxAllocatedPageCount1 = -1; + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + var garnetServer = redis.GetServer(TestUtils.EndPoint); + var info = TestUtils.GetStoreAddressInfo(garnetServer); + ClassicAssert.AreEqual(PageHeader.Size, info.TailAddress); + + // Insert records until head address moves. We want to fit two records per page; pages are 1024 bytes so after subtracting + // PageHeader.Size we have 960 / 2 = 480 bytes per record. Keys are 8 bytes, valueLength requires 2 bytes as it will be + // more than 255, we have no optionals (ETag or Expiration), and we are inline so have no ObjectLogPosition, so: + // RecordInfo.Size + (MinLengthMetadataBytes + 1) + 8 + valueLength = 480, so valueLength = 480-22 = 458 bytes. + // It's rounded up to kRecordAlignment (8) anyway. + var val = new RedisValue(new string('x', 458)); + + var i = 0; + var prevHead = info.HeadAddress; + var prevTail = info.TailAddress; + while (info.HeadAddress == prevHead) + { + var key = $"key{i++:00000}"; + _ = db.StringSet(key, val); + + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + info = TestUtils.GetStoreAddressInfo(garnetServer); + } + + lastIdxFirstRound = i - 1; + + // Verify that records were inserted up to the configured memory size limit + // We may have overflowed by multiple pages. + Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(tracker.TargetDeltaRange.high)); + + // Find the first key index that still exists in the server + ClassicAssert.IsTrue(db.KeyExists($"key{lastIdxFirstRound:00000}")); + var c = lastIdxFirstRound; + while (c > 0 && db.KeyExists($"key{--c:00000}")) + continue; + + // Record the number of keys inserted in the first round + keysInsertedFirstRound = lastIdxFirstRound + 1 - c; + allocatedPagesFirstRound = store.hlogBase.AllocatedPageCount - 1; // APC includes the one-ahead page allocation + Assert.That(allocatedPagesFirstRound, Is.LessThanOrEqualTo(store.hlogBase.MaxAllocatedPageCount)); + + (highTarget1, lowTarget1) = tracker.TargetDeltaRange; + maxAllocatedPageCount1 = tracker.logAccessor.allocatorBase.MaxAllocatedPageCount; + + //////////////////////////////////////////////////////// + // Try to set memory size to a larger value than current + var result = db.Execute("CONFIG", "SET", option, largerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + + currMemorySize = TestUtils.GetEffectiveMemorySize(largerSize, pageSize, out _); + + // Continue to insert records until new memory capacity is reached + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + while (info.HeadAddress == prevHead) + { + var key = $"key{i++:00000}"; + _ = db.StringSet(key, val); + + prevHead = info.HeadAddress; + prevTail = info.TailAddress; + info = TestUtils.GetStoreAddressInfo(garnetServer); + } + + lastIdxSecondRound = i - 1; + allocatedPagesSecondRound = store.hlogBase.AllocatedPageCount - 1; // APC includes the one-ahead page allocation + + // Verify that memory is fully utilized + Assert.That(prevTail - prevHead, Is.LessThanOrEqualTo(currMemorySize)); + + // SAVE and wait for completion + garnetServer.Save(SaveType.BackgroundSave); + while (garnetServer.LastSave().Ticks == DateTimeOffset.FromUnixTimeSeconds(0).Ticks) + Thread.Sleep(10); + } + + // Doing this here so lastIdxFirstRound remains visible for debugging without getting a warning. + Assert.That(lastIdxSecondRound, Is.GreaterThan(lastIdxFirstRound)); + + /////////////////////////////////////////////////////////// + // Restart server with initial memory size and recover data + server.Dispose(deleteDir: false); + server = TestUtils.CreateGarnetServer(null, + memorySize: memorySize, + indexSize: indexSize, + pageSize: pageSize, + useReviv: useReviv, + tryRecover: true); + server.Start(); + + store = server.Provider.StoreWrapper.store; + tracker = store.Log.LogSizeTracker; + var allocatedPagesRestore = store.hlogBase.AllocatedPageCount - 1; // APC includes the one-ahead page allocation + + var (highTargetRestore, lowTargetRestore) = tracker.TargetDeltaRange; + Assert.That(highTargetRestore, Is.EqualTo(highTarget1)); + Assert.That(lowTargetRestore, Is.EqualTo(lowTarget1)); + var maxAllocatedPageCount2 = tracker.logAccessor.allocatorBase.MaxAllocatedPageCount; + Assert.That(maxAllocatedPageCount2, Is.EqualTo(maxAllocatedPageCount1)); + + // Recovery and insertion don't track sizes exactly the same way with logSizeTracker enabled, so this is not entirely deterministic; just verify the ranges. + Assert.That(allocatedPagesRestore, Is.GreaterThanOrEqualTo(allocatedPagesFirstRound)); + Assert.That(allocatedPagesRestore, Is.LessThanOrEqualTo(allocatedPagesSecondRound)); + Assert.That(allocatedPagesRestore, Is.GreaterThanOrEqualTo(lowTargetRestore / store.Log.allocatorBase.PageSize)); + Assert.That(allocatedPagesRestore, Is.LessThanOrEqualTo(RoundUp(highTargetRestore, store.Log.allocatorBase.PageSize) / store.Log.allocatorBase.PageSize)); + + using (var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true))) + { + var db = redis.GetDatabase(0); + + // Find the smallest key index that still exists in the server + var c = lastIdxSecondRound; + while (c > 0 && db.KeyExists($"key{--c:00000}")) + continue; + + // Verify the head/tail addresses are within range and that the number of existing keys matches the head/tail range. We should have two keys per page. + var addressRange = store.Log.TailAddress - store.Log.HeadAddress; + var addressRangePages = RoundUp(addressRange, store.Log.allocatorBase.PageSize) / store.Log.allocatorBase.PageSize; + Assert.That(addressRange, Is.LessThanOrEqualTo(highTargetRestore)); + Assert.That(lastIdxSecondRound + 1 - c, Is.EqualTo(allocatedPagesRestore * 2)); // AllocatedPageCount includes the "allocate-ahead" page + Assert.That(lastIdxSecondRound + 1 - c, Is.EqualTo(addressRangePages * 2)); + + // Verify that all previous keys are not present in the database + while (c > 0) + ClassicAssert.IsFalse(db.KeyExists($"key{--c:00000}")); + } + } + } + + /// + /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET. + /// + [TestFixture(RevivificationMode.NoReviv)] + [TestFixture(RevivificationMode.UseReviv)] + public class RespConfigIndexUtilizationTests : TestBase + { + GarnetServer server; + private readonly string memorySize = "3m"; + private readonly string indexSize = "512"; + private readonly string pageSize = "1024"; + private readonly bool useReviv; + + public RespConfigIndexUtilizationTests(RevivificationMode revivMode) + { + this.useReviv = revivMode == RevivificationMode.UseReviv; + } + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(null, + memorySize: memorySize, + indexSize: indexSize, + pageSize: pageSize, + useReviv: useReviv); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + /// + /// This test verifies that dynamically changing the index size configuration using CONFIG SET + /// incurs the expected shifts in the overflow buckets of the store, and that no data is lost in the process. + /// + /// Larger index size than configured + /// Larger index size than previous + [Test] + [TestCase("1024", "4096")] + public void ConfigSetIndexSizeUtilizationTest(string largerSize1, string largerSize2) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + var option = "index"; + var parsedIndexSize = ServerOptions.ParseSize(indexSize, out _); + + var currIndexSize = server.Provider.StoreWrapper.store.IndexSize; + + // Verify initial index size and overflow bucket allocations are zero + ClassicAssert.AreEqual(parsedIndexSize / 64, currIndexSize); + ClassicAssert.AreEqual(0, GetOverflowBucketAllocations()); + + // Generate data with random keys (so that hashtable overflows) + var val = new RedisValue("x"); + var keys = new string[500]; + for (var i = 0; i < keys.Length; i++) + keys[i] = TestUtils.GetRandomString(8); + + // Insert first batch of data + for (var i = 0; i < 250; i++) + _ = db.StringSet(keys[i], val); + + // Verify that overflow bucket allocations are non-zero after initial insertions + var currOverflowBucketAllocations = GetOverflowBucketAllocations(); + ClassicAssert.Greater(currOverflowBucketAllocations, 0); + var prevOverflowBucketAllocations = currOverflowBucketAllocations; + + // Try to set index size to a larger value than current + var result = db.Execute("CONFIG", "SET", option, largerSize1); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Verify that overflow bucket allocations have not increased after growing the index. + // Note: the split algorithm may produce equal overflow counts in some cases because + // it inserts chain-start entries for both sides of the split, so we use LessOrEqual. + currOverflowBucketAllocations = GetOverflowBucketAllocations(); + ClassicAssert.LessOrEqual(currOverflowBucketAllocations, prevOverflowBucketAllocations); + + // Insert second batch of data + for (var i = 250; i < 500; i++) + _ = db.StringSet(keys[i], val); + + prevOverflowBucketAllocations = GetOverflowBucketAllocations(); + + // Try to set index size to a larger value than current + result = db.Execute("CONFIG", "SET", option, largerSize2); + ClassicAssert.AreEqual("OK", result.ToString()); + + // Verify that overflow bucket allocations have not increased again + currOverflowBucketAllocations = GetOverflowBucketAllocations(); + ClassicAssert.LessOrEqual(currOverflowBucketAllocations, prevOverflowBucketAllocations); + + // Verify that all keys still exist in the database + foreach (var key in keys) + ClassicAssert.IsTrue(db.KeyExists(key)); + + long GetOverflowBucketAllocations() => + server.Provider.StoreWrapper.store.OverflowBucketAllocations; + } + } + + /// + /// Test memory utilization behavior when dynamically changing the memory size configuration using CONFIG SET memory. + /// + [TestFixture(RevivificationMode.NoReviv)] + [TestFixture(RevivificationMode.UseReviv)] + public class RespConfigHeapUtilizationTests : TestBase + { + GarnetServer server; + private readonly string memorySize = "3m"; + private readonly string indexSize = "512"; + private readonly string pageSize = "1024"; + private readonly bool useReviv; + + public RespConfigHeapUtilizationTests(RevivificationMode revivMode) + { + this.useReviv = revivMode == RevivificationMode.UseReviv; + } + + [SetUp] + public void Setup() + { + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); + server = TestUtils.CreateGarnetServer(null, + memorySize: memorySize, + indexSize: indexSize, + pageSize: pageSize, + useReviv: useReviv); + server.Start(); + } + + [TearDown] + public void TearDown() + { + server.Dispose(); + TestUtils.OnTearDown(); + } + + /// + /// This test verifies that dynamically shrinking the main-log memory target via CONFIG SET memory + /// drives the log size tracker to evict pages / release heap so that total memory usage is reduced. + /// + /// Memory size smaller than the current total log+heap usage + [Test] + [TestCase("8192")] + public void ConfigSetHeapMemorySizeUtilizationTest(string smallerSize) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig(allowAdmin: true)); + var db = redis.GetDatabase(0); + var option = "memory"; + + var store = server.Provider.StoreWrapper.store; + var tracker = store.Log.LogSizeTracker; + + // Verify initial state: no data yet, allocated page count at the minimum. + ClassicAssert.AreEqual(RespConfigTests.MinLogAllocatedPageCount, store.Log.AllocatedPageCount); + + using var trimCompleteEvent = new ManualResetEventSlim(false); + tracker.PostMemoryTrim = (allocatedPageCount, headAddress) => { trimCompleteEvent.Set(); }; + + // Insert enough list objects so that (a) the inline log grows beyond MinResizeTargetPageCount pages + // (required before DetermineEvictionRange can evict anything), and (b) the tracked heap size is + // well above the new target we're about to configure. Each list contributes ~4KB of heap + // (16 items * 256 bytes), and each record adds a small inline entry to the log. + const int numKeys = 128; + var values = new RedisValue[16]; + var valPayload = new string('x', 256); + for (var i = 0; i < values.Length; i++) + values[i] = valPayload; + for (var i = 0; i < numKeys; i++) + _ = db.ListRightPush($"key{i:00000}", values); + + // Sanity-check the preconditions for the shrink/eviction we are about to trigger. + var apcBefore = store.Log.AllocatedPageCount; + var heapBefore = tracker.LogHeapSizeBytes; + Assert.That(apcBefore, Is.GreaterThan(LogSizeTracker.MinResizeTargetPageCount), + "Test precondition: need more than MinResizeTargetPageCount pages for eviction to be possible."); + Assert.That(heapBefore, Is.GreaterThan(0), "Test precondition: heap should be non-empty after inserts."); + + // Shrink the memory target. The 'shrink' branch of LogSizeTracker.UpdateTargetSize signals the + // resizer task; because TotalSize is now well above highTargetSize, the task calls + // DetermineEvictionRange + ShiftAddresses and then invokes PostMemoryTrim. + var result = db.Execute("CONFIG", "SET", option, smallerSize); + ClassicAssert.AreEqual("OK", result.ToString()); + var smallerTarget = ServerOptions.ParseSize(smallerSize, out _); + Assert.That(tracker.TargetSize, Is.EqualTo(smallerTarget)); + + // Wait for the trim callback. + Assert.That(trimCompleteEvent.Wait(TimeSpan.FromSeconds(3 * LogSizeTracker.ResizeTaskDelaySeconds)), + "Timeout occurred. PostMemoryTrim was not invoked after CONFIG SET memory shrink."); + + // Verify that memory usage actually dropped: we expect at least one of the allocated page count + // or the tracked heap size to have decreased as a result of the eviction scan. + var apcAfter = store.Log.AllocatedPageCount; + var heapAfter = tracker.LogHeapSizeBytes; + Assert.That(apcAfter < apcBefore || heapAfter < heapBefore, Is.True, + $"Expected APC ({apcBefore}->{apcAfter}) or heap ({heapBefore}->{heapAfter}) to decrease after trim."); + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/RespEtagTests.cs b/test/standalone/Garnet.test/RespEtagTests.cs similarity index 62% rename from test/Garnet.test/RespEtagTests.cs rename to test/standalone/Garnet.test/RespEtagTests.cs index 26067ecbfef..106af9c6ba6 100644 --- a/test/Garnet.test/RespEtagTests.cs +++ b/test/standalone/Garnet.test/RespEtagTests.cs @@ -8,7 +8,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -16,9 +15,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespEtagTests : AllureTestBase + public class RespETagTests : TestBase { private GarnetServer server; private Random r; @@ -57,11 +55,34 @@ public void SETReturnsEtagForNewData() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); IDatabase db = redis.GetDatabase(0); - RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); long etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, etag); } + [Test] + public void SetWithEtagClearsTTLWhenNoExpiryProvided() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + IDatabase db = redis.GetDatabase(0); + + // Set key with ETag and expiration + var etag = long.Parse(db.Execute("SETWITHETAG", "mykey", "val1", "EX", "100").ToString()); + ClassicAssert.AreEqual(1, etag); + + // Confirm TTL exists + var ttl = db.KeyTimeToLive("mykey"); + ClassicAssert.IsTrue(ttl.HasValue); + + // SETWITHETAG without EX/PX should clear the expiration + etag = long.Parse(db.Execute("SETWITHETAG", "mykey", "val2").ToString()); + ClassicAssert.AreEqual(2, etag); + + // TTL should be cleared + ttl = db.KeyTimeToLive("mykey"); + ClassicAssert.IsFalse(ttl.HasValue); + } + [Test] public void SetIfMatchReturnsNewValueAndEtagWhenEtagMatches() { @@ -69,7 +90,7 @@ public void SetIfMatchReturnsNewValueAndEtagWhenEtagMatches() IDatabase db = redis.GetDatabase(0); var key = "florida"; - RedisResult res = db.Execute("SET", [key, "one", "WITHETAG"]); + RedisResult res = db.Execute("SETWITHETAG", [key, "one"]); long initalEtag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, initalEtag); @@ -124,7 +145,7 @@ public void SetIfMatchWorksWithExpiration() var key = "florida"; // Scenario: Key existed before and had no expiration - RedisResult res = db.Execute("SET", key, "one", "WITHETAG"); + RedisResult res = db.Execute("SETWITHETAG", key, "one"); long initalEtag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, initalEtag); @@ -146,7 +167,7 @@ public void SetIfMatchWorksWithExpiration() db.KeyDelete(key); // cleanup // Scenario: Key existed before and had expiration - res = db.Execute("SET", key, "one", "WITHETAG", "PX", 100000); + res = db.Execute("SETWITHETAG", key, "one", "PX", 100000); // confirm expiration added -> TTL should exist ttl = db.KeyTimeToLive(key); @@ -165,75 +186,6 @@ public void SetIfMatchWorksWithExpiration() ClassicAssert.AreEqual(3, updatedEtagRes); db.KeyDelete(key); // cleanup - - // Scenario: SET without etag and existing expiration when sent with setifmatch will add etag and retain the expiration too - res = db.Execute("SET", key, "one", "EX", 100000); - // when no etag then count 0 as it's existing etag - updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 0)[0].ToString()); - ClassicAssert.AreEqual(1, updatedEtagRes); - - // confirm expiration retained -> TTL should exist - ttl = db.KeyTimeToLive(key); - ClassicAssert.IsTrue(ttl.HasValue); - - // confirm has etag now - var etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); - ClassicAssert.AreEqual(1, etag); - - db.KeyDelete(key); // cleanup - - - // Scenario: SET without etag and without expiration when sent with setifmatch will add etag and retain the expiration too - // copy update - res = db.Execute("SET", key, "one"); - // when no etag then count 0 as it's existing etag - updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "nextone", 0, "EX", 10000)[0].ToString()); - ClassicAssert.AreEqual(1, updatedEtagRes); - - // confirm expiration retained -> TTL should exist - ttl = db.KeyTimeToLive(key); - ClassicAssert.IsTrue(ttl.HasValue); - - // confirm has etag now - etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); - ClassicAssert.AreEqual(1, etag); - - // same length update - res = db.Execute("SET", key, "one"); - // when no etag then count 0 as it's existing etag - updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "two", 0, "EX", 10000)[0].ToString()); - ClassicAssert.AreEqual(1, updatedEtagRes); - - // confirm expiration retained -> TTL should exist - ttl = db.KeyTimeToLive(key); - ClassicAssert.IsTrue(ttl.HasValue); - - // confirm has etag now - etag = long.Parse(db.Execute("GETWITHETAG", key)[0].ToString()); - ClassicAssert.AreEqual(1, etag); - - db.KeyDelete(key); // cleanup - - // Scenario: smaller length update - res = db.Execute("SET", key, "oneofusoneofus"); - // when no etag then count 0 as it's existing etag - updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "i", 0, "EX", 10000)[0].ToString()); - ClassicAssert.AreEqual(1, updatedEtagRes); - - // confirm expiration retained -> TTL should exist - ttl = db.KeyTimeToLive(key); - ClassicAssert.IsTrue(ttl.HasValue); - - - // Scenario: smaller length update (IPU) of a key with existing etag should increment the ETAG and retain the expiration - res = db.Execute("SET", key, "oneofusoneofus", "EX", 10000); - // when no etag then count 0 as it's existing etag - updatedEtagRes = long.Parse(db.Execute("SETIFMATCH", key, "i", 0)[0].ToString()); - ClassicAssert.AreEqual(1, updatedEtagRes); - - // confirm expiration retained -> TTL should exist - ttl = db.KeyTimeToLive(key); - ClassicAssert.IsTrue(ttl.HasValue); } #endregion @@ -252,7 +204,7 @@ public void GetWithEtagReturnsValAndEtagForKey() ClassicAssert.IsTrue(nonExistingData.IsNull); // insert data - var initEtag = db.Execute("SET", key, "hkhalid", "WITHETAG"); + var initEtag = db.Execute("SETWITHETAG", key, "hkhalid"); ClassicAssert.AreEqual(1, long.Parse(initEtag.ToString())); RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); @@ -275,7 +227,7 @@ public void GetIfNotMatchReturnsDataWhenEtagDoesNotMatch() ClassicAssert.IsTrue(nonExistingData.IsNull); // insert data - var _ = db.Execute("SET", key, "maximus", "WITHETAG"); + var _ = db.Execute("SETWITHETAG", key, "maximus"); RedisResult[] noDataOnMatch = (RedisResult[])db.Execute("GETIFNOTMATCH", key, 1); ClassicAssert.AreEqual("1", noDataOnMatch[0].ToString()); @@ -288,66 +240,6 @@ public void GetIfNotMatchReturnsDataWhenEtagDoesNotMatch() ClassicAssert.AreEqual(1, etag); ClassicAssert.AreEqual("maximus", value); } - - [Test] - public void SetWithEtagWorksWithMetadata() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - IDatabase db = redis.GetDatabase(0); - - // Scenario: set withetag with expiration on non existing key - var key1 = "key1"; - var res1 = db.Execute("SET", key1, "value1", "WITHETAG", "EX", 10); - long etag1 = (long)res1; - ClassicAssert.AreEqual(1, etag1); - db.KeyDelete(key1); // Cleanup - - // Scenario: set with etag with expiration NX with existing key - var key2 = "key2"; - db.Execute("SET", key2, "value2", "WITHETAG"); - var res2 = db.Execute("SET", key2, "value3", "WITHETAG", "NX", "EX", 10); - ClassicAssert.IsTrue(res2.IsNull); - db.KeyDelete(key2); // Cleanup - - // Scenario: set with etag with expiration NX with non-existent key - var key3 = "key3"; - var res3 = db.Execute("SET", key3, "value4", "WITHETAG", "NX", "EX", 10); - long etag3 = (long)res3; - ClassicAssert.AreEqual(1, etag3); - db.KeyDelete(key3); // Cleanup - - // Scenario: set with etag with expiration XX - var key4 = "key4"; - db.Execute("SET", key4, "value5", "WITHETAG"); - var res4 = db.Execute("SET", key4, "value6", "WITHETAG", "XX", "EX", 10); - long etag4 = (long)res4; - ClassicAssert.AreEqual(2, etag4); - db.KeyDelete(key4); // Cleanup - - // Scenario: set with etag with expiration on existing data with etag - var key5 = "key5"; - db.Execute("SET", key5, "value7", "WITHETAG"); - var res5 = db.Execute("SET", key5, "value8", "WITHETAG", "EX", 10); - long etag5 = (long)res5; - ClassicAssert.AreEqual(2, etag5); - db.KeyDelete(key5); // Cleanup - - // Scenario: set with etag with expiration on existing data without etag - var key6 = "key6"; - db.Execute("SET", key6, "value9"); - var res6 = db.Execute("SET", key6, "value10", "WITHETAG", "EX", 10); - long etag6 = (long)res6; - ClassicAssert.AreEqual(1, etag6); - db.KeyDelete(key6); // Cleanup - - // Scenario: set with keepttl on key with etag and expiration should retain metadata and - var key7 = "key7"; - db.Execute("SET", key7, "value11", "WITHETAG", "EX", 10); - var res7 = db.Execute("SET", key7, "value12", "WITHETAG", "KEEPTTL"); - long etag7 = (long)res7; - ClassicAssert.AreEqual(2, etag7); - } - [Test] public void SetIfGreaterWorksWithInitialETag() { @@ -357,7 +249,7 @@ public void SetIfGreaterWorksWithInitialETag() var key = "meow-key"; var value = "m"; - RedisResult res = db.Execute("SET", key, value, "WITHETAG"); + RedisResult res = db.Execute("SETWITHETAG", key, value); ClassicAssert.AreEqual(1, (long)res); // not greater etag sent so we expect a higher etag returned @@ -421,7 +313,7 @@ public void DelIfGreaterOnAnAlreadyExistingKeyWithEtagWorks() var key = "meow-key"; var value = "m"; - RedisResult res = db.Execute("SET", key, value, "WITHETAG"); + RedisResult res = db.Execute("SETWITHETAG", key, value); ClassicAssert.AreEqual(1, (long)res); // does not delete when called with lesser or equal etag @@ -483,7 +375,7 @@ public void DelIfGreaterOnAnAlreadyExistingKeyWithEtagRCUWorks() string key = "rcuplease"; string value = "havepatiencercushallbedonethisvalueisunnecssarilylongsoicanmakesureRCUdoesnotAllocateThismuch,anythinglesserthanthisisgoodenough"; - RedisResult res = db.Execute("SET", key, value, "WITHETAG"); + RedisResult res = db.Execute("SETWITHETAG", key, value); ClassicAssert.AreEqual(1, (long)res); StoreAddressInfo info = TestUtils.GetStoreAddressInfo(garnetServer); @@ -689,7 +581,7 @@ public void SETOnAlreadyExistingSETDataOverridesItWithInitialEtag() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); IDatabase db = redis.GetDatabase(0); - RedisResult res = db.Execute("SET", "rizz", "buzz", "WITHETAG"); + RedisResult res = db.Execute("SETWITHETAG", "rizz", "buzz"); long etag = (long)res; ClassicAssert.AreEqual(1, etag); @@ -700,7 +592,7 @@ public void SETOnAlreadyExistingSETDataOverridesItWithInitialEtag() ClassicAssert.IsTrue(updateRes[1].IsNull); // inplace update - res = db.Execute("SET", "rizz", "meow", "WITHETAG"); + res = db.Execute("SETWITHETAG", "rizz", "meow"); etag = (long)res; ClassicAssert.AreEqual(3, etag); @@ -711,24 +603,18 @@ public void SETOnAlreadyExistingSETDataOverridesItWithInitialEtag() ClassicAssert.IsTrue(updateRes[1].IsNull); // Copy update - res = db.Execute("SET", ["rizz", "oneofus", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "oneofus"]); etag = (long)res; - - // now we should do a getwithetag and see the etag as 0 - res = db.Execute("SET", ["rizz", "oneofus"]); - ClassicAssert.AreEqual(res.ToString(), "OK"); - - var getwithetagRes = (RedisResult[])db.Execute("GETWITHETAG", "rizz"); - ClassicAssert.AreEqual("0", getwithetagRes[0].ToString()); + ClassicAssert.AreEqual(5, etag); } [Test] - public void SETWithWITHETAGOnAlreadyExistingSETDataOverridesItButUpdatesEtag() + public void SETWITHETAGOnAlreadyExistingSETDataOverridesItButUpdatesEtag() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); IDatabase db = redis.GetDatabase(0); - RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); long etag = (long)res; ClassicAssert.AreEqual(1, etag); @@ -739,7 +625,7 @@ public void SETWithWITHETAGOnAlreadyExistingSETDataOverridesItButUpdatesEtag() ClassicAssert.IsTrue(updateRes[1].IsNull); // inplace update - res = db.Execute("SET", ["rizz", "meow", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "meow"]); etag = (long)res; ClassicAssert.AreEqual(3, etag); @@ -750,13 +636,13 @@ public void SETWithWITHETAGOnAlreadyExistingSETDataOverridesItButUpdatesEtag() ClassicAssert.IsTrue(updateRes[1].IsNull); // Copy update - res = db.Execute("SET", ["rizz", "oneofus", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "oneofus"]); etag = (long)res; ClassicAssert.AreEqual(5, etag); } [Test] - public void SETWithWITHETAGOnAlreadyExistingNonEtagDataOverridesItToInitialEtag() + public void SETWITHETAGOnAlreadyExistingNonEtagDataOverridesItToInitialEtag() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); IDatabase db = redis.GetDatabase(0); @@ -764,7 +650,7 @@ public void SETWithWITHETAGOnAlreadyExistingNonEtagDataOverridesItToInitialEtag( ClassicAssert.IsTrue(db.StringSet("rizz", "used")); // inplace update - RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); long etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, etag); @@ -773,7 +659,7 @@ public void SETWithWITHETAGOnAlreadyExistingNonEtagDataOverridesItToInitialEtag( ClassicAssert.IsTrue(db.StringSet("rizz", "my")); // Copy update - res = db.Execute("SET", ["rizz", "some", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "some"]); etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, etag); } @@ -801,11 +687,11 @@ public void SETOnAlreadyExistingNonEtagDataOverridesIt() ClassicAssert.IsTrue(db.StringSet("rizz", "used")); // inplace update - RedisResult res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + RedisResult res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); long etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, etag); - res = db.Execute("SET", ["rizz", "buzz", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "buzz"]); etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(2, etag); @@ -814,7 +700,7 @@ public void SETOnAlreadyExistingNonEtagDataOverridesIt() ClassicAssert.IsTrue(db.StringSet("rizz", "my")); // Copy update - res = db.Execute("SET", ["rizz", "some", "WITHETAG"]); + res = db.Execute("SETWITHETAG", ["rizz", "some"]); etag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, etag); } @@ -888,7 +774,7 @@ public void SingleEtagSetGet() var db = redis.GetDatabase(0); string origValue = "abcdefg"; - db.Execute("SET", ["mykey", origValue, "WITHETAG"]); + db.Execute("SETWITHETAG", ["mykey", origValue]); string retValue = db.StringGet("mykey"); @@ -902,7 +788,7 @@ public async Task SingleUnicodeEtagSetGetGarnetClient() db.Connect(); string origValue = "笑い男"; - await db.ExecuteForLongResultAsync("SET", ["mykey", origValue, "WITHETAG"]).ConfigureAwait(false); + await db.ExecuteForLongResultAsync("SETWITHETAG", ["mykey", origValue]).ConfigureAwait(false); string retValue = await db.StringGetAsync("mykey").ConfigureAwait(false); @@ -921,7 +807,7 @@ public async Task LargeEtagSetGet() for (int i = 0; i < length; i++) value[i] = (byte)((byte)'a' + ((byte)i % 26)); - RedisResult res = await db.ExecuteAsync("SET", ["mykey", value, "WITHETAG"]).ConfigureAwait(false); + RedisResult res = await db.ExecuteAsync("SETWITHETAG", ["mykey", value]).ConfigureAwait(false); long initalEtag = long.Parse(res.ToString()); ClassicAssert.AreEqual(1, initalEtag); @@ -940,7 +826,7 @@ public void SetExpiryForEtagSetData() string origValue = "abcdefghij"; // set with etag - long initalEtag = long.Parse(db.Execute("SET", ["mykey", origValue, "EX", 2, "WITHETAG"]).ToString()); + long initalEtag = long.Parse(db.Execute("SETWITHETAG", ["mykey", origValue, "EX", 2]).ToString()); ClassicAssert.AreEqual(1, initalEtag); string retValue = db.StringGet("mykey"); @@ -978,7 +864,7 @@ public void SetExpiryHighPrecisionForEtagSetDatat() var origValue = "abcdeghijklmno"; // set with etag - long initalEtag = long.Parse(db.Execute("SET", ["mykey", origValue, "PX", 1900, "WITHETAG"]).ToString()); + long initalEtag = long.Parse(db.Execute("SETWITHETAG", ["mykey", origValue, "PX", 1900]).ToString()); ClassicAssert.AreEqual(1, initalEtag); string retValue = db.StringGet("mykey"); @@ -993,155 +879,6 @@ public void SetExpiryHighPrecisionForEtagSetDatat() ClassicAssert.AreEqual(null, retValue); } - [Test] - public void SetExpiryIncrForEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - // Key storing integer - var nVal = -100000; - var strKey = "key1"; - db.Execute("SET", [strKey, nVal, "WITHETAG"]); - db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); - - string res1 = db.StringGet(strKey); - - long n = db.StringIncrement(strKey); - - // This should increase the ETAG internally so we have a check for that here - var checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(2, checkEtag); - - string res = db.StringGet(strKey); - long nRetVal = Convert.ToInt64(res); - ClassicAssert.AreEqual(n, nRetVal); - ClassicAssert.AreEqual(-99999, nRetVal); - - n = db.StringIncrement(strKey); - - // This should increase the ETAG internally so we have a check for that here - checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(3, checkEtag); - - nRetVal = Convert.ToInt64(db.StringGet(strKey)); - ClassicAssert.AreEqual(n, nRetVal); - ClassicAssert.AreEqual(-99998, nRetVal); - - var res69 = db.KeyTimeToLive(strKey); - - Thread.Sleep(5000); - - // Expired key, restart increment,after exp this is treated as new record - n = db.StringIncrement(strKey); - ClassicAssert.AreEqual(1, n); - - nRetVal = Convert.ToInt64(db.StringGet(strKey)); - ClassicAssert.AreEqual(1, nRetVal); - - var etagGet = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); - // Etag will show up as 0 since the previous one had expired - ClassicAssert.AreEqual("0", etagGet[0].ToString()); - ClassicAssert.AreEqual(1, Convert.ToInt64(etagGet[1])); - } - - [Test] - public void IncrDecrChangeDigitsWithExpiry() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var strKey = "key1"; - - db.Execute("SET", [strKey, 9, "WITHETAG"]); - - long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(1, checkEtag); - - db.KeyExpire(strKey, TimeSpan.FromSeconds(5)); - - long n = db.StringIncrement(strKey); - long nRetVal = Convert.ToInt64(db.StringGet(strKey)); - ClassicAssert.AreEqual(n, nRetVal); - ClassicAssert.AreEqual(10, nRetVal); - - checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(2, checkEtag); - - n = db.StringDecrement(strKey); - nRetVal = Convert.ToInt64(db.StringGet(strKey)); - ClassicAssert.AreEqual(n, nRetVal); - ClassicAssert.AreEqual(9, nRetVal); - - checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(3, checkEtag); - - Thread.Sleep(TimeSpan.FromSeconds(5)); - - var res = (string)db.StringGet(strKey); - ClassicAssert.IsNull(res); - } - - [Test] - public void StringSetOnAnExistingEtagDataOverrides() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var strKey = "mykey"; - db.Execute("SET", [strKey, 9, "WITHETAG"]); - - long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(1, checkEtag); - - // Unless the SET was called with WITHETAG a call to set will override the SET to a new - // value altogether, this will make it lose it's etag capability. This is a limitation for Etags - // because plain sets are upserts (blind updates), and currently we cannot increase the latency in - // the common path for set to check beyong Readonly address for the existence of a record with ETag. - // This means that sets are complete upserts and clients need to use setifmatch, or set with WITHETAG - // if they want each consequent set to maintain the key value pair's etag property. - ClassicAssert.IsTrue(db.StringSet(strKey, "ciaociao")); - - string retVal = db.StringGet(strKey).ToString(); - ClassicAssert.AreEqual("ciaociao", retVal); - - var res = (RedisResult[])db.Execute("GETWITHETAG", [strKey]); - ClassicAssert.AreEqual("0", res[0].ToString()); - ClassicAssert.AreEqual("ciaociao", res[1].ToString()); - } - - [Test] - public void StringSetOnAnExistingEtagDataUpdatesEtagIfEtagRetain() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var strKey = "mykey"; - db.Execute("SET", strKey, "9", "WITHETAG"); - - long checkEtag = (long)db.Execute("GETWITHETAG", [strKey])[0]; - ClassicAssert.AreEqual(1, checkEtag); - - // Unless you explicitly call SET with WITHETAG option you will lose the etag on the previous key-value pair - db.Execute("SET", [strKey, "ciaociao", "WITHETAG"]); - - string retVal = db.StringGet(strKey).ToString(); - ClassicAssert.AreEqual("ciaociao", retVal); - - var res = (RedisResult[])db.Execute("GETWITHETAG", strKey); - ClassicAssert.AreEqual(2, (long)res[0]); - - // on subsequent upserts we are still increasing the etag transparently - db.Execute("SET", [strKey, "ciaociaociao", "WITHETAG"]); - - retVal = db.StringGet(strKey).ToString(); - ClassicAssert.AreEqual("ciaociaociao", retVal); - - res = (RedisResult[])db.Execute("GETWITHETAG", strKey); - ClassicAssert.AreEqual(3, (long)res[0]); - ClassicAssert.AreEqual("ciaociaociao", res[1].ToString()); - } - [Test] public void LockTakeReleaseOnAValueInitiallySET() { @@ -1151,7 +888,7 @@ public void LockTakeReleaseOnAValueInitiallySET() string key = "lock-key"; string value = "lock-value"; - var initalEtag = long.Parse(db.Execute("SET", [key, value, "WITHETAG"]).ToString()); + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [key, value]).ToString()); ClassicAssert.AreEqual(1, initalEtag); var success = db.LockTake(key, value, TimeSpan.FromSeconds(100)); @@ -1190,16 +927,13 @@ public void SingleDecrForEtagSetData(string strKey, int nVal) var db = redis.GetDatabase(0); // Key storing integer - var initalEtag = long.Parse(db.Execute("SET", [strKey, nVal, "WITHETAG"]).ToString()); + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [strKey, nVal]).ToString()); ClassicAssert.AreEqual(1, initalEtag); long n = db.StringDecrement(strKey); ClassicAssert.AreEqual(nVal - 1, n); long nRetVal = Convert.ToInt64(db.StringGet(strKey)); ClassicAssert.AreEqual(n, nRetVal); - - long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(2, checkEtag); } [Test] @@ -1213,16 +947,13 @@ public void SingleDecrByForEtagSetData(long nVal, long nDecr) var db = redis.GetDatabase(0); // Key storing integer val var strKey = "key1"; - var initalEtag = long.Parse(db.Execute("SET", [strKey, nVal, "WITHETAG"]).ToString()); + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [strKey, nVal]).ToString()); ClassicAssert.AreEqual(1, initalEtag); long n = db.StringDecrement(strKey, nDecr); int nRetVal = Convert.ToInt32(db.StringGet(strKey)); ClassicAssert.AreEqual(n, nRetVal); - - long checkEtag = long.Parse(db.Execute("GETWITHETAG", [strKey])[0].ToString()); - ClassicAssert.AreEqual(2, checkEtag); } [Test] @@ -1240,7 +971,7 @@ public void SimpleIncrementInvalidValueForEtagSetdata(RespCommand cmd) { var key = $"key{i}"; var exception = false; - var initalEtag = long.Parse(db.Execute("SET", [key, values[i], "WITHETAG"]).ToString()); + var initalEtag = long.Parse(db.Execute("SETWITHETAG", [key, values[i]]).ToString()); ClassicAssert.AreEqual(1, initalEtag); try @@ -1282,19 +1013,19 @@ public void SimpleIncrementOverflowForEtagSetData(RespCommand cmd) switch (cmd) { case RespCommand.INCR: - _ = db.Execute("SET", [key, long.MaxValue.ToString(), "WITHETAG"]); + _ = db.Execute("SETWITHETAG", [key, long.MaxValue.ToString()]); _ = db.StringIncrement(key); break; case RespCommand.DECR: - _ = db.Execute("SET", [key, long.MinValue.ToString(), "WITHETAG"]); + _ = db.Execute("SETWITHETAG", [key, long.MinValue.ToString()]); _ = db.StringDecrement(key); break; case RespCommand.INCRBY: - _ = db.Execute("SET", [key, 0, "WITHETAG"]); + _ = db.Execute("SETWITHETAG", [key, 0]); _ = db.Execute("INCRBY", [key, ulong.MaxValue.ToString()]); break; case RespCommand.DECRBY: - _ = db.Execute("SET", [key, 0, "WITHETAG"]); + _ = db.Execute("SETWITHETAG", [key, 0]); _ = db.Execute("DECRBY", [key, ulong.MaxValue.ToString()]); break; } @@ -1324,7 +1055,7 @@ public void SimpleIncrementByFloatForEtagSetData(double initialValue, double inc using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); var key = "key1"; - db.Execute("SET", key, initialValue, "WITHETAG"); + db.Execute("SETWITHETAG", key, initialValue); var expectedResult = initialValue + incrByValue; @@ -1336,12 +1067,6 @@ public void SimpleIncrementByFloatForEtagSetData(double initialValue, double inc Assert.That(actualResult, Is.EqualTo(expectedResult).Within(1.0 / Math.Pow(10, 15))); Assert.That(actualResult, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); - - RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", key); - long etag = (long)res[0]; - double value = double.Parse(res[1].ToString(), CultureInfo.InvariantCulture); - Assert.That(value, Is.EqualTo(actualResultRaw).Within(1.0 / Math.Pow(10, 15))); - ClassicAssert.AreEqual(2, etag); } [Test] @@ -1353,7 +1078,7 @@ public void SingleDeleteForEtagSetData() // Key storing integer var nVal = 100; var strKey = "key1"; - db.Execute("SET", [strKey, nVal, "WITHETAG"]); + db.Execute("SETWITHETAG", [strKey, nVal]); db.KeyDelete(strKey); var retVal = Convert.ToBoolean(db.StringGet(strKey)); ClassicAssert.AreEqual(retVal, false); @@ -1373,7 +1098,7 @@ public void SingleDeleteWithObjectStoreDisabledForEtagSetData() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); - db.Execute("SET", [key, value, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, value]); var resp = (string)db.StringGet(key); ClassicAssert.AreEqual(resp, value); @@ -1405,7 +1130,7 @@ public void SingleDeleteWithObjectStoreDisable_LTMForEtagSetData() { data.Add(new Tuple(TestUtils.GetRandomString(keyLen), TestUtils.GetRandomString(valLen))); var pair = data.Last(); - db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); } @@ -1448,7 +1173,7 @@ public void MultiKeyDeleteForEtagSetData([Values] bool withoutObjectStore) { data.Add(new Tuple(TestUtils.GetRandomString(keyLen), TestUtils.GetRandomString(valLen))); var pair = data.Last(); - db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); } var keys = data.Select(x => (RedisKey)x.Item1).ToArray(); @@ -1483,7 +1208,7 @@ public void MultiKeyUnlinkForEtagSetData([Values] bool withoutObjectStore) { data.Add(new Tuple(TestUtils.GetRandomString(keyLen), TestUtils.GetRandomString(valLen))); var pair = data.Last(); - db.Execute("SET", [pair.Item1, pair.Item2, "WITHETAG"]); + db.Execute("SETWITHETAG", [pair.Item1, pair.Item2]); } var keys = data.Select(x => (object)x.Item1).ToArray(); @@ -1512,7 +1237,7 @@ public void SingleExistsForEtagSetData([Values] bool withoutObjectStore) var strKey = "key1"; ClassicAssert.IsFalse(db.KeyExists(strKey)); - db.Execute("SET", [strKey, nVal, "WITHETAG"]); + db.Execute("SETWITHETAG", [strKey, nVal]); bool fExists = db.KeyExists("key1", CommandFlags.None); ClassicAssert.AreEqual(fExists, true); @@ -1536,103 +1261,12 @@ public void MultipleExistsKeysAndObjectsAndEtagData() db.StringSet("foo", "bar"); - db.Execute("SET", ["rizz", "bar", "WITHETAG"]); + db.Execute("SETWITHETAG", ["rizz", "bar"]); var exists = db.KeyExists(["key", "listKey", "zset:test", "foo", "rizz"]); ClassicAssert.AreEqual(4, exists); } - #region RENAME - - - [Test] - public void RenameEtagTests() - { - // old key had etag => new key zero'd etag when made without withetag (new key did not exists) - // old key had etag => new key zero'd etag when made without withetag (new key exists without etag) - // old key had etag => new key has updated etag when made with withetag (new key exists withetag) - // old key not have etag => new key made with updated etag when made withetag (new key did exist withetag) - // old key had etag and, new key has initial etag when made with withetag (new key did not exists) - // old key not have etag and, new key made with initial etag when made withetag (new key did not exist) - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - IDatabase db = redis.GetDatabase(0); - - string origValue = "test1"; - string oldKey = "key1"; - string newKey = "key2"; - - // Scenario: old key had etag and => new key zero'd etag when made without withetag (new key did not exists) - - long etag = long.Parse(db.Execute("SET", [oldKey, origValue, "WITHETAG"]).ToString()); - ClassicAssert.AreEqual(1, etag); - - db.KeyRename(oldKey, newKey); - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 0, origValue)); - // old key has been deleted, and new key exists without etag at this point - - // Scenario: old key had etag => new key zero'd etag when made without withetag (new key exists without etag) - db.Execute("SET", oldKey, origValue, "WITHETAG"); - - db.KeyRename(oldKey, newKey); - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 0, origValue)); - db.KeyDelete(newKey); - - // Scenario: old key had etag => new key has updated etag when made with withetag (new key exists withetag) - // setup new key with updated etag - db.Execute("SET", newKey, origValue + "delta", "WITHETAG"); - db.Execute("SETIFMATCH", newKey, origValue, 1); // updates etag to 2 - // old key with etag - etag = long.Parse(db.Execute("SET", [oldKey, origValue, "WITHETAG"]).ToString()); - ClassicAssert.AreEqual(1, etag); - - db.Execute("RENAME", oldKey, newKey, "WITHETAG"); // should update etag to 3 - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 3, origValue)); - // at this point new key exists with etag, old key does not exist at all - - // Scenario: old key not have etag => new key made with updated etag when made withetag (new key did exist withetag) - db.Execute("SET", oldKey, origValue); - - db.Execute("RENAME", oldKey, newKey, "WITHETAG"); - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 4, origValue)); - db.KeyDelete(newKey); - - // Scenario: old key had etag => new key has initial etag when made with withetag (new key did not exists) - db.Execute("SET", oldKey, origValue, "WITHETAG"); - - db.Execute("RENAME", oldKey, newKey, "WITHETAG"); - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 1, origValue)); - db.KeyDelete(newKey); - - // Scenario: old key not have etag => new key made with initial etag when made withetag (new key did not exist) - db.Execute("SET", oldKey, origValue); - - db.Execute("RENAME", oldKey, newKey, "WITHETAG"); - - ClassicAssert.IsTrue(db.StringGet(oldKey).IsNull); - ClassicAssert.IsTrue(EtagAndValMatches(db, newKey, 1, origValue)); - db.KeyDelete(newKey); - } - - private bool EtagAndValMatches(IDatabase db, string key, long expectedEtag, string expectedValue) - { - var res = (RedisResult[])db.Execute("GETWITHETAG", key); - var responseEtag = long.Parse(res[0].ToString()); - var responseValue = res[1].ToString(); - return responseValue == expectedValue && responseEtag == expectedEtag; - } - - #endregion - [Test] public void PersistTTLTestForEtagSetData() { @@ -1646,7 +1280,7 @@ public void PersistTTLTestForEtagSetData() var ttl = db.Execute("TTL", key); ClassicAssert.AreEqual(-2, (long)ttl); - db.Execute("SET", [key, val, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, val]); ttl = db.Execute("TTL", key); ClassicAssert.AreEqual(-1, (long)ttl); @@ -1692,7 +1326,7 @@ public void PersistTestForEtagSetData() int expire = 100; var keyA = "keyA"; - db.Execute("SET", [keyA, keyA, "WITHETAG"]); + db.Execute("SETWITHETAG", [keyA, keyA]); var response = db.KeyPersist(keyA); ClassicAssert.IsFalse(response); @@ -1728,7 +1362,7 @@ public void KeyExpireStringTestForEtagSetData(string command) var db = redis.GetDatabase(0); var key = "keyA"; - db.Execute("SET", [key, key, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, key]); var value = db.StringGet(key); ClassicAssert.AreEqual(key, (string)value); @@ -1754,7 +1388,7 @@ public void KeyExpireOptionsTestForEtagSetData(string command) var key = "keyA"; object[] args = [key, 1000, ""]; - db.Execute("SET", [key, key, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, key]); args[2] = "XX";// XX -- Set expiry only when the key has an existing expiry bool resp = (bool)db.Execute($"{command}", args); @@ -1817,23 +1451,21 @@ public void MainObjectKeyForEtagSetData() const string key = "test:1"; - ClassicAssert.AreEqual(1, long.Parse(db.Execute("SET", key, "v1", "WITHETAG").ToString())); + ClassicAssert.AreEqual(1, long.Parse(db.Execute("SETWITHETAG", key, "v1").ToString())); - // Do SetAdd using the same key - ClassicAssert.IsTrue(db.SetAdd(key, "v2")); + // Do SetAdd using the same key, expected error + Assert.Throws(() => db.SetAdd(key, "v2"), + Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE)); - // Two keys "test:1" - this is expected as of now - // because Garnet has a separate main and object store + // One key "test:1" with a string value is expected var keys = server.Keys(db.Database, key).ToList(); - ClassicAssert.AreEqual(2, keys.Count); + ClassicAssert.AreEqual(1, keys.Count); ClassicAssert.AreEqual(key, (string)keys[0]); - ClassicAssert.AreEqual(key, (string)keys[1]); + var value = db.StringGet(key); + ClassicAssert.AreEqual("v1", (string)value); // do ListRightPush using the same key, expected error - var ex = Assert.Throws(() => db.ListRightPush(key, "v3")); - var expectedError = Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE); - ClassicAssert.IsNotNull(ex); - ClassicAssert.AreEqual(expectedError, ex.Message); + Assert.Throws(() => db.ListRightPush(key, "v3"), Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE)); } [Test] @@ -1848,7 +1480,7 @@ public void GetSliceTestForEtagSetData() var resp = (string)db.StringGetRange(key, 2, 10); ClassicAssert.AreEqual(string.Empty, resp); - ClassicAssert.AreEqual(1, long.Parse(db.Execute("SET", key, value, "WITHETAG").ToString())); + ClassicAssert.AreEqual(1, long.Parse(db.Execute("SETWITHETAG", key, value).ToString())); //0,0 resp = (string)db.StringGetRange(key, 0, 0); @@ -1953,89 +1585,6 @@ public void GetSliceTestForEtagSetData() resp = db.StringGetRange(key, negstart, negend); ClassicAssert.AreEqual(value.Substring(5, 3), resp); } - - [Test] - public void SetRangeTestForEtagSetData([Values] RevivificationMode revivificationModeUsedBySetupOnly) - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - string key = "setRangeKey"; - string value = "0123456789"; - string newValue = "ABCDE"; - - db.Execute("SET", key, value, "WITHETAG"); - - var resp = db.StringGet(key); - ClassicAssert.AreEqual("0123456789", resp.ToString()); - - // new key, length 10, offset 5 -> 15 ("\0\0\0\0\00123456789") - resp = db.StringSetRange(key, 5, value); - ClassicAssert.AreEqual("15", resp.ToString()); - resp = db.StringGet(key); - ClassicAssert.AreEqual("012340123456789", resp.ToString()); - - // should update the etag internally - var updatedEtagRes = db.Execute("GETWITHETAG", key); - ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); - - ClassicAssert.IsTrue(db.KeyDelete(key)); - - // new key, length 10, offset -1 -> RedisServerException ("ERR offset is out of range") - var ex = Assert.Throws(() => db.StringSetRange(key, -1, value)); - ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); - - // existing key, length 10, offset 0, value length 5 -> 10 ("ABCDE56789") - db.Execute("SET", key, value, "WITHETAG"); - - resp = db.StringSetRange(key, 0, newValue); - ClassicAssert.AreEqual("10", resp.ToString()); - resp = db.StringGet(key); - ClassicAssert.AreEqual("ABCDE56789", resp.ToString()); - - // should update the etag internally - updatedEtagRes = db.Execute("GETWITHETAG", key); - ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); - - ClassicAssert.IsTrue(db.KeyDelete(key)); - - // key, length 10, offset 5, value length 5 -> 10 ("01234ABCDE") - db.Execute("SET", key, value, "WITHETAG"); - - resp = db.StringSetRange(key, 5, newValue); - ClassicAssert.AreEqual("10", resp.ToString()); - - updatedEtagRes = db.Execute("GETWITHETAG", key); - ClassicAssert.AreEqual(2, long.Parse(updatedEtagRes[0].ToString())); - - resp = db.StringGet(key); - ClassicAssert.AreEqual("01234ABCDE", resp.ToString()); - ClassicAssert.IsTrue(db.KeyDelete(key)); - - // existing key, length 10, offset 10, value length 5 -> 15 ("0123456789ABCDE") - db.Execute("SET", [key, value, "WITHETAG"]); - resp = db.StringSetRange(key, 10, newValue); - ClassicAssert.AreEqual("15", resp.ToString()); - resp = db.StringGet(key); - ClassicAssert.AreEqual("0123456789ABCDE", resp.ToString()); - ClassicAssert.IsTrue(db.KeyDelete(key)); - - // existing key, length 10, offset 15, value length 5 -> 20 ("0123456789\0\0\0\0\0ABCDE") - db.Execute("SET", [key, value, "WITHETAG"]); - - resp = db.StringSetRange(key, 15, newValue); - ClassicAssert.AreEqual("20", resp.ToString()); - resp = db.StringGet(key); - ClassicAssert.AreEqual("0123456789\0\0\0\0\0ABCDE", resp.ToString()); - ClassicAssert.IsTrue(db.KeyDelete(key)); - - // existing key, length 10, offset -1, value length 5 -> RedisServerException ("ERR offset is out of range") - db.Execute("SET", [key, value, "WITHETAG"]); - - ex = Assert.Throws(() => db.StringSetRange(key, -1, newValue)); - ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_GENERIC_OFFSETOUTOFRANGE), ex.Message); - } - [Test] public void KeepTtlTestForDataInitiallySET() { @@ -2074,7 +1623,7 @@ public void StrlenTestOnEtagSetData() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); - db.Execute("SET", ["mykey", "foo bar", "WITHETAG"]); + db.Execute("SETWITHETAG", ["mykey", "foo bar"]); ClassicAssert.AreEqual(7, db.StringLength("mykey")); ClassicAssert.AreEqual(0, db.StringLength("nokey")); @@ -2096,7 +1645,7 @@ public void TTLTestMillisecondsForEtagSetData() var pttl = db.Execute("PTTL", key); ClassicAssert.AreEqual(-2, (long)pttl); - db.Execute("SET", [key, val, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, val]); pttl = db.Execute("PTTL", key); ClassicAssert.AreEqual(-1, (long)pttl); @@ -2133,7 +1682,7 @@ public void GetDelTestForEtagSetData() var val = "myKeyValue"; // Key Setup - db.Execute("SET", [key, val, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, val]); var retval = db.StringGet(key); ClassicAssert.AreEqual(val, retval.ToString()); @@ -2153,7 +1702,7 @@ public void GetDelTestForEtagSetData() key = "myKeyWithMetadata"; val = "myValueWithMetadata"; - db.Execute("SET", [key, val, "WITHETAG"]); + db.Execute("SETWITHETAG", [key, val]); db.KeyExpire(key, TimeSpan.FromSeconds(10000)); retval = db.StringGet(key); @@ -2167,239 +1716,6 @@ public void GetDelTestForEtagSetData() ClassicAssert.AreEqual(string.Empty, retval.ToString()); } - [Test] - public void AppendTestForEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var key = "myKey"; - var val = "myKeyValue"; - var val2 = "myKeyValue2"; - - db.Execute("SET", [key, val, "WITHETAG"]); - - var len = db.StringAppend(key, val2); - ClassicAssert.AreEqual(val.Length + val2.Length, len); - - var _val = db.StringGet(key); - ClassicAssert.AreEqual(val + val2, _val.ToString()); - - long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - - db.KeyDelete(key); - - // Test appending an empty string - db.Execute("SET", [key, val, "WITHETAG"]); - - var len1 = db.StringAppend(key, ""); - ClassicAssert.AreEqual(val.Length, len1); - - _val = db.StringGet(key); - ClassicAssert.AreEqual(val, _val.ToString()); - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - // we appended nothing so this remains 1 - ClassicAssert.AreEqual(1, etagToCheck); - - // Test appending to a non-existent key - var nonExistentKey = "nonExistentKey"; - var len2 = db.StringAppend(nonExistentKey, val2); - ClassicAssert.AreEqual(val2.Length, len2); - - _val = db.StringGet(nonExistentKey); - ClassicAssert.AreEqual(val2, _val.ToString()); - - db.KeyDelete(key); - - // Test appending to a key with a large value - var largeVal = new string('a', 1000000); - db.Execute("SET", [key, largeVal, "WITHETAG"]); - var len3 = db.StringAppend(key, val2); - ClassicAssert.AreEqual(largeVal.Length + val2.Length, len3); - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - - // Test appending to a key with metadata - var keyWithMetadata = "keyWithMetadata"; - db.Execute("SET", [keyWithMetadata, val, "WITHETAG"]); - db.KeyExpire(keyWithMetadata, TimeSpan.FromSeconds(10000)); - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); - ClassicAssert.AreEqual(1, etagToCheck); - - var len4 = db.StringAppend(keyWithMetadata, val2); - ClassicAssert.AreEqual(val.Length + val2.Length, len4); - - _val = db.StringGet(keyWithMetadata); - ClassicAssert.AreEqual(val + val2, _val.ToString()); - - var time = db.KeyTimeToLive(keyWithMetadata); - ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [keyWithMetadata]))[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - } - - [Test] - public void SetBitOperationsOnEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - string key = "miki"; - // 64 BIT BITMAP - Byte[] initialBitmap = new byte[8]; - string bitMapAsStr = Encoding.UTF8.GetString(initialBitmap); ; - - db.Execute("SET", [key, bitMapAsStr, "WITHETAG"]); - - long setbits = db.StringBitCount(key); - ClassicAssert.AreEqual(0, setbits); - - long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(1, etagToCheck); - - // set all 64 bits one by one - long expectedBitCount = 0; - long expectedEtag = 1; - for (int i = 0; i < 64; i++) - { - // SET the ith bit in the bitmap - bool originalValAtBit = db.StringSetBit(key, i, true); - ClassicAssert.IsFalse(originalValAtBit); - - expectedBitCount++; - expectedEtag++; - - bool currentBitVal = db.StringGetBit(key, i); - ClassicAssert.IsTrue(currentBitVal); - - setbits = db.StringBitCount(key); - ClassicAssert.AreEqual(expectedBitCount, setbits); - - // Use BitPosition to find the first set bit - long firstSetBitPosition = db.StringBitPosition(key, true); - ClassicAssert.AreEqual(0, firstSetBitPosition); // As we are setting bits in order, first set bit should be 0 - - // find the first unset bit - long firstUnsetBitPos = db.StringBitPosition(key, false); - long firstUnsetBitPosExpected = i == 63 ? -1 : i + 1; - ClassicAssert.AreEqual(firstUnsetBitPosExpected, firstUnsetBitPos); // As we are setting bits in order, first unset bit should be 1 ahead - - - // with each bit set that we do, we are increasing the etag as well by 1 - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(expectedEtag, etagToCheck); - } - - // unset all 64 bits one by one in reverse order - for (int i = 63; i > -1; i--) - { - bool originalValAtBit = db.StringSetBit(key, i, false); - ClassicAssert.IsTrue(originalValAtBit); - - expectedEtag++; - expectedBitCount--; - - bool currentBitVal = db.StringGetBit(key, i); - ClassicAssert.IsFalse(currentBitVal); - - setbits = db.StringBitCount(key); - ClassicAssert.AreEqual(expectedBitCount, setbits); - - // find the first set bit - long firstSetBit = db.StringBitPosition(key, true); - long expectedSetBit = i == 0 ? -1 : 0; - ClassicAssert.AreEqual(expectedSetBit, firstSetBit); - - // Use BitPosition to find the first unset bit - long firstUnsetBitPosition = db.StringBitPosition(key, false); - ClassicAssert.AreEqual(i, firstUnsetBitPosition); // After unsetting, the first unset bit should be i - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(expectedEtag, etagToCheck); - } - } - - [Test] - public void BitFieldSetGetOnEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var key = "mewo"; - - // Arrange - Set an 8-bit unsigned value at offset 0 - db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte - - // Act - Set value to 127 (binary: 01111111) - db.Execute("BITFIELD", key, "SET", "u8", "0", "127"); - - long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - - // Get value back - var getResult = (RedisResult[])db.Execute("BITFIELD", key, "GET", "u8", "0"); - - // Assert - ClassicAssert.AreEqual(127, (long)getResult[0]); // Ensure the value set was retrieved correctly - } - - [Test] - public void BitFieldIncrementWithWrapOverflowOnEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var key = "mewo"; - - // Arrange - Set an 8-bit unsigned value at offset 0 - db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte - - // Act - Set initial value to 255 and try to increment by 1 - db.Execute("BITFIELD", key, "SET", "u8", "0", "255"); - long etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - - var incrResult = db.Execute("BITFIELD", key, "INCRBY", "u8", "0", "1"); - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(3, etagToCheck); - - // Assert - ClassicAssert.AreEqual(0, (long)incrResult); // Should wrap around and return 0 - } - - [Test] - public void BitFieldIncrementWithSaturateOverflowOnEtagSetData() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - var key = "mewo"; - - // Arrange - Set an 8-bit unsigned value at offset 0 - db.Execute("SET", [key, Encoding.UTF8.GetString(new byte[1]), "WITHETAG"]); // Initialize key with an empty byte - - // Act - Set initial value to 250 and try to increment by 10 with saturate overflow - var bitfieldRes = db.Execute("BITFIELD", key, "SET", "u8", "0", "250"); - ClassicAssert.AreEqual(0, (long)bitfieldRes); - - var result = (RedisResult[])db.Execute("GETWITHETAG", [key]); - long etagToCheck = long.Parse(result[0].ToString()); - ClassicAssert.AreEqual(2, etagToCheck); - - var incrResult = db.Execute("BITFIELD", key, "OVERFLOW", "SAT", "INCRBY", "u8", "0", "10"); - - etagToCheck = long.Parse(((RedisResult[])db.Execute("GETWITHETAG", [key]))[0].ToString()); - ClassicAssert.AreEqual(3, etagToCheck); - - // Assert - ClassicAssert.AreEqual(255, (long)incrResult); // Should saturate at the max value of 255 for u8 - } - [Test] public void HyperLogLogCommandsShouldReturnWrongTypeErrorForEtagSetData() { @@ -2409,51 +1725,19 @@ public void HyperLogLogCommandsShouldReturnWrongTypeErrorForEtagSetData() var key = "mewo"; var key2 = "dude"; - db.Execute("SET", [key, "mars", "WITHETAG"]); - db.Execute("SET", [key2, "marsrover", "WITHETAG"]); - - RedisServerException ex = Assert.Throws(() => db.Execute("PFADD", [key, "woohoo"])); - - ClassicAssert.IsNotNull(ex); - ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); - - ex = Assert.Throws(() => db.Execute("PFMERGE", [key, key2])); - - ClassicAssert.IsNotNull(ex); - ClassicAssert.AreEqual(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL), ex.Message); - } - - [Test] - public void SetWithWITHETAGOnANewUpsertWillCreateKeyValueWithoutEtag() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - - string key = "mickey"; - string val = "mouse"; - - // a new upsert on a non-existing key will retain the "nil" etag - db.Execute("SET", [key, val, "WITHETAG"]).ToString(); - - RedisResult[] res = (RedisResult[])db.Execute("GETWITHETAG", [key]); - RedisResult etag = res[0]; - string value = res[1].ToString(); - - ClassicAssert.AreEqual("1", etag.ToString()); - ClassicAssert.AreEqual(val, value); + _ = db.Execute("SETWITHETAG", [key, "mars"]); + _ = db.Execute("SETWITHETAG", [key2, "marsrover"]); - string newval = "clubhouse"; + // TODO: This is RedisServerException in the InPlaceUpdater call, but GetRMWModifiedFieldInfo currently throws RedisConnectionException. + // This can be different in CIs vs. locally. + Assert.That(() => db.Execute("PFADD", [key, "woohoo"]), + Throws.TypeOf().With.Message.EndsWith(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL)) + .Or.TypeOf()); - // a new upsert on an existing key will reset the etag on the key - db.Execute("SET", [key, newval]).ToString(); - res = (RedisResult[])db.Execute("GETWITHETAG", [key]); - etag = res[0]; - value = res[1].ToString(); - - ClassicAssert.AreEqual("0", etag.ToString()); - ClassicAssert.AreEqual(newval, value); + Assert.That(() => db.Execute("PFMERGE", [key, key2]), + Throws.TypeOf().With.Message.EndsWith(Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE_HLL)) + .Or.TypeOf()); } - #endregion } } \ No newline at end of file diff --git a/test/Garnet.test/RespGetLowMemoryTests.cs b/test/standalone/Garnet.test/RespGetLowMemoryTests.cs similarity index 97% rename from test/Garnet.test/RespGetLowMemoryTests.cs rename to test/standalone/Garnet.test/RespGetLowMemoryTests.cs index fccfa8c0cf3..9689b9c89c2 100644 --- a/test/Garnet.test/RespGetLowMemoryTests.cs +++ b/test/standalone/Garnet.test/RespGetLowMemoryTests.cs @@ -1,20 +1,18 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespGetLowMemoryTests : AllureTestBase + public class RespGetLowMemoryTests : TestBase { GarnetServer server; Random r; diff --git a/test/Garnet.test/RespInfoTests.cs b/test/standalone/Garnet.test/RespInfoTests.cs similarity index 95% rename from test/Garnet.test/RespInfoTests.cs rename to test/standalone/Garnet.test/RespInfoTests.cs index 944adb2057e..20820676fa5 100644 --- a/test/Garnet.test/RespInfoTests.cs +++ b/test/standalone/Garnet.test/RespInfoTests.cs @@ -6,16 +6,14 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespInfoTests : AllureTestBase + public class RespInfoTests : TestBase { GarnetServer server; @@ -190,11 +188,8 @@ public async Task InfoHlogScanTest() // hydrate var startingHA = server.Provider.StoreWrapper.store.Log.HeadAddress; - var startingHAObj = server.Provider.StoreWrapper.objectStore.Log.HeadAddress; - await Task.WhenAll( - HydrateStore(db, (db, key, value) => db.StringSetAsync(key, value), () => startingHA == server.Provider.StoreWrapper.store.Log.HeadAddress), - HydrateStore(db, (db, key, value) => db.SetAddAsync(key, value), () => startingHAObj == server.Provider.StoreWrapper.objectStore.Log.HeadAddress) - ).ConfigureAwait(false); + await HydrateStore(db, (db, key, value) => db.StringSetAsync(key, value), + () => startingHA == server.Provider.StoreWrapper.store.Log.HeadAddress).ConfigureAwait(false); // Wait for the immediate expirations to kick in await Task.Delay(500).ConfigureAwait(false); diff --git a/test/Garnet.test/RespLowMemoryTests.cs b/test/standalone/Garnet.test/RespLowMemoryTests.cs similarity index 75% rename from test/Garnet.test/RespLowMemoryTests.cs rename to test/standalone/Garnet.test/RespLowMemoryTests.cs index 62f2d950c60..cbd54e93a0d 100644 --- a/test/Garnet.test/RespLowMemoryTests.cs +++ b/test/standalone/Garnet.test/RespLowMemoryTests.cs @@ -1,17 +1,16 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; +using Tsavorite.core; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespLowMemoryTests : AllureTestBase + public class RespLowMemoryTests : TestBase { GarnetServer server; @@ -53,18 +52,17 @@ public void PersistCopyUpdateTest() var server = redis.GetServer(TestUtils.EndPoint); var info = TestUtils.GetStoreAddressInfo(server); - // Start at tail address of 64 - ClassicAssert.AreEqual(64, info.TailAddress); + // Start at tail address of PageHeader.Size (64) + ClassicAssert.AreEqual(PageHeader.Size, info.TailAddress); var expire = 100; var key0 = $"key{0:00000}"; _ = db.StringSet(key0, key0, TimeSpan.FromSeconds(expire)); - // Record size for key0 is 8 bytes header + 16 bytes key + 16 bytes value + 8 bytes expiry = 48 bytes - // so the new tail address should be 64 + 48 = 112 - // That is, key0 is located at [64, 112) + // Record size for key0 is RecordInfo.Size (8) + MinLengthMetadataBytes (5) + 2 * 8 bytes (key and value) + 8 bytes expiry = 13 + 24 = 37 bytes rounded up to 40 + // so the new tail address should be 64 + 40 = 104 (that is, the record for key0 is located at [64, 104)). info = TestUtils.GetStoreAddressInfo(server); - ClassicAssert.AreEqual(112, info.TailAddress); + ClassicAssert.AreEqual(104, info.TailAddress); // Make the record read-only by adding more records MakeReadOnly(info.TailAddress, server, db); @@ -73,16 +71,16 @@ public void PersistCopyUpdateTest() var previousTail = info.TailAddress; // The first record inserted (key0) is now read-only - ClassicAssert.IsTrue(info.ReadOnlyAddress >= 112); + ClassicAssert.IsTrue(info.ReadOnlyAddress >= 104); // Persist the key, which should cause RMW to CopyUpdate to tail var response = db.KeyPersist(key0); ClassicAssert.IsTrue(response); - // Now key0 is only 40 bytes, as we are removing the expiration - // That is, key0 is now moved to [previousTail, previousTail + 40) + // Now key0 is only 32 bytes, as we are removing the expiration + // That is, key0 is now moved to [previousTail, previousTail + 32) info = TestUtils.GetStoreAddressInfo(server); - ClassicAssert.AreEqual(previousTail + 40, info.TailAddress); + ClassicAssert.AreEqual(previousTail + 32, info.TailAddress); // Verify that key0 exists with correct value ClassicAssert.AreEqual(key0, (string)db.StringGet(key0)); diff --git a/test/Garnet.test/RespMetricsTest.cs b/test/standalone/Garnet.test/RespMetricsTest.cs similarity index 95% rename from test/Garnet.test/RespMetricsTest.cs rename to test/standalone/Garnet.test/RespMetricsTest.cs index 1ab26aa6ece..457075a4723 100644 --- a/test/Garnet.test/RespMetricsTest.cs +++ b/test/standalone/Garnet.test/RespMetricsTest.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Linq; using System.Threading; -using Allure.NUnit; using Garnet.common; using Microsoft.Extensions.Logging; using NUnit.Framework; @@ -13,9 +12,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespMetricsTest : AllureTestBase + public class RespMetricsTest : TestBase { GarnetServer server; ILoggerFactory loggerFactory; @@ -33,7 +31,7 @@ public void Setup() server = null; r = new Random(674386); TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - loggerFactory = TestUtils.CreateLoggerFactoryInstance(TestContext.Progress, LogLevel.Error); + (loggerFactory, _) = TestUtils.CreateLoggerFactoryInstance(TestContext.Progress, LogLevel.Error); } [TearDown] diff --git a/test/Garnet.test/RespPubSubTests.cs b/test/standalone/Garnet.test/RespPubSubTests.cs similarity index 98% rename from test/Garnet.test/RespPubSubTests.cs rename to test/standalone/Garnet.test/RespPubSubTests.cs index ea65051fe94..d8248f22af6 100644 --- a/test/Garnet.test/RespPubSubTests.cs +++ b/test/standalone/Garnet.test/RespPubSubTests.cs @@ -1,20 +1,18 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Linq; using System.Security.Cryptography; using System.Threading; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - class RespPubSubTests : AllureTestBase + class RespPubSubTests : TestBase { GarnetServer server; diff --git a/test/Garnet.test/RespScanCommandsTests.cs b/test/standalone/Garnet.test/RespScanCommandsTests.cs similarity index 87% rename from test/Garnet.test/RespScanCommandsTests.cs rename to test/standalone/Garnet.test/RespScanCommandsTests.cs index 73fe60e9ab8..f319d7bd8fc 100644 --- a/test/Garnet.test/RespScanCommandsTests.cs +++ b/test/standalone/Garnet.test/RespScanCommandsTests.cs @@ -1,10 +1,9 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. using System; using System.Collections.Generic; using System.Linq; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; @@ -12,9 +11,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespScanCommandsTests : AllureTestBase + public class RespScanCommandsTests : TestBase { GarnetServer server; private IReadOnlyDictionary respCustomCommandsInfo; @@ -94,10 +92,10 @@ public void SeKeysCursorTest() // set 10_000 strings const int KeyCount = 10_000; - for (int i = 0; i < KeyCount; ++i) + for (int i = 0; i < KeyCount; i++) db.StringSet($"try:{i}", i); - // get and count keys using SE Redis, using the default pageSize of 250 + // get and count keys using SE Redis, using the default pageSizeStr of 250 var server = redis.GetServers()[db.Database]; var keyCount1 = server.Keys().ToArray().Length; ClassicAssert.AreEqual(KeyCount, keyCount1, "IServer.Keys()"); @@ -108,6 +106,66 @@ public void SeKeysCursorTest() ClassicAssert.AreEqual(KeyCount, keyCount2, "KEYS *"); } + /// + /// Regression test for the lookup-based KEYS conversion (Task 2 of the "tempKv elimination" + /// change). Validates rule S2: KEYS uses IterateLookup with maxAddress = untilAddress so that + /// each live key is emitted exactly once even under concurrent RCUs (in-place updates) that + /// might otherwise either suppress an in-range record or surface duplicates. + /// + [Test] + public void SeKeysNoDuplicatesUnderRcu() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + const int KeyCount = 2_000; + for (int i = 0; i < KeyCount; i++) + db.StringSet($"rcu:{i}", "v0"); + + using var stop = new System.Threading.CancellationTokenSource(); + + // Background writer continuously SETs keys to new values, generating RCU traffic. + var writer = System.Threading.Tasks.Task.Run(() => + { + using var rmwRedis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var rmwDb = rmwRedis.GetDatabase(0); + var rng = new Random(42); + long iter = 0; + while (!stop.Token.IsCancellationRequested) + { + var idx = rng.Next(KeyCount); + rmwDb.StringSet($"rcu:{idx}", $"v{iter++}"); + } + }); + + try + { + // Issue several KEYS calls concurrent with the writer. Each call must return + // exactly KeyCount entries with no duplicates. + for (int round = 0; round < 5; round++) + { + var res = (RedisValue[])db.Execute("KEYS", "rcu:*"); + ClassicAssert.IsNotNull(res); + ClassicAssert.AreEqual(KeyCount, res.Length, $"KEYS round {round}: expected {KeyCount} entries, got {res.Length}"); + + var unique = new HashSet(); + foreach (var k in res) + ClassicAssert.IsTrue(unique.Add(k.ToString()), $"KEYS round {round}: duplicate key returned: {k}"); + } + } + finally + { + stop.Cancel(); + // Ensure the background writer stops within a bounded window so it cannot leak + // a connection into subsequent tests, and surface any exception it threw so it + // doesn't get silently swallowed. + var stopped = writer.Wait(TimeSpan.FromSeconds(30)); + ClassicAssert.IsTrue(stopped, "Background writer did not stop within 30s after cancellation"); + if (writer.IsFaulted) + throw writer.Exception!.Flatten(); + } + } + [Test] public void CanDoMemoryUsage() { @@ -129,25 +187,25 @@ public void CanDoMemoryUsage() } var r = db.Execute("MEMORY", ["USAGE", "keyOne"]); - ClassicAssert.AreEqual("40", r.ToString()); + ClassicAssert.AreEqual("32", r.ToString()); r = db.Execute("MEMORY", ["USAGE", "myss"]); - ClassicAssert.AreEqual("344", r.ToString()); + ClassicAssert.AreEqual("376", r.ToString()); r = db.Execute("MEMORY", ["USAGE", "mylist"]); - ClassicAssert.AreEqual("176", r.ToString()); + ClassicAssert.AreEqual("208", r.ToString()); r = db.Execute("MEMORY", ["USAGE", "myset"]); - ClassicAssert.AreEqual("200", r.ToString()); + ClassicAssert.AreEqual("232", r.ToString()); r = db.Execute("MEMORY", ["USAGE", "myhash"]); - ClassicAssert.AreEqual("264", r.ToString()); + ClassicAssert.AreEqual("296", r.ToString()); r = db.Execute("MEMORY", ["USAGE", "foo"]); ClassicAssert.IsTrue(r.IsNull); r = db.Execute("MEMORY", ["USAGE", "hllKey"]); - ClassicAssert.AreEqual("304", r.ToString()); + ClassicAssert.AreEqual("296", r.ToString()); } @@ -346,7 +404,7 @@ public void CanUseScanAllKeys() recordsReturned += keysMatch.Length; } while (cursor != 0); - ClassicAssert.IsTrue(recordsReturned == nKeys); + ClassicAssert.AreEqual(nKeys, recordsReturned); } [Test] diff --git a/test/Garnet.test/RespSlowLogTests.cs b/test/standalone/Garnet.test/RespSlowLogTests.cs similarity index 96% rename from test/Garnet.test/RespSlowLogTests.cs rename to test/standalone/Garnet.test/RespSlowLogTests.cs index 40bd8935b2c..4cbec5645a1 100644 --- a/test/Garnet.test/RespSlowLogTests.cs +++ b/test/standalone/Garnet.test/RespSlowLogTests.cs @@ -1,16 +1,14 @@ -// Copyright (c) Microsoft Corporation. +// Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespSlowLogTests : AllureTestBase + public class RespSlowLogTests : TestBase { GarnetServer server; int slowLogThreshold = 3_000_000; diff --git a/test/Garnet.test/RespTests.cs b/test/standalone/Garnet.test/RespTests.cs similarity index 96% rename from test/Garnet.test/RespTests.cs rename to test/standalone/Garnet.test/RespTests.cs index c9f9934d02d..81113ea78da 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/standalone/Garnet.test/RespTests.cs @@ -10,7 +10,6 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.client; using Garnet.common; using Garnet.server; @@ -20,9 +19,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespTests : AllureTestBase + public class RespTests : TestBase { GarnetServer server; Random r; @@ -41,6 +39,7 @@ public void Setup() public void TearDown() { server.Dispose(); + TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); TestUtils.OnTearDown(); } @@ -1399,8 +1398,6 @@ public void SimpleIncrementByFloat(double initialValue, double incrByValue) } [Test] - [TestCase(double.MinValue, double.MinValue)] - [TestCase(double.MaxValue, double.MaxValue)] [TestCase("abc", 10)] [TestCase(10, "xyz")] [TestCase(10, "inf")] @@ -1411,17 +1408,32 @@ public void SimpleIncrementByFloatWithInvalidFloat(object initialValue, object i var db = redis.GetDatabase(0); var key = "key1"; if (initialValue is double) - { db.StringSet(key, (double)initialValue); - } else if (initialValue is string) - { db.StringSet(key, (string)initialValue); - } Assert.Throws(() => db.Execute("INCRBYFLOAT", key, incrByValue)); } + [Test] + [TestCase(double.MinValue, double.MinValue)] + [TestCase(double.MaxValue, double.MaxValue)] + public void SimpleIncrementByFloatWithOutOfRangeFloat(object initialValue, object incrByValue) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + if (initialValue is double) + db.StringSet(key, (double)initialValue); + else if (initialValue is string) + db.StringSet(key, (string)initialValue); + + // TODO: This is RedisServerException in the InPlaceUpdater call, but GetRMWModifiedFieldInfo currently throws RedisConnectionException. + // This can be different in CIs vs. locally. + Assert.That(() => db.Execute("INCRBYFLOAT", key, incrByValue), + Throws.TypeOf().Or.TypeOf()); + } + [Test] public void SingleDelete() { @@ -1517,7 +1529,7 @@ public void GarnetObjectStoreDisabledError() var mykey = "mykey"; for (var i = 0; i < iter; i++) { - var exception = Assert.Throws(() => _ = db.ListLength(mykey)); + var exception = Assert.Throws(() => _ = db.ListLength(mykey)); ClassicAssert.AreEqual("ERR Garnet Exception: Object store is disabled", exception.Message); } @@ -1728,7 +1740,7 @@ public void MultipleExistsKeysAndObjects() #region ExpireTime [Test] - public void ExpiretimeWithStingValue() + public void ExpiretimeWithStringValue() { using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var db = redis.GetDatabase(0); @@ -2265,8 +2277,8 @@ public void SingleRenameNXWithEtagSetOldAndNewKey() var key = "key1"; var newKey = "key2"; - db.Execute("SET", key, origValue, "WITHETAG"); - db.Execute("SET", newKey, "foo", "WITHETAG"); + db.Execute("SETWITHETAG", key, origValue); + db.Execute("SETWITHETAG", newKey, "foo"); var result = db.KeyRename(key, newKey, When.NotExists); ClassicAssert.IsFalse(result); @@ -2281,7 +2293,7 @@ public void SingleRenameNXWithEtagSetOldKey() var key = "key1"; var newKey = "key2"; - db.Execute("SET", key, origValue, "WITHETAG"); + db.Execute("SETWITHETAG", key, origValue); var result = db.KeyRename(key, newKey, When.NotExists); ClassicAssert.IsTrue(result); @@ -2291,11 +2303,6 @@ public void SingleRenameNXWithEtagSetOldKey() var oldKeyRes = db.StringGet(key); ClassicAssert.IsTrue(oldKeyRes.IsNull); - - // Since the original key was set with etag, the new key should have an etag attached to it - var etagRes = (RedisResult[])db.Execute("GETWITHETAG", newKey); - ClassicAssert.AreEqual(0, (long)etagRes[0]); - ClassicAssert.AreEqual(origValue, etagRes[1].ToString()); } #endregion @@ -2583,7 +2590,9 @@ public void KeyExpireStringTest(string command) ClassicAssert.AreEqual(key, (string)value); if (command.Equals("EXPIRE")) - db.KeyExpire(key, TimeSpan.FromSeconds(1)); + { + var res = db.KeyExpire(key, TimeSpan.FromSeconds(1)); + } else db.Execute(command, [key, 1000]); @@ -2860,7 +2869,7 @@ public void KeyExpireAtWithNxOptionAndKeyHasNoExpire(string command, bool isObje var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); var actualResult = (int)db.Execute(command, key, expireTimeUnix, "nX"); - ClassicAssert.AreEqual(actualResult, 1); + ClassicAssert.AreEqual(1, actualResult); var actualTtl = db.KeyTimeToLive(key); ClassicAssert.IsTrue(actualTtl.HasValue); @@ -3416,21 +3425,36 @@ public void MainObjectKey() // Do StringSet ClassicAssert.IsTrue(db.StringSet(key, "v1")); - // Do SetAdd using the same key - ClassicAssert.IsTrue(db.SetAdd(key, "v2")); + // Do SetAdd using the same key, expected error + Assert.Throws(() => db.SetAdd(key, "v2"), + Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE)); - // Two keys "test:1" - this is expected as of now - // because Garnet has a separate main and object store + // One key "test:1" with a string value is expected var keys = server.Keys(db.Database, key).ToList(); - ClassicAssert.AreEqual(2, keys.Count); + ClassicAssert.AreEqual(1, keys.Count); ClassicAssert.AreEqual(key, (string)keys[0]); - ClassicAssert.AreEqual(key, (string)keys[1]); + var value = db.StringGet(key); + ClassicAssert.AreEqual("v1", (string)value); // do ListRightPush using the same key, expected error - var ex = Assert.Throws(() => db.ListRightPush(key, "v3")); - var expectedError = Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE); - ClassicAssert.IsNotNull(ex); - ClassicAssert.AreEqual(expectedError, ex.Message); + Assert.Throws(() => db.ListRightPush(key, "v3"), Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE)); + + // Delete the key + ClassicAssert.IsTrue(db.KeyDelete(key)); + + // Do SetAdd using the same key + ClassicAssert.IsTrue(db.SetAdd(key, "v2")); + + // Do StringIncrement using the same key, expected error + //Assert.Throws(() => db.StringIncrement(key), Encoding.ASCII.GetString(CmdStrings.RESP_ERR_WRONG_TYPE)); + + // One key "test:1" with a set value is expected + keys = server.Keys(db.Database, key).ToList(); + ClassicAssert.AreEqual(1, keys.Count); + ClassicAssert.AreEqual(key, (string)keys[0]); + var members = db.SetMembers(key); + ClassicAssert.AreEqual(1, members.Length); + ClassicAssert.AreEqual("v2", (string)members[0]); } [Test] @@ -3780,7 +3804,7 @@ public void AppendTest() var val = "myKeyValue"; var val2 = "myKeyValue2"; - db.StringSet(key, val); + _ = db.StringSet(key, val); var len = db.StringAppend(key, val2); ClassicAssert.AreEqual(val.Length + val2.Length, len); @@ -3788,7 +3812,7 @@ public void AppendTest() ClassicAssert.AreEqual(val + val2, _val.ToString()); // Test appending an empty string - db.StringSet(key, val); + _ = db.StringSet(key, val); var len1 = db.StringAppend(key, ""); ClassicAssert.AreEqual(val.Length, len1); @@ -3802,23 +3826,50 @@ public void AppendTest() _val = db.StringGet(nonExistentKey); ClassicAssert.AreEqual(val2, _val.ToString()); + } + + [Test] + public void AppendLargeStringValueTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key1 = "myKey1"; + var key2 = "myKey2"; + var val2 = "myKeyValue2"; - // Test appending to a key with a large value var largeVal = new string('a', 1000000); - db.StringSet(key, largeVal); - var len3 = db.StringAppend(key, val2); - ClassicAssert.AreEqual(largeVal.Length + val2.Length, len3); - // Test appending to a key with metadata - var keyWithMetadata = "keyWithMetadata"; - db.StringSet(keyWithMetadata, val, TimeSpan.FromSeconds(10000)); - var len4 = db.StringAppend(keyWithMetadata, val2); - ClassicAssert.AreEqual(val.Length + val2.Length, len4); + // Test appending to a key with a large value + _ = db.StringSet(key1, largeVal); + var len = db.StringAppend(key1, val2); + ClassicAssert.AreEqual(largeVal.Length + val2.Length, len); + + // Test appending a large value to a key + _ = db.StringSet(key2, val2); + var len2 = db.StringAppend(key2, largeVal); + ClassicAssert.AreEqual(largeVal.Length + val2.Length, len); + } + + [Test] + public void AppendWithExpirationTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key = "keyWithExpiration"; + var val = "myKeyValue"; + var val2 = "myKeyValue2"; - _val = db.StringGet(keyWithMetadata); + // Test appending to a key with expiration + _ = db.StringSet(key, val, TimeSpan.FromSeconds(10000)); + var len = db.StringAppend(key, val2); + ClassicAssert.AreEqual(val.Length + val2.Length, len); + + var _val = db.StringGet(key); ClassicAssert.AreEqual(val + val2, _val.ToString()); - var time = db.KeyTimeToLive(keyWithMetadata); + var time = db.KeyTimeToLive(key); ClassicAssert.IsTrue(time.Value.TotalSeconds > 0); } @@ -5183,5 +5234,86 @@ public void ClientUnblockInvalidModeTest() Assert.Throws(() => mainDB.Execute("CLIENT", "UNBLOCK", 123, "INVALID")); } + + [Test] + public void ExcessiveArgumentCountRejected() + { + // Send a RESP array header with an argument count exceeding the maximum allowed, + // followed by a command name so parsing proceeds far enough to hit the guard. + // The server should reject the command with a protocol error and close the connection. + using var socket = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + socket.ReceiveTimeout = 5000; + socket.Connect(System.Net.IPAddress.Loopback, TestUtils.TestPort); + + // *2000000\r\n$3\r\nFOO\r\n — array with 2M elements, command name "FOO" + // The command will be identified as unknown but count (1999999) exceeds MaxParams (1,048,576) + var payload = Encoding.ASCII.GetBytes("*2000000\r\n$3\r\nFOO\r\n"); + socket.Send(payload); + + // Read the response - server should send an error and close the connection + var buffer = new byte[4096]; + var totalRead = 0; + try + { + while (totalRead < buffer.Length) + { + var bytesRead = socket.Receive(buffer, totalRead, buffer.Length - totalRead, System.Net.Sockets.SocketFlags.None); + if (bytesRead == 0) break; + totalRead += bytesRead; + } + } + catch (System.Net.Sockets.SocketException) + { + // Connection may be reset by server - this is expected + } + + var response = Encoding.ASCII.GetString(buffer, 0, totalRead); + var expectedResponse = + "-ERR unknown command\r\n" + + "-ERR Protocol Error: RESP array argument count '1999999' exceeds maximum allowed count of '1048576'.\r\n"; + ClassicAssert.AreEqual(expectedResponse, response); + + // Same test with a known command (PING) — should still be rejected due to excessive count + using var socket2 = new System.Net.Sockets.Socket( + System.Net.Sockets.AddressFamily.InterNetwork, + System.Net.Sockets.SocketType.Stream, + System.Net.Sockets.ProtocolType.Tcp); + socket2.ReceiveTimeout = 5000; + socket2.Connect(System.Net.IPAddress.Loopback, TestUtils.TestPort); + + // *2000000\r\n$4\r\nPING\r\n — known command but excessive count + var payload2 = Encoding.ASCII.GetBytes("*2000000\r\n$4\r\nPING\r\n"); + socket2.Send(payload2); + + var buffer2 = new byte[4096]; + var totalRead2 = 0; + try + { + while (totalRead2 < buffer2.Length) + { + var bytesRead = socket2.Receive(buffer2, totalRead2, buffer2.Length - totalRead2, System.Net.Sockets.SocketFlags.None); + if (bytesRead == 0) break; + totalRead2 += bytesRead; + } + } + catch (System.Net.Sockets.SocketException) + { + // Connection may be reset by server - this is expected + } + + var response2 = Encoding.ASCII.GetString(buffer2, 0, totalRead2); + var expectedResponse2 = + "-ERR Protocol Error: RESP array argument count '1999999' exceeds maximum allowed count of '1048576'.\r\n"; + ClassicAssert.AreEqual(expectedResponse2, response2); + + // Verify the server is still operational after rejecting the malicious connection + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + db.StringSet("testkey", "testvalue"); + ClassicAssert.AreEqual("testvalue", db.StringGet("testkey").ToString()); + } } } \ No newline at end of file diff --git a/test/Garnet.test/RespTestsUtils.cs b/test/standalone/Garnet.test/RespTestsUtils.cs similarity index 100% rename from test/Garnet.test/RespTestsUtils.cs rename to test/standalone/Garnet.test/RespTestsUtils.cs diff --git a/test/Garnet.test/RespTlsTests.cs b/test/standalone/Garnet.test/RespTlsTests.cs similarity index 99% rename from test/Garnet.test/RespTlsTests.cs rename to test/standalone/Garnet.test/RespTlsTests.cs index d2e3602f7e4..bcaeb1c8801 100644 --- a/test/Garnet.test/RespTlsTests.cs +++ b/test/standalone/Garnet.test/RespTlsTests.cs @@ -7,7 +7,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.common; using Garnet.server; using NUnit.Framework; @@ -16,9 +15,8 @@ namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class RespTlsTests : AllureTestBase + public class RespTlsTests : TestBase { GarnetServer server; diff --git a/test/Garnet.test/ServerCredential.cs b/test/standalone/Garnet.test/ServerCredential.cs similarity index 100% rename from test/Garnet.test/ServerCredential.cs rename to test/standalone/Garnet.test/ServerCredential.cs diff --git a/test/Garnet.test/SortedSetRemoveTxn.cs b/test/standalone/Garnet.test/SortedSetRemoveTxn.cs similarity index 98% rename from test/Garnet.test/SortedSetRemoveTxn.cs rename to test/standalone/Garnet.test/SortedSetRemoveTxn.cs index 1a2d341dd64..4cb442f94d1 100644 --- a/test/Garnet.test/SortedSetRemoveTxn.cs +++ b/test/standalone/Garnet.test/SortedSetRemoveTxn.cs @@ -21,7 +21,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce var offset = 0; var subscriptionContainerKey = GetNextArg(ref procInput.parseState, ref offset); - AddKey(subscriptionContainerKey, LockType.Exclusive, true); + AddKey(subscriptionContainerKey, LockType.Exclusive, StoreType.Object); return true; } diff --git a/test/Garnet.test/TaskManagerTests.cs b/test/standalone/Garnet.test/TaskManagerTests.cs similarity index 98% rename from test/Garnet.test/TaskManagerTests.cs rename to test/standalone/Garnet.test/TaskManagerTests.cs index 08a611a3449..cb57946ffa7 100644 --- a/test/Garnet.test/TaskManagerTests.cs +++ b/test/standalone/Garnet.test/TaskManagerTests.cs @@ -4,16 +4,14 @@ using System; using System.Threading; using System.Threading.Tasks; -using Allure.NUnit; using Garnet.server; using NUnit.Framework; using NUnit.Framework.Legacy; namespace Garnet.test { - [AllureNUnit] [TestFixture] - internal class TaskManagerTests : AllureTestBase + internal class TaskManagerTests : TestBase { [Test] public async Task TestBasicRegisterAndRunAsync() diff --git a/test/standalone/Garnet.test/TestBase.cs b/test/standalone/Garnet.test/TestBase.cs new file mode 100644 index 00000000000..3242e1f1e0f --- /dev/null +++ b/test/standalone/Garnet.test/TestBase.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; + +namespace Garnet.test +{ + /// + /// Base class for tests — tracks currently running tests for diagnostics. + /// + public abstract class TestBase + { + // Thread-safe collection to store currently running tests + public static readonly ConcurrentDictionary RunningTests = new(); + + [SetUp] + public void TrackRunningTest() + { + // Add test to the running list + RunningTests[TestContext.CurrentContext.Test.Name] = true; + + if (TestContext.CurrentContext.CurrentRepeatCount > 0) + Debug.WriteLine($"*** Current test iteration {TestContext.CurrentContext.CurrentRepeatCount + 1}: {TestContext.CurrentContext.Test.Name} ***"); + } + + [TearDown] + public void RemoveRunningTest() + { + Assert.That(RunningTests.TryRemove(TestContext.CurrentContext.Test.Name, out _), Is.True, $"Could not find running test {TestContext.CurrentContext.Test.Name}"); + } + } +} + +[SetUpFixture] +public sealed class GlobalUnhandledExceptionHandling +{ + [OneTimeSetUp] + public void Install() + { + AppDomain.CurrentDomain.UnhandledException += (s, e) => + { + DumpTests(); + }; + + TaskScheduler.UnobservedTaskException += (s, e) => + { + DumpTests(); + e.SetObserved(); // Optionally mark observed so it doesn't escalate later + }; + + static void DumpTests() + { + if (Garnet.test.TestBase.RunningTests.Count == 0) + return; + var sb = new StringBuilder(); + _ = sb.AppendLine("*** CURRENTLY RUNNING TESTS ***:"); + foreach (var key in Garnet.test.TestBase.RunningTests.Keys) + _ = sb.AppendLine(key); + Console.WriteLine(sb.ToString()); + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/TestProcedureBitmap.cs b/test/standalone/Garnet.test/TestProcedureBitmap.cs similarity index 95% rename from test/Garnet.test/TestProcedureBitmap.cs rename to test/standalone/Garnet.test/TestProcedureBitmap.cs index 9e12a7cd354..379e80241f3 100644 --- a/test/Garnet.test/TestProcedureBitmap.cs +++ b/test/standalone/Garnet.test/TestProcedureBitmap.cs @@ -35,9 +35,9 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (bitmapB.Length == 0) return false; - AddKey(bitmapA, LockType.Exclusive, false); - AddKey(destinationKey, LockType.Exclusive, false); - AddKey(bitmapB, LockType.Exclusive, false); + AddKey(bitmapA, LockType.Exclusive, StoreType.Main); + AddKey(destinationKey, LockType.Exclusive, StoreType.Main); + AddKey(bitmapB, LockType.Exclusive, StoreType.Main); return true; } @@ -84,7 +84,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p result = false; goto returnTo; } - api.GET(destinationKeyBitOp, out var valueData); + api.GET(destinationKeyBitOp, out PinnedSpanByte valueData); var actualResultBitOp = BitConverter.ToInt64(valueData.ToArray(), 0); long expectedResultBitOp = ~src; diff --git a/test/Garnet.test/TestProcedureHLL.cs b/test/standalone/Garnet.test/TestProcedureHLL.cs similarity index 96% rename from test/Garnet.test/TestProcedureHLL.cs rename to test/standalone/Garnet.test/TestProcedureHLL.cs index a47f4b74ff4..8d01e0a1e59 100644 --- a/test/Garnet.test/TestProcedureHLL.cs +++ b/test/standalone/Garnet.test/TestProcedureHLL.cs @@ -26,7 +26,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (hll.Length == 0) return false; - AddKey(hll, LockType.Exclusive, false); + AddKey(hll, LockType.Exclusive, StoreType.Main); return true; } diff --git a/test/Garnet.test/TestProcedureHash.cs b/test/standalone/Garnet.test/TestProcedureHash.cs similarity index 92% rename from test/Garnet.test/TestProcedureHash.cs rename to test/standalone/Garnet.test/TestProcedureHash.cs index 3b4172dc0e5..ae8cdddf2c6 100644 --- a/test/Garnet.test/TestProcedureHash.cs +++ b/test/standalone/Garnet.test/TestProcedureHash.cs @@ -27,7 +27,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (setA.Length == 0) return false; - AddKey(setA, LockType.Exclusive, true); + AddKey(setA, LockType.Exclusive, StoreType.Object); return true; } @@ -40,8 +40,8 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput procInput) where TGarnetApi : IGarnetApi { var offset = 0; - var pairs = new (ArgSlice field, ArgSlice value)[6]; - var fields = new ArgSlice[pairs.Length]; + var pairs = new (PinnedSpanByte field, PinnedSpanByte value)[6]; + var fields = new PinnedSpanByte[pairs.Length]; var myHash = GetNextArg(ref procInput.parseState, ref offset); @@ -122,12 +122,12 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput return false; // HGET (hashobject exists, field not found) - status = api.HashGet(myHash, ArgSlice.FromPinnedSpan("nonexistingfield"u8), out value); + status = api.HashGet(myHash, PinnedSpanByte.FromPinnedSpan("nonexistingfield"u8), out value); if (status != GarnetStatus.OK || value.Length != 0) return false; // HGET (hashobject not found) - status = api.HashGet(ArgSlice.FromPinnedSpan("nonexistinghash"u8), pairs[0].field, out value); + status = api.HashGet(PinnedSpanByte.FromPinnedSpan("nonexistinghash"u8), pairs[0].field, out value); if (status != GarnetStatus.NOTFOUND || value.Length != 0) return false; diff --git a/test/Garnet.test/TestProcedureLists.cs b/test/standalone/Garnet.test/TestProcedureLists.cs similarity index 93% rename from test/Garnet.test/TestProcedureLists.cs rename to test/standalone/Garnet.test/TestProcedureLists.cs index 880cea06955..9cf678d10aa 100644 --- a/test/Garnet.test/TestProcedureLists.cs +++ b/test/standalone/Garnet.test/TestProcedureLists.cs @@ -29,9 +29,9 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (lstKey.Length == 0 || lstKeyB.Length == 0 || lstKeyC.Length == 0) return false; - AddKey(lstKey, LockType.Exclusive, true); - AddKey(lstKeyB, LockType.Exclusive, true); - AddKey(lstKeyC, LockType.Exclusive, true); + AddKey(lstKey, LockType.Exclusive, StoreType.Object); + AddKey(lstKeyB, LockType.Exclusive, StoreType.Object); + AddKey(lstKeyC, LockType.Exclusive, StoreType.Object); return true; } @@ -45,7 +45,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput procInput) where TGarnetApi : IGarnetApi { var offset = 0; - var elements = new ArgSlice[10]; + var elements = new PinnedSpanByte[10]; var lstKeyA = GetNextArg(ref procInput, ref offset); var lstKeyB = GetNextArg(ref procInput, ref offset); diff --git a/test/Garnet.test/TestProcedureSet.cs b/test/standalone/Garnet.test/TestProcedureSet.cs similarity index 96% rename from test/Garnet.test/TestProcedureSet.cs rename to test/standalone/Garnet.test/TestProcedureSet.cs index 2f33531bd42..a31a4e983c0 100644 --- a/test/Garnet.test/TestProcedureSet.cs +++ b/test/standalone/Garnet.test/TestProcedureSet.cs @@ -26,7 +26,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (setA.Length == 0) return false; - AddKey(setA, LockType.Exclusive, true); + AddKey(setA, LockType.Exclusive, StoreType.Object); return true; } @@ -39,7 +39,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput procInput) where TGarnetApi : IGarnetApi { var offset = 0; - var elements = new ArgSlice[10]; + var elements = new PinnedSpanByte[10]; var setA = GetNextArg(ref procInput, ref offset); diff --git a/test/Garnet.test/TestProcedureSortedSets.cs b/test/standalone/Garnet.test/TestProcedureSortedSets.cs similarity index 89% rename from test/Garnet.test/TestProcedureSortedSets.cs rename to test/standalone/Garnet.test/TestProcedureSortedSets.cs index 14c949a4543..ff3b04f2e48 100644 --- a/test/Garnet.test/TestProcedureSortedSets.cs +++ b/test/standalone/Garnet.test/TestProcedureSortedSets.cs @@ -27,7 +27,7 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce if (ssA.Length == 0) return false; - AddKey(ssA, LockType.Exclusive, true); + AddKey(ssA, LockType.Exclusive, StoreType.Object); return true; } @@ -41,8 +41,8 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput procInput) where TGarnetApi : IGarnetApi { var offset = 0; - var ssItems = new (ArgSlice score, ArgSlice member)[10]; - var ssMembers = new ArgSlice[10]; + var ssItems = new (PinnedSpanByte score, PinnedSpanByte member)[10]; + var ssMembers = new PinnedSpanByte[10]; var ssA = GetNextArg(ref procInput, ref offset); @@ -57,7 +57,7 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput var maxRange = GetNextArg(ref procInput, ref offset); var match = GetNextArg(ref procInput, ref offset); - var ssB = new ArgSlice(); + var ssB = new PinnedSpanByte(); api.SortedSetAdd(ssB, ssItems[0].score, ssItems[0].member, out int count); if (count != 0) return false; @@ -73,7 +73,7 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput var strMatch = Encoding.ASCII.GetString(match.ReadOnlySpan); // Exercise SortedSetScan - api.SortedSetScan(ssA, 0, strMatch, ssItems.Length, out ArgSlice[] itemsInScan); + api.SortedSetScan(ssA, 0, strMatch, ssItems.Length, out PinnedSpanByte[] itemsInScan); // The pattern "*em*" should match all items if (itemsInScan.Length != (ssItems.Length * 2) + 1) @@ -114,15 +114,15 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput if (status != GarnetStatus.OK || newScore != 12345) return false; - status = api.SortedSetExpire(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], DateTimeOffset.UtcNow.AddMinutes(10), ExpireOption.None, out var expireResults); + status = api.SortedSetExpire(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), PinnedSpanByte.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], DateTimeOffset.UtcNow.AddMinutes(10), ExpireOption.None, out var expireResults); if (status != GarnetStatus.OK || expireResults.Length != 2 || expireResults[0] != 1 || expireResults[1] != -2) return false; - status = api.SortedSetTimeToLive(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var expireIn); + status = api.SortedSetTimeToLive(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), PinnedSpanByte.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var expireIn); if (status != GarnetStatus.OK || expireIn.Length != 2 || expireIn[0].TotalMilliseconds <= 0 || expireIn[0].TotalMilliseconds > TimeSpan.FromMinutes(10).TotalMilliseconds || expireIn[1].TotalMilliseconds != 0) return false; - status = api.SortedSetPersist(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), ArgSlice.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var persistResults); + status = api.SortedSetPersist(ssA, [.. ssItems.Skip(4).Take(1).Select(x => x.member), PinnedSpanByte.FromPinnedSpan(Encoding.UTF8.GetBytes("nonExist"))], out var persistResults); if (status != GarnetStatus.OK || persistResults.Length != 2 || persistResults[0] != 1 || persistResults[1] != -2) return false; diff --git a/test/Garnet.test/TestProcess.cs b/test/standalone/Garnet.test/TestProcess.cs similarity index 90% rename from test/Garnet.test/TestProcess.cs rename to test/standalone/Garnet.test/TestProcess.cs index abd8c7ce22c..9717ab81777 100644 --- a/test/Garnet.test/TestProcess.cs +++ b/test/standalone/Garnet.test/TestProcess.cs @@ -25,7 +25,7 @@ internal class GarnetServerTestProcess : IDisposable public StringBuilder OutputLog { get; } - internal GarnetServerTestProcess(Dictionary env, int port = 7000) + internal GarnetServerTestProcess(Dictionary env, int port) { var a = Assembly.GetAssembly(typeof(Program)); var name = a.Location; @@ -33,35 +33,21 @@ internal GarnetServerTestProcess(Dictionary env, int port = 7000 using var cts = new CancellationTokenSource(); - if (Debugger.IsAttached) - { - // If debugging, give us a bit longer before timeouts start happening - cts.CancelAfter(300_000); - } - else - { - cts.CancelAfter(30_000); - } + // If debugging, give us a bit longer before timeouts start happening + cts.CancelAfter(Debugger.IsAttached ? 300_000 : 30_000); while (!TestUtils.IsPortAvailable(port)) { if (cts.IsCancellationRequested) - { throw new GarnetException($"Port {port} is not available, and did not become available before timeout"); - } // Wait for port to be available Thread.Sleep(1_000); } - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - name = string.Concat(name.AsSpan(0, pos), ".exe"); - } - else - { - name = name.Substring(0, pos); - } + name = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? string.Concat(name.AsSpan(0, pos), ".exe") + : name.Substring(0, pos); var endPoint = new IPEndPoint(IPAddress.Loopback, port); Options = TestUtils.GetConfig([endPoint]); @@ -89,23 +75,18 @@ internal GarnetServerTestProcess(Dictionary env, int port = 7000 OutputLog = new(); _ = OutputLog.AppendLine($"Started PID: {process.Id}"); + foreach (var arg in psi.ArgumentList) - { _ = OutputLog.AppendLine($"Arg: {arg}"); - } foreach (var (k, v) in psi.Environment) - { _ = OutputLog.AppendLine($"Env: {k}={v}"); - } process.OutputDataReceived += (obj, lineArgs) => { if (lineArgs.Data == null) - { return; - } lock (OutputLog) { @@ -122,9 +103,7 @@ internal GarnetServerTestProcess(Dictionary env, int port = 7000 (obj, lineArgs) => { if (lineArgs.Data == null) - { return; - } lock (OutputLog) { @@ -142,7 +121,6 @@ internal GarnetServerTestProcess(Dictionary env, int port = 7000 catch { RecordTestOutput(OutputLog); - throw; } diff --git a/test/Garnet.test/TestUtils.cs b/test/standalone/Garnet.test/TestUtils.cs similarity index 87% rename from test/Garnet.test/TestUtils.cs rename to test/standalone/Garnet.test/TestUtils.cs index 77a37b5e889..8cb55612040 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/standalone/Garnet.test/TestUtils.cs @@ -41,6 +41,7 @@ public struct StoreAddressInfo public long ReadOnlyAddress; public long TailAddress; public long MemorySize; + public long ReadCacheHeadAddress; public long ReadCacheBeginAddress; public long ReadCacheTailAddress; } @@ -73,15 +74,41 @@ public enum RevivificationMode UseReviv = 1, } + /// + /// Unique base port for each test sub-project, enabling parallel test runs without port conflicts. + /// + public enum TestPortAssignment + { + GarnetTest = 33278, + GarnetTestAlternate = 34278, // Alternate port for GarnetTest; used by NetworkTests.cs + GarnetTestAcl = 34300, + GarnetTestCollections = 34400, + GarnetTestComplexString = 34500, + GarnetTestExtensions = 34600, + GarnetTestRangeIndex = 34700, + GarnetTestScripting = 34800, + GarnetTestVectorSet = 34900, + } + internal static class TestUtils { - public static readonly int TestPort = 33278; + public static int TestPort = (int)TestPortAssignment.GarnetTest; // No OneTimeSetUp needed for "Garnet.test" to set this /// /// Test server end point /// public static EndPoint EndPoint = new IPEndPoint(IPAddress.Loopback, TestPort); + /// + /// Sets the test port for the current sub-project, updating both and . + /// Call from a [SetUpFixture] in each sub-project. + /// + public static void SetTestPort(TestPortAssignment port) + { + TestPort = (int)port; + EndPoint = new IPEndPoint(IPAddress.Loopback, TestPort); + } + /// /// Whether to use a test progress logger /// @@ -103,7 +130,7 @@ internal static string AzureTestContainer return container; } } - internal static string AzureTestDirectory => TestContext.CurrentContext.Test.MethodName; + internal static string AzureTestDirectory => $"{Environment.ProcessId}_{TestContext.CurrentContext.Test.MethodName}"; internal const string AzureEmulatedStorageString = "UseDevelopmentStorage=true;"; internal static AzureStorageNamedDeviceFactoryCreator AzureStorageNamedDeviceFactoryCreator = IsRunningAzureTests ? new AzureStorageNamedDeviceFactoryCreator(AzureEmulatedStorageString, null) : null; @@ -229,8 +256,8 @@ public static GarnetServer CreateGarnetServer( bool tryRecover = false, bool lowMemory = false, string memorySize = default, - string objectStoreLogMemorySize = default, string pageSize = default, + int pageCount = 0, bool enableAOF = false, bool enableTLS = false, bool disableObjects = false, @@ -243,11 +270,6 @@ public static GarnetServer CreateGarnetServer( string defaultPassword = null, bool useAcl = false, // NOTE: Temporary until ACL is enforced as default string aclFile = null, - string objectStorePageSize = default, - string objectStoreHeapMemorySize = default, - string objectStoreIndexSize = "16k", - string objectStoreIndexMaxSize = default, - string objectStoreReadCacheHeapMemorySize = default, string indexSize = "1m", string indexMaxSize = default, string[] extensionBinPaths = null, @@ -259,7 +281,9 @@ public static GarnetServer CreateGarnetServer( ConnectionProtectionOption enableModuleCommand = ConnectionProtectionOption.No, bool enableLua = false, bool enableReadCache = false, - bool enableObjectStoreReadCache = false, + string readCacheMemorySize = default, + string readCachePageSize = default, + int readCachePageCount = 0, ILogger logger = null, IEnumerable loadModulePaths = null, string pubSubPageSize = null, @@ -279,9 +303,12 @@ public static GarnetServer CreateGarnetServer( bool useInChainRevivOnly = false, bool useLogNullDevice = false, bool enableVectorSetPreview = true, + bool enableRangeIndexPreview = false, string aofMemorySize = "64m", - string aofPageSize = null - ) + string aofPageSize = null, + bool copyReadsToTail = false, + int replayTaskCount = 1 + ) { if (useAzureStorage) IgnoreIfNotRunningAzureTests(); @@ -289,48 +316,44 @@ public static GarnetServer CreateGarnetServer( if (useAzureStorage && !useLogNullDevice) logDir = $"{AzureTestContainer}/{AzureTestDirectory}"; - if (logCheckpointDir != null && !useAzureStorage && !useLogNullDevice) logDir = new DirectoryInfo(string.IsNullOrEmpty(logDir) ? "." : logDir).FullName; + if (logCheckpointDir != null && !useAzureStorage && !useLogNullDevice) + logDir = new DirectoryInfo(string.IsNullOrEmpty(logDir) ? "." : logDir).FullName; var checkpointDir = logCheckpointDir; if (useAzureStorage) checkpointDir = $"{AzureTestContainer}/{AzureTestDirectory}"; - if (logCheckpointDir != null && !useAzureStorage) checkpointDir = new DirectoryInfo(string.IsNullOrEmpty(checkpointDir) ? "." : checkpointDir).FullName; + if (logCheckpointDir != null && !useAzureStorage) + checkpointDir = new DirectoryInfo(string.IsNullOrEmpty(checkpointDir) ? "." : checkpointDir).FullName; if (useAcl) { if (authenticationSettings != null) - { throw new ArgumentException($"Cannot set both {nameof(useAcl)} and {nameof(authenticationSettings)}"); - } - authenticationSettings = new AclAuthenticationPasswordSettings(aclFile, defaultPassword); } else if (defaultPassword != null) { if (authenticationSettings != null) - { throw new ArgumentException($"Cannot set both {nameof(defaultPassword)} and {nameof(authenticationSettings)}"); - } - authenticationSettings = new PasswordAuthenticationSettings(defaultPassword); } // Increase minimum thread pool size to 16 if needed int threadPoolMinThreads = 0; ThreadPool.GetMinThreads(out int workerThreads, out int completionPortThreads); - if (workerThreads < 16 || completionPortThreads < 16) threadPoolMinThreads = 16; + if (workerThreads < 16 || completionPortThreads < 16) + threadPoolMinThreads = 16; GarnetServerOptions opts = new(logger) { EnableStorageTier = logDir != null, LogDir = logDir, CheckpointDir = checkpointDir, - EndPoints = endpoints ?? ([EndPoint]), + EndPoints = endpoints ?? [EndPoint], DisablePubSub = disablePubSub, Recover = tryRecover, - IndexSize = indexSize, - ObjectStoreIndexSize = objectStoreIndexSize, + IndexMemorySize = indexSize, EnableAOF = enableAOF, EnableLua = enableLua, AofMemorySize = aofMemorySize, @@ -363,58 +386,62 @@ public static GarnetServer CreateGarnetServer( EnableDebugCommand = enableDebugCommand, EnableModuleCommand = enableModuleCommand, EnableReadCache = enableReadCache, - EnableObjectStoreReadCache = enableObjectStoreReadCache, ReplicationOffsetMaxLag = asyncReplay ? -1 : 0, + AofReplayTaskCount = replayTaskCount, LuaOptions = enableLua ? new LuaOptions(luaMemoryMode, luaMemoryLimit, luaTimeout ?? Timeout.InfiniteTimeSpan, luaLoggingMode, luaAllowedFunctions ?? [], logger) : null, UnixSocketPath = unixSocketPath, UnixSocketPermission = unixSocketPermission, SlowLogThreshold = slowLogThreshold, ExpiredKeyDeletionScanFrequencySecs = expiredKeyDeletionScanFrequencySecs, - EnableVectorSetPreview = enableVectorSetPreview + EnableVectorSetPreview = enableVectorSetPreview, + EnableRangeIndexPreview = enableRangeIndexPreview, + CopyReadsToTail = copyReadsToTail, }; if (!string.IsNullOrEmpty(memorySize)) - opts.MemorySize = memorySize; - - if (!string.IsNullOrEmpty(objectStoreLogMemorySize)) - opts.ObjectStoreLogMemorySize = objectStoreLogMemorySize; + opts.LogMemorySize = memorySize; if (!string.IsNullOrEmpty(pageSize)) opts.PageSize = pageSize; - if (!string.IsNullOrEmpty(pubSubPageSize)) - opts.PubSubPageSize = pubSubPageSize; - - if (!string.IsNullOrEmpty(objectStorePageSize)) - opts.ObjectStorePageSize = objectStorePageSize; + if (pageCount != 0) + { + opts.PageCount = pageCount; - if (!string.IsNullOrEmpty(objectStoreHeapMemorySize)) - opts.ObjectStoreHeapMemorySize = objectStoreHeapMemorySize; + // If there is a pageCount and no memorySize, then we are bypassing the size tracker (which is automatically started if memorySize is specified). + if (string.IsNullOrEmpty(memorySize)) + opts.LogMemorySize = string.Empty; + } - if (!string.IsNullOrEmpty(objectStoreReadCacheHeapMemorySize)) - opts.ObjectStoreReadCacheHeapMemorySize = objectStoreReadCacheHeapMemorySize; + if (!string.IsNullOrEmpty(pubSubPageSize)) + opts.PubSubPageSize = pubSubPageSize; - if (indexMaxSize != default) opts.IndexMaxSize = indexMaxSize; - if (objectStoreIndexMaxSize != default) opts.ObjectStoreIndexMaxSize = objectStoreIndexMaxSize; + if (indexMaxSize != default) + opts.IndexMaxMemorySize = indexMaxSize; if (!string.IsNullOrEmpty(aofPageSize)) opts.AofPageSize = aofPageSize; if (lowMemory) { - opts.MemorySize = opts.ObjectStoreLogMemorySize = memorySize == default ? "1024" : memorySize; - opts.PageSize = opts.ObjectStorePageSize = pageSize == default ? "512" : pageSize; - if (enableReadCache) - { - opts.ReadCacheMemorySize = opts.MemorySize; - opts.ReadCachePageSize = opts.PageSize; - } + opts.LogMemorySize = string.IsNullOrEmpty(memorySize) ? "2k" : memorySize; // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking + opts.PageSize = pageSize == default ? "512" : pageSize; - if (enableObjectStoreReadCache) - { - opts.ObjectStoreReadCacheLogMemorySize = opts.MemorySize; - opts.ObjectStoreReadCachePageSize = opts.PageSize; - } + // If there is a pageCount and no memorySize, then we are bypassing the size tracker (which is automatically started if memorySize is specified). + // This is especially useful for two-page tests, which is less than LogSizeTracker.MinTargetPageCount pages. + if (pageCount != 0 && memorySize == default) + opts.LogMemorySize = string.Empty; + } + + if (enableReadCache) + { + opts.ReadCacheMemorySize = readCacheMemorySize ?? opts.LogMemorySize; + opts.ReadCachePageSize = readCachePageSize ?? opts.PageSize; + opts.ReadCachePageCount = readCachePageCount != 0 ? readCachePageCount : opts.PageCount; + + // If there is a pageCount and no memorySize, then we are bypassing the size tracker (which is automatically started if memorySize is specified). + if (opts.ReadCachePageCount != 0 && string.IsNullOrEmpty(opts.ReadCacheMemorySize)) + opts.ReadCacheMemorySize = string.Empty; } ILoggerFactory loggerFactory = null; @@ -424,7 +451,7 @@ public static GarnetServer CreateGarnetServer( { if (useTestLogger) { - _ = builder.AddProvider(new NUnitLoggerProvider(TestContext.Progress, TestContext.CurrentContext.Test.MethodName, null, false, false, LogLevel.Trace)); + _ = builder.AddProvider(new NUnitLoggerProvider(TestContext.Progress, $"{Environment.ProcessId}_{TestContext.CurrentContext.Test.MethodName}", null, false, false, LogLevel.Trace)); } if (logTo != null) @@ -445,7 +472,6 @@ public static GarnetServer CreateGarnetServer( opts.RevivInChainOnly = false; opts.RevivBinRecordCounts = []; opts.RevivBinRecordSizes = []; - opts.RevivObjBinRecordCount = 256; } if (useInChainRevivOnly) @@ -467,13 +493,20 @@ public static GarnetServer CreateGarnetServer( /// /// /// - public static ILoggerFactory CreateLoggerFactoryInstance(TextWriter textWriter, LogLevel logLevel, string scope = "", HashSet skipCmd = null, bool recvOnly = false, bool matchLevel = false) + public static (ILoggerFactory, NUnitLoggerProvider) CreateLoggerFactoryInstance( + TextWriter textWriter, + LogLevel logLevel, + string scope = "", + HashSet skipCmd = null, + bool recvOnly = false, + bool matchLevel = false) { - return LoggerFactory.Create(builder => + var provider = new NUnitLoggerProvider(textWriter, scope, skipCmd, recvOnly, matchLevel, logLevel); + return (LoggerFactory.Create(builder => { - builder.AddProvider(new NUnitLoggerProvider(textWriter, scope, skipCmd, recvOnly, matchLevel, logLevel)); + builder.AddProvider(provider); builder.SetMinimumLevel(logLevel); - }); + }), provider); } public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnetCluster( @@ -499,8 +532,6 @@ public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnet int CommitFrequencyMs = 0, bool useAofNullDevice = false, bool DisableStorageTier = false, - bool EnableIncrementalSnapshots = false, - bool FastCommit = true, string authUsername = null, string authPassword = null, bool useAcl = false, // NOTE: Temporary until ACL is enforced as default @@ -529,6 +560,8 @@ public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnet int checkpointThrottleFlushDelayMs = 0, bool clusterReplicaResumeWithData = false, int replicaSyncTimeout = 60, + int sublogCount = 1, + int replayTaskCount = 1, int expiredObjectCollectionFrequencySecs = 0, ClusterPreferredEndpointType clusterPreferredEndpointType = ClusterPreferredEndpointType.Ip, string clusterAnnounceHostname = null, @@ -567,8 +600,6 @@ public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnet commitFrequencyMs: CommitFrequencyMs, useAofNullDevice: useAofNullDevice, disableStorageTier: DisableStorageTier, - enableIncrementalSnapshots: EnableIncrementalSnapshots, - fastCommit: FastCommit, authUsername: authUsername, authPassword: authPassword, useAcl: useAcl, @@ -596,6 +627,8 @@ public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnet checkpointThrottleFlushDelayMs: checkpointThrottleFlushDelayMs, clusterReplicaResumeWithData: clusterReplicaResumeWithData, replicaSyncTimeout: replicaSyncTimeout, + sublogCount: sublogCount, + replayTaskCount: replayTaskCount, expiredObjectCollectionFrequencySecs: expiredObjectCollectionFrequencySecs, clusterPreferredEndpointType: clusterPreferredEndpointType, clusterAnnounceHostname: clusterAnnounceHostname, @@ -609,7 +642,7 @@ public static (GarnetServer[] Nodes, GarnetServerOptions[] Options) CreateGarnet var iter = 0; while (!IsPortAvailable(ipEndpoint.Port)) { - ClassicAssert.Less(30, iter, "Failed to connect within 30 seconds"); + ClassicAssert.Less(iter, 30, "Failed to connect within 30 seconds"); TestContext.Progress.WriteLine($"Waiting for Port {ipEndpoint.Port} to become available for {TestContext.CurrentContext.WorkerId}:{iter++}"); Thread.Sleep(1000); } @@ -644,8 +677,6 @@ public static GarnetServerOptions GetGarnetServerOptions( int commitFrequencyMs = 0, bool useAofNullDevice = false, bool disableStorageTier = false, - bool enableIncrementalSnapshots = false, - bool fastCommit = true, string authUsername = null, string authPassword = null, bool useAcl = false, // NOTE: Temporary until ACL is enforced as default @@ -677,11 +708,14 @@ public static GarnetServerOptions GetGarnetServerOptions( int checkpointThrottleFlushDelayMs = 0, bool clusterReplicaResumeWithData = false, int replicaSyncTimeout = 60, + int sublogCount = 1, + int replayTaskCount = 1, int expiredObjectCollectionFrequencySecs = 0, ClusterPreferredEndpointType clusterPreferredEndpointType = ClusterPreferredEndpointType.Ip, string clusterAnnounceHostname = null, bool enableVectorSetPreview = true, int vectorSetReplayTaskCount = 0, + bool enableRangeIndexPreview = false, int threadPoolMinIOCompletionThreads = 0) { if (useAzureStorage) @@ -731,7 +765,6 @@ public static GarnetServerOptions GetGarnetServerOptions( { ThreadPoolMinThreads = 512, SegmentSize = segmentSize, - ObjectStoreSegmentSize = segmentSize, EnableStorageTier = useAzureStorage || (!disableStorageTier && logDir != null), LogDir = disableStorageTier ? null : logDir, CheckpointDir = checkpointDir, @@ -741,16 +774,14 @@ public static GarnetServerOptions GetGarnetServerOptions( EnableDebugCommand = ConnectionProtectionOption.Yes, EnableModuleCommand = ConnectionProtectionOption.Yes, Recover = tryRecover, - IndexSize = "1m", - ObjectStoreIndexSize = "16k", + IndexMemorySize = "1m", EnableCluster = enableCluster, CleanClusterConfig = cleanClusterConfig, ClusterTimeout = timeout, QuietMode = true, EnableAOF = enableAOF, - MemorySize = "1g", + LogMemorySize = "1g", GossipDelay = gossipDelay, - EnableFastCommit = fastCommit, MetricsSamplingFrequency = metricsSamplingFrequency, TlsOptions = useTLS ? new GarnetTlsOptions( certFileName: certFile, @@ -782,7 +813,6 @@ public static GarnetServerOptions GetGarnetServerOptions( OnDemandCheckpoint = onDemandCheckpoint, CommitFrequencyMs = commitFrequencyMs, UseAofNullDevice = useAofNullDevice, - EnableIncrementalSnapshots = enableIncrementalSnapshots, AuthSettings = useAcl ? authenticationSettings : (authPassword != null ? authenticationSettings : null), ClusterUsername = authUsername, ClusterPassword = authPassword, @@ -806,16 +836,19 @@ public static GarnetServerOptions GetGarnetServerOptions( CheckpointThrottleFlushDelayMs = checkpointThrottleFlushDelayMs, ClusterReplicaResumeWithData = clusterReplicaResumeWithData, ReplicaSyncTimeout = replicaSyncTimeout <= 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromSeconds(replicaSyncTimeout), + AofPhysicalSublogCount = sublogCount, + AofReplayTaskCount = replayTaskCount, EnableVectorSetPreview = enableVectorSetPreview, VectorSetReplayTaskCount = vectorSetReplayTaskCount, + EnableRangeIndexPreview = enableRangeIndexPreview, ExpiredObjectCollectionFrequencySecs = expiredObjectCollectionFrequencySecs, ThreadPoolMinIOCompletionThreads = threadPoolMinIOCompletionThreads, }; if (lowMemory) { - opts.MemorySize = opts.ObjectStoreLogMemorySize = memorySize == default ? "1024" : memorySize; - opts.PageSize = opts.ObjectStorePageSize = pageSize == default ? "512" : pageSize; + opts.LogMemorySize = string.IsNullOrEmpty(memorySize) ? "2k" : memorySize; // Must be LogSizeTracker.MinTargetPageCount pages due to memory size tracking + opts.PageSize = pageSize == default ? "512" : pageSize; } return opts; @@ -878,7 +911,7 @@ public static ConfigurationOptions GetConfig( AbortOnConnectFail = true, Password = authPassword, User = authUsername, - ClientName = TestContext.CurrentContext.Test.MethodName, + ClientName = $"{Environment.ProcessId}_{TestContext.CurrentContext.Test.MethodName}", Protocol = protocol, }; @@ -981,10 +1014,12 @@ public static EndPointCollection GetShardEndPoints(int shards, IPAddress address internal static string MethodTestDir => UnitTestWorkingDir(); /// - /// Find root test based on prefix Garnet.test + /// Find root test directory (test/) based on prefix Garnet.test. + /// After splitting on "Garnet.test", we land in test/standalone/ or test/cluster/, + /// so navigate up one level to reach test/. /// internal static string RootTestsProjectPath => - TestContext.CurrentContext.TestDirectory.Split("Garnet.test")[0]; + Path.GetFullPath(Path.Combine(TestContext.CurrentContext.TestDirectory.Split("Garnet.test")[0], "..")); /// /// Build path for unit test working directory. @@ -992,12 +1027,12 @@ public static EndPointCollection GetShardEndPoints(int shards, IPAddress address /// internal static string UnitTestWorkingDir() { - // Include process id to avoid conflicts between parallel test runs - var testPath = $"{Environment.ProcessId}_{TestContext.CurrentContext.Test.ClassName}_{TestContext.CurrentContext.Test.MethodName}"; + // Include process id to avoid conflicts between parallel test runs, and remove the prefix to keep the length short. + var testPath = $"{Environment.ProcessId}_{TestContext.CurrentContext.Test.ClassName.Split("Garnet.test")[0]}_{TestContext.CurrentContext.Test.MethodName}"; // Incorporate arguments (as a hash code) so different runs of the same method get different folders // - // Using hashes instead of the arugments themselves to keep length down + // Using hashes instead of the arguments themselves to keep length down if ((TestContext.CurrentContext.Test.Arguments?.Length ?? 0) > 0) { HashCode hash = new(); @@ -1035,6 +1070,8 @@ internal static void DeleteDirectory(string path, bool wait = false) { if (!Directory.Exists(path)) return; + + // Recursively delete subdirectories, then fall through to delete this directory. foreach (string directory in Directory.GetDirectories(path)) DeleteDirectory(directory, wait); break; @@ -1044,7 +1081,7 @@ internal static void DeleteDirectory(string path, bool wait = false) } } - bool retry = true; + var retry = true; while (retry) { // Exceptions may happen due to a handle briefly remaining held after Dispose(). @@ -1054,9 +1091,9 @@ internal static void DeleteDirectory(string path, bool wait = false) if (Directory.Exists(path)) Directory.Delete(path, true); } - catch (Exception ex) when (ex is IOException || - ex is UnauthorizedAccessException) + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + // If we're not waiting, try once more then give up. if (!wait) { try { Directory.Delete(path, true); } @@ -1064,6 +1101,7 @@ internal static void DeleteDirectory(string path, bool wait = false) return; } retry = true; + _ = Thread.Yield(); } } } @@ -1157,10 +1195,10 @@ public static void CreateTestLibrary(string[] namespaces, string[] referenceFile } } - public static StoreAddressInfo GetStoreAddressInfo(IServer server, bool includeReadCache = false, bool isObjectStore = false) + public static StoreAddressInfo GetStoreAddressInfo(IServer server, bool includeReadCache = false) { StoreAddressInfo result = default; - var info = isObjectStore ? server.Info("OBJECTSTORE") : server.Info("STORE"); + var info = server.Info("STORE"); foreach (var section in info) { foreach (var entry in section) @@ -1175,6 +1213,8 @@ public static StoreAddressInfo GetStoreAddressInfo(IServer server, bool includeR result.TailAddress = long.Parse(entry.Value); else if (entry.Key.Equals("Log.MemorySizeBytes")) result.MemorySize = long.Parse(entry.Value); + else if (includeReadCache && entry.Key.Equals("ReadCache.HeadAddress")) + result.ReadCacheHeadAddress = long.Parse(entry.Value); else if (includeReadCache && entry.Key.Equals("ReadCache.BeginAddress")) result.ReadCacheBeginAddress = long.Parse(entry.Value); else if (includeReadCache && entry.Key.Equals("ReadCache.TailAddress")) @@ -1193,9 +1233,8 @@ public static StoreAddressInfo GetStoreAddressInfo(IServer server, bool includeR /// Effective memory size public static long GetEffectiveMemorySize(string memorySize, string pageSize, out long parsedPageSize) { - parsedPageSize = ServerOptions.ParseSize(pageSize, out _); - var parsedMemorySize = 1L << GarnetServerOptions.MemorySizeBits(memorySize, pageSize, out var epc); - return parsedMemorySize - (epc * parsedPageSize); + parsedPageSize = ServerOptions.PreviousPowerOf2(ServerOptions.ParseSize(pageSize, out _)); + return ServerOptions.ParseSize(memorySize, out _); } /// @@ -1213,15 +1252,12 @@ internal static void OnTearDown(bool waitForDelete = false, ILogger logger = nul { DeleteDirectory(MethodTestDir, wait: waitForDelete); var count = Tsavorite.core.LightEpoch.ActiveInstanceCount(); - - var failMessage = ""; - if (count != 0) { // Reset all instances to avoid impacting other tests Tsavorite.core.LightEpoch.ResetAllInstances(); logger?.LogError("Tsavorite.core.LightEpoch instances still active: {count}", count); - failMessage += $"Tsavorite.core.LightEpoch instances still active: {count}; "; + Assert.Fail($"Tsavorite.core.LightEpoch instances still active: {count}"); } var count2 = client.LightEpoch.ActiveInstanceCount(); @@ -1230,12 +1266,7 @@ internal static void OnTearDown(bool waitForDelete = false, ILogger logger = nul // Reset all instances to avoid impacting other tests client.LightEpoch.ResetAllInstances(); logger?.LogError("Garnet.client.LightEpoch instances still active: {count2}", count2); - failMessage += $"Garnet.client.LightEpoch instances still active: {count2}; "; - } - - if (failMessage != "") - { - ClassicAssert.Fail(failMessage); + Assert.Fail($"Garnet.client.LightEpoch instances still active: {count2}"); } } } diff --git a/test/Garnet.test/UnixSocketTests.cs b/test/standalone/Garnet.test/UnixSocketTests.cs similarity index 98% rename from test/Garnet.test/UnixSocketTests.cs rename to test/standalone/Garnet.test/UnixSocketTests.cs index 3689b38f0e6..9eda549ebe7 100644 --- a/test/Garnet.test/UnixSocketTests.cs +++ b/test/standalone/Garnet.test/UnixSocketTests.cs @@ -6,16 +6,14 @@ using System.IO; using System.Net.Sockets; using System.Threading.Tasks; -using Allure.NUnit; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; namespace Garnet.test { - [AllureNUnit] [TestFixture] - public class UnixSocketTests : AllureTestBase + public class UnixSocketTests : TestBase { [SetUp] public void Setup() diff --git a/test/Garnet.test/WriteWithExpiryTxn.cs b/test/standalone/Garnet.test/WriteWithExpiryTxn.cs similarity index 97% rename from test/Garnet.test/WriteWithExpiryTxn.cs rename to test/standalone/Garnet.test/WriteWithExpiryTxn.cs index 2f2993444cc..4faacf2cc93 100644 --- a/test/Garnet.test/WriteWithExpiryTxn.cs +++ b/test/standalone/Garnet.test/WriteWithExpiryTxn.cs @@ -19,7 +19,7 @@ sealed class WriteWithExpiryTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { int offset = 0; - AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, StoreType.Main); return true; } diff --git a/test/Garnet.test/redis.conf b/test/standalone/Garnet.test/redis.conf similarity index 100% rename from test/Garnet.test/redis.conf rename to test/standalone/Garnet.test/redis.conf diff --git a/test/Garnet.test/runGarnetTests.cmd b/test/standalone/Garnet.test/runGarnetTests.cmd similarity index 79% rename from test/Garnet.test/runGarnetTests.cmd rename to test/standalone/Garnet.test/runGarnetTests.cmd index 560f2eb1e1a..1af35385eb6 100644 --- a/test/Garnet.test/runGarnetTests.cmd +++ b/test/standalone/Garnet.test/runGarnetTests.cmd @@ -15,6 +15,6 @@ dotnet build -c !config!&& cd ../../../ echo [==================] FOR /L %%A IN (1,1,%numCount%) DO ( echo [============================= Running iteration number %%A out of %numCount%, started on %time% =============================] - cd .\Garnet\test\Garnet.test\ && dotnet test -c !config! --logger:"console;verbosity=detailed" --filter !filter! --no-build && cd ../../../ + cd .\Garnet\test\standalone\Garnet.test\ && dotnet test -c !config! --logger:"console;verbosity=detailed" --filter !filter! --no-build && cd ../../../../ echo [============================= Ended iteration number %%A out of %numCount%, completed on %time% ===============================] ) \ No newline at end of file diff --git a/test/Garnet.test/test.bat b/test/standalone/Garnet.test/test.bat similarity index 100% rename from test/Garnet.test/test.bat rename to test/standalone/Garnet.test/test.bat diff --git a/test/Garnet.test/test.sh b/test/standalone/Garnet.test/test.sh similarity index 100% rename from test/Garnet.test/test.sh rename to test/standalone/Garnet.test/test.sh diff --git a/test/standalone/garnet.runsettings b/test/standalone/garnet.runsettings new file mode 100644 index 00000000000..c8703b53003 --- /dev/null +++ b/test/standalone/garnet.runsettings @@ -0,0 +1,7 @@ + + + + + 0 + + diff --git a/website/blog/2025-01-18-etag-when-and-how.md b/website/blog/2025-01-18-etag-when-and-how.md index 14b7f97d71a..a0beca0c4bd 100644 --- a/website/blog/2025-01-18-etag-when-and-how.md +++ b/website/blog/2025-01-18-etag-when-and-how.md @@ -9,7 +9,7 @@ tags: [garnet, concurrency, caching, lock-free, etags] Native ETags in a cache-store enable real-world use cases such as maintaining cache consistency, reducing network bandwidth utilization, and avoiding full-blown transactions for several applications. -Garnet provides native ETag support for raw strings (data added and retrieved using operations such as `GET` and `SET`). It is not available for objects (such as sorted-set, hash, list). This feature is available without requiring any migration, allowing your existing key-value pairs to start leveraging ETags immediately. You can find the ETag API documentation [here](/docs/commands/garnet-specific-commands#native-etag-support). +Garnet provides native ETag support for raw strings via a dedicated set of ETag commands (`SETWITHETAG`, `GETWITHETAG`, `SETIFMATCH`, `SETIFGREATER`, `GETIFNOTMATCH`, `DELIFGREATER`). It is not available for objects (such as sorted-set, hash, list). **Important:** Users must partition their keys and use only ETag commands on ETag-managed keys. Mixing ETag and non-ETag commands on the same key results in undefined behavior. You can find the ETag API documentation [here](/docs/commands/garnet-specific-commands#native-etag-support). This article explores when and how you can use this new Garnet feature for both your current and future applications. diff --git a/website/docs/cluster/replication.md b/website/docs/cluster/replication.md index 84232d1f27e..380b0924254 100644 --- a/website/docs/cluster/replication.md +++ b/website/docs/cluster/replication.md @@ -247,4 +247,27 @@ Note that diskless replication does not take an actual checkpoint. Hence every time a full sync is performed, the AOF is not automatically truncated (unless FAT flag is used). This happens to ensure durability in the event of a failure which will not be possible if the AOF gets truncated without a persistent checkpoint. However, the store version gets incremented to ensure consistency across different instances that may be fully synced at different times. -Users can still utilize SAVE/BGSAVE commands or --aof-size-limit to periodically take a checkpoint and safely truncates the AOF. \ No newline at end of file +Users can still utilize SAVE/BGSAVE commands or --aof-size-limit to periodically take a checkpoint and safely truncates the AOF. + +# Parallel Replication + +Garnet supports a parallel replication feature, leveraging multiple tsavorite logs to improve write throughput at the primary and replay speed at the replica. +This feature can be configured using the following configuration parameters + +| Parameter | Purpose | +|-----------|---------| +| `AofPhysicalSublogCount` | Number of physical `TsavoriteLog` instances by GarnetLog. | +| `AofReplayTaskCount` | Replay tasks per physical sublog at replica. | +| `AofTailWitnessFreqMs` | Background task frequency for advancing time for idle sublogs. | + +By default Garnet is configured to operate without parallel replication when AOF is enabled with cluster mode. +Parallel replication is enabled when `AofPhysicalSublogCount` > 1 or `AofReplayTaskCount` > 1. +Using `AofPhysicalSublogCount` > 1 requires also adjusting the `AofTailWitnessFreqMs` value which is used to ensure time is advanced appropriately. +For more information, check the [development instructions](../dev/cluster/replication-dev). +Note that an in-place upgrade or downgrade between legacy and parallel replication schemes is not currently supported. + +To migrate between replication schemes, you have two options: + +1. **Use the migration functionality** — Migrate keys and slots from an instance using the legacy replication scheme to an instance configured with parallel replication (or vice versa). + +2. **Dump and restore** — Export keys from the source instance and let your client application handle the migration to a target node configured with the desired replication scheme. \ No newline at end of file diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index dfb9ab38087..42b7422ed1b 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -404,3 +404,12 @@ Note that this list is subject to change as we continue to expand our API comman | | [UNWATCH](transactions.md#unwatch) | ➕ | | | | [WATCH](transactions.md#watch) | ➕ | | | **JSON Module** | [JSON Module](json.md) | ➕ | Partially Implemented | +| **RANGE INDEX (Preview)** | [RI.CREATE](range-index.md#ricreate) | ➕ | Preview | +| | [RI.SET](range-index.md#riset) | ➕ | Preview | +| | [RI.GET](range-index.md#riget) | ➕ | Preview | +| | [RI.DEL](range-index.md#ridel) | ➕ | Preview | +| | [RI.SCAN](range-index.md#riscan) | ➕ | Preview | +| | [RI.RANGE](range-index.md#rirange) | ➕ | Preview | +| | [RI.EXISTS](range-index.md#riexists) | ➕ | Preview | +| | [RI.CONFIG](range-index.md#riconfig) | ➕ | Preview | +| | [RI.METRICS](range-index.md#rimetrics) | ➕ | Preview | diff --git a/website/docs/commands/garnet-specific.md b/website/docs/commands/garnet-specific.md index 56e7a9f03fc..00a26aa0a86 100644 --- a/website/docs/commands/garnet-specific.md +++ b/website/docs/commands/garnet-specific.md @@ -166,33 +166,29 @@ for details. Garnet provides support for ETags on raw strings. By using the ETag-related commands outlined below, you can associate any **string based key-value pair** inserted into Garnet with an automatically updated ETag. Compatibility with non-ETag commands and the behavior of data inserted with ETags are detailed at the end of this document. -To initialize a key value pair with an ETag you can use either the SET command with the newly added "WITHETAG" optional flag, or you can take any existing Key value pair and call SETIFMATCH with the ETag argument as 0 (Any key value pair without an explicit ETag has an ETag of 0 implicitly). Read more about Etag use cases and patterns [here](../../blog/etags-when-and-how) +To initialize a key value pair with an ETag, use the `SETWITHETAG` command. You can also take any existing Key value pair and call SETIFMATCH with the ETag argument as 0 (Any key value pair without an explicit ETag has an ETag of 0 implicitly). Read more about Etag use cases and patterns [here](../../blog/etags-when-and-how) --- -### **SET (WITHETAG)** +### **SETWITHETAG** #### **Syntax** -```bash - SET key value [NX | XX] [EX seconds | PX milliseconds] [KEEPTTL] WITHETAG +``` +SETWITHETAG key value [EX seconds | PX milliseconds] ``` -Set **key** to hold the string value along with an ETag. If key already holds a value, it is overwritten, regardless of its type. Any previous time to live associated with the **key** is discarded on successful SET operation. +Set **key** to hold the string value along with an ETag. If the key already holds a value, it is overwritten. If the key had an existing ETag, the ETag is incremented; otherwise, a new ETag of 1 is assigned. **Options:** * EX seconds -- Set the specified expire time, in seconds (a positive integer). * PX milliseconds -- Set the specified expire time, in milliseconds (a positive integer). -* NX -- Only set the key if it does not already exist. -* XX -- Only set the key if it already exists. -* KEEPTTL -- Retain the time to live associated with the key. -* WITHETAG -- **Adding this sets the Key Value pair with an initial ETag**, if called on an existing key value pair with an ETag, this command will update the ETag transparently. #### Resp Reply -* Integer reply: WITHETAG given: The ETag associated with the value. +* Integer reply: The ETag associated with the value. --- @@ -296,17 +292,33 @@ Deletes a key only if the provided Etag is strictly greater than the existing Et ETags are currently not supported for servers running in Cluster mode. This will be supported soon. -Below is the expected behavior of ETag-associated key-value pairs when non-ETag commands are used. +:::warning Important: Key Partitioning Required +All non-ETag commands (SET, GET, APPEND, INCR, MSET, BITOP, RENAME, etc.) are completely **ETag-blind**. They do not read, check, update, or preserve ETags. -- **MSET, BITOP**: These commands will replace an existing ETag-associated key-value pair with a non-ETag key-value pair, effectively removing the ETag. +**Users MUST partition their keys**: use ONLY ETag commands (`SETWITHETAG`, `GETWITHETAG`, `SETIFMATCH`, `SETIFGREATER`, `GETIFNOTMATCH`, `DELIFGREATER`) on ETag-managed keys. -- **SET**: Only if used with additional option "WITHETAG" will calling SET update the etag while inserting the new key-value pair over the existing key-value pair. +Mixing ETag and non-ETag commands on the same key (e.g., using `SET` on a key created with `SETWITHETAG`) will result in **undefined ETag behavior** — the ETag may be lost, stale, or corrupted. This is by design for maximum performance: non-ETag commands pay zero overhead for ETag functionality. +::: -- **RENAME**: RENAME takes an option for WITHETAG. When called WITHETAG it will rename the key with an etag if the key being renamed to did not exist, else it will increment the existing etag of the key being renamed to. - -- **Custom Commands**: While etag based key value pairs **can be used blindly inside of custom transactions and custom procedures**, ETag set key value pairs are **not supported to be used from inside of Custom Raw String Functions.** +**Correct usage:** +```bash +# ETag keys — use only ETag commands +SETWITHETAG etag:user:1 "data" # Initialize with ETag +GETWITHETAG etag:user:1 # Read with ETag +SETIFMATCH etag:user:1 "new" 1 # Conditional update + +# Non-ETag keys — use normal commands +SET user:2 "data" +GET user:2 +APPEND user:2 " more" +``` -All other commands will update the etag internally if they modify the underlying data, and any responses from them will not expose the etag to the client. To the users the etag and it's updates remain hidden in non-etag commands. +**Incorrect usage (undefined behavior):** +```bash +SETWITHETAG mykey "data" # Sets ETag +SET mykey "other" # ETag behavior is UNDEFINED +GETWITHETAG mykey # May return stale/missing ETag +``` --- diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 8f6a34fe168..29b8b6553cf 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -366,14 +366,11 @@ One of the following: #### Syntax ```bash - RENAME key newkey [WITHETAG] + RENAME key newkey ``` Renames key to newkey. It returns an error when key does not exist. If newkey already exists it is overwritten, when this happens RENAME executes an implicit [DEL](#del) operation. -#### **Options:** -* WITHETAG - If the newkey did not exist, the newkey will now have an ETag associated with it after the rename. If the newkey existed before with an ETag the RENAME will update the ETag. If the newkey existed before without an ETag, then after the RENAME the newkey would have an ETag associated with it. You can read more about ETags [here](../commands/garnet-specific-commands#native-etag-support). - #### Resp Reply Simple string reply: OK. @@ -385,14 +382,11 @@ Simple string reply: OK. #### Syntax ```bash - RENAMENX key newkey [WITHETAG] + RENAMENX key newkey ``` Renames key to newkey if newkey does not yet exist. It returns an error when key does not exist. -#### **Options:** -* WITHETAG - The newkey will now have an ETag associated with it after the rename. You can read more about ETags [here](../commands/garnet-specific-commands#native-etag-support). - #### Resp Reply One of the following: diff --git a/website/docs/commands/range-index.md b/website/docs/commands/range-index.md new file mode 100644 index 00000000000..2a811f12583 --- /dev/null +++ b/website/docs/commands/range-index.md @@ -0,0 +1,383 @@ +--- +id: range-index +sidebar_label: Range Index +title: Range Index Commands +slug: range-index +--- + +# Range Index (Preview) + +Range Index is a Garnet data type backed by [Bf-Tree](https://github.com/microsoft/bf-tree), a high-performance B-tree +optimized for range queries on byte-string keys. It enables ordered key-value storage within a single Garnet key, +supporting point reads, inserts, deletes, and efficient range scans — capabilities not available with standard Redis +data structures. + +:::note +Range Index is a preview feature. Enable it with the `--enable-range-index-preview` server flag. +::: + +## Overview + +Each Range Index is an independent ordered key-value store identified by a Garnet key name. Within that index, entries +are sorted lexicographically by their field (key) bytes, enabling range scans and prefix queries. + +All commands use the `RI.*` prefix. Deletion of the index itself uses the standard `DEL` / `UNLINK` commands. + +### Storage Backends + +Each Range Index is created with one of two storage backends: + +- **`DISK`** (default) — Leaf pages are stored in a data file on disk, with an in-memory circular buffer as a + hot-data cache. Total capacity is limited by disk space. Supports all operations including scan. This is the + recommended mode for production use. +- **`MEMORY`** — All data lives in a bounded in-memory circular buffer. Total capacity is limited by `CACHESIZE`. + Scan operations are **not supported** in this mode. + +### Persistence + +Range Index is fully integrated with Garnet's checkpoint and AOF mechanisms: + +- **Checkpoint (BGSAVE):** BfTree data is snapshotted alongside the Tsavorite store checkpoint. + Recovery restores the tree to the exact checkpoint state. +- **AOF:** Write operations (RI.SET, RI.DEL) are logged to the append-only file. On recovery, + the checkpoint is restored first, then AOF entries are replayed to recover post-checkpoint mutations. +- **Eviction and lazy restore:** When memory pressure causes the stub to be evicted from the log, + the BfTree data file is preserved. On next access, the tree is lazily restored from its snapshot. + +--- + +## Lifecycle Commands + +### RI.CREATE + +Create a new Range Index. + +#### Syntax + +```bash +RI.CREATE key [DISK | MEMORY] [CACHESIZE bytes] [MINRECORD bytes] [MAXRECORD bytes] [MAXKEYLEN bytes] [PAGESIZE bytes] +``` + +#### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `DISK` / `MEMORY` | `DISK` | Storage backend | +| `CACHESIZE` | Library default | Circular buffer size in bytes (hot-data cache for DISK; total capacity for MEMORY) | +| `MINRECORD` | Library default | Minimum record size in the circular buffer | +| `MAXRECORD` | Library default | Maximum record size in the circular buffer | +| `MAXKEYLEN` | Library default | Maximum key (field) length in bytes | +| `PAGESIZE` | Auto-computed | Leaf page size (auto-computed from MAXRECORD if not specified) | + +#### Examples + +```bash +RI.CREATE myindex DISK CACHESIZE 67108864 MAXKEYLEN 64 +RI.CREATE myindex MEMORY CACHESIZE 16777216 MINRECORD 8 MAXRECORD 4096 +RI.CREATE myindex DISK +``` + +#### Resp Reply + +Simple string reply: `OK` on success, or error if the key already exists or configuration is invalid. + +--- + +### RI.EXISTS + +Check if a key is a Range Index. + +#### Syntax + +```bash +RI.EXISTS key +``` + +Returns `1` if the key exists and is a Range Index, `0` otherwise (including if the key is a different type). + +#### Resp Reply + +Integer reply: `1` or `0`. + +--- + +### RI.CONFIG + +Return the configuration of a Range Index. + +#### Syntax + +```bash +RI.CONFIG key +``` + +#### Resp Reply + +Array reply: alternating field-name and value pairs: + +``` +1) "storage_backend" +2) "DISK" +3) "cache_size" +4) "67108864" +5) "min_record_size" +6) "8" +7) "max_record_size" +8) "4096" +9) "max_key_len" +10) "64" +11) "leaf_page_size" +12) "4096" +``` + +Returns a WRONGTYPE error if the key is not a Range Index. + +--- + +### RI.METRICS + +Return runtime metrics for a Range Index. + +#### Syntax + +```bash +RI.METRICS key +``` + +#### Resp Reply + +Array reply: alternating metric-name and value pairs. Available metrics depend on the BfTree native library. + +Returns a WRONGTYPE error if the key is not a Range Index. + +--- + +### DEL / UNLINK + +Deleting a Range Index uses the standard `DEL` or `UNLINK` commands. The underlying BfTree is freed automatically. + +```bash +DEL myindex +UNLINK myindex +``` + +--- + +## Write Commands + +### RI.SET + +Insert or update a field-value entry in the Range Index. + +#### Syntax + +```bash +RI.SET key field value +``` + +#### Examples + +```bash +RI.SET myindex "user:1001" "Alice" +RI.SET myindex "emp:042" "Bob,Engineering,L5" +``` + +#### Resp Reply + +Simple string reply: `OK` on success, or error if the key/value size violates the index constraints. + +--- + +### RI.DEL + +Delete a field from the Range Index. + +#### Syntax + +```bash +RI.DEL key field +``` + +#### Examples + +```bash +RI.DEL myindex "user:1001" +``` + +#### Resp Reply + +Integer reply: `:1` if the field was deleted, `:0` if not found. + +--- + +## Read Commands + +### RI.GET + +Read a single field from the Range Index. + +#### Syntax + +```bash +RI.GET key field +``` + +#### Examples + +```bash +RI.GET myindex "user:1001" +``` + +#### Resp Reply + +Bulk string reply: the value associated with the field, or nil if not found. + +--- + +## Scan / Range Query Commands + +These are the core differentiating commands that leverage Bf-Tree's range scan capability. + +### RI.SCAN + +Scan entries starting at a key, returning up to `COUNT` entries in lexicographic order. + +#### Syntax + +```bash +RI.SCAN key start COUNT n [FIELDS KEY | VALUE | BOTH] +``` + +#### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `COUNT n` | Required | Maximum number of entries to return | +| `FIELDS` | `BOTH` | What to return: `KEY` (keys only), `VALUE` (values only), or `BOTH` (key-value pairs) | + +#### Examples + +```bash +RI.SCAN myindex "user:1000" COUNT 10 +RI.SCAN myindex "a" COUNT 100 FIELDS KEY +RI.SCAN myindex "" COUNT 50 FIELDS VALUE +``` + +#### Resp Reply + +With `FIELDS BOTH` (default): Array of 2-element arrays `[key, value]`. + +``` +1) 1) "user:1000" + 2) "Alice" +2) 1) "user:1001" + 2) "Bob" +``` + +With `FIELDS KEY`: Array of bulk strings (keys only). + +With `FIELDS VALUE`: Array of bulk strings (values only). + +#### Errors + +- `-ERR memory-only mode does not support scan` if the index uses the MEMORY backend. + +--- + +### RI.RANGE + +Scan all entries in the closed range `[start, end]`. + +#### Syntax + +```bash +RI.RANGE key start end [FIELDS KEY | VALUE | BOTH] +``` + +#### Examples + +```bash +RI.RANGE myindex "user:1000" "user:2000" +RI.RANGE myindex "a" "z" FIELDS KEY +RI.RANGE myindex "emp:001" "emp:100" FIELDS BOTH +``` + +#### Resp Reply + +Same format as `RI.SCAN`. + +--- + +## Type Safety + +Range Index keys are type-safe: + +- **`GET`** / **`SET`** on a Range Index key returns a `WRONGTYPE` error. +- **`RI.*`** commands on a non-Range-Index key return a `WRONGTYPE` error. +- **`DEL`** / **`UNLINK`** / **`TYPE`** work on any key type, including Range Index. +- **`TYPE`** on a Range Index key returns `rangeindex`. + +--- + +## Example Session + +```bash +> RI.CREATE r1 DISK CACHESIZE 33554432 MINRECORD 8 ++OK + +> RI.SET r1 "emp:001" "Alice,Engineering,L5" ++OK + +> RI.SET r1 "emp:002" "Bob,Sales,L3" ++OK + +> RI.SET r1 "emp:010" "Charlie,Engineering,L7" ++OK + +> RI.GET r1 "emp:002" +"Bob,Sales,L3" + +> RI.SCAN r1 "emp:001" COUNT 2 FIELDS BOTH +1) 1) "emp:001" + 2) "Alice,Engineering,L5" +2) 1) "emp:002" + 2) "Bob,Sales,L3" + +> RI.RANGE r1 "emp:001" "emp:010" FIELDS KEY +1) "emp:001" +2) "emp:002" +3) "emp:010" + +> RI.DEL r1 "emp:002" +(integer) 1 + +> RI.EXISTS r1 +(integer) 1 + +> TYPE r1 +rangeindex + +> DEL r1 +(integer) 1 + +> RI.EXISTS r1 +(integer) 0 +``` + +--- + +## Configuration + +Enable Range Index with the server flag: + +```bash +garnet-server --enable-range-index-preview +``` + +Or in `garnet.conf`: + +``` +EnableRangeIndexPreview true +``` + +Range Index is disabled by default. When disabled, all `RI.*` commands return an error. diff --git a/website/docs/commands/raw-string.md b/website/docs/commands/raw-string.md index 4d8f99c019f..4174e09f45e 100644 --- a/website/docs/commands/raw-string.md +++ b/website/docs/commands/raw-string.md @@ -316,7 +316,7 @@ Simple string reply: OK. #### Syntax ```bash - SET key value [NX | XX] [GET] [EX seconds | PX milliseconds] [KEEPTTL] [WITHETAG] + SET key value [NX | XX] [GET] [EX seconds | PX milliseconds] [KEEPTTL] ``` Set **key** to hold the string value. If key already holds a value, it is overwritten, regardless of its type. Any previous time to live associated with the **key** is discarded on successful SET operation. @@ -328,7 +328,6 @@ Set **key** to hold the string value. If key already holds a value, it is overwr * NX -- Only set the key if it does not already exist. * XX -- Only set the key if it already exists. * KEEPTTL -- Retain the time to live associated with the key. -* WITHETAG -- Adding this option sets the Key Value pair with an initial ETag, if called on an existing key value pair with an ETag, this command will update the ETag transparently. This is a Garnet specific command, you can read more about ETag support [here](../commands/garnet-specific-commands#native-etag-support). WITHETAG and GET options cannot be sent at the same time. #### Resp Reply @@ -338,7 +337,6 @@ Any of the following: * Simple string reply: OK. GET not given: The key was set. * Nil reply: GET given: The key didn't exist before the SET. * Bulk string reply: GET given: The previous value of the key. -* Integer reply: WITHETAG given: The ETag either created on the value, or the updated Etag. --- diff --git a/website/docs/dev/cluster/replication.md b/website/docs/dev/cluster/replication.md new file mode 100644 index 00000000000..b3c32a599eb --- /dev/null +++ b/website/docs/dev/cluster/replication.md @@ -0,0 +1,247 @@ +--- +id: replication-dev +sidebar_label: Replication +title: Replication Overview +--- + +# Garnet Replication Overview + +Garnet cluster mode allows users to setup a replication stream by assigning certain nodes of the cluster to be replicas of a single primary. +The replicas are configured by default to serve only reads, redirecting any write request to their primary. +Replicas server only a single primary but any primary can have multiple replicas. +Replicas aim to be an exact copy of the primary by receiving and replaying individual operations through log shipping. +For this reason, the nodes exercising replication need to be setup with the AOF feature enabled. + +# Garnet Replication Attach/Sync Workflow + +Every node in the cluster starts as a primary node and it can be assigned to be a replica by using either CLUSTER REPLICATE or REPLICAOF commands. +When this command is issued to a specific node, depending on how the node is configured (i.e. using diskless or disk-based replication), it will follow +a synchronization workflow according to the following steps: + +1. Replica initiates attach and sends its local latest checkpoint and AOF information to the primary +2. The primary will decide if the a checkpoint needs to be shipped to the replica. For diskless replication, the primary will execute a streaming checkpoint only if the replica does not have enough data to partially synchronize using its local AOF. +3. The primary signals to the replica to recover from its latest checkpoint and replay its local AOF. +4. After recovery the replica, signals back to the primary to let it know that it is ready to start receiving the AOF pages not contained in the checkpoint that it recovered from. +5. A background AofSyncTask is spawned to start iterating from the address beyond the one that is covered from the replicas recovered checkpoint. + +The primary maintains one distinct AofSyncTask per replica, keeping track the pages it has send. + +## Replication Options + +Users can configure Garnet replication by adjusting the AOF parameters in garnet options as well as a number of other options. +The most important options are shown below: +- EnableAOF: Enables AOF +- AofMemorySize: Sets the total size of the AOF memory buffer after which any newly append records will spill to disk +- AofPageSize: Determines the AOF page size +- FastAofTruncate: This option is used to aggressively shift the AOF address begin address in order to prevent the AOF from spilling to disk which could result in performance degradation. This happens in safe manner and is a best effort approach, which means data will be not lost and may spill to disk if there is delay is shipping the log pages to the replica (which will prevent the begin address from shifting ahead of the sync iterator for that replica). U +- UseAofNullDevice: Treat the AOF memory buffer a circular buffer for writing AOF records, ensuring no disk IO. This can be used in combination with fast-aof-truncate but could lead to potential data loss. One should adjust the AofMemorySize to ensure there is enough space for incoming AOF records to be written and shipped towards the replica before truncating. +- CommitFrequencyMs: Write ahead logging (append-only file) commit issue frequency in milliseconds. 0 = issue an immediate commit per operation, -1 = manually issue commits using COMMITAOF command. To avoid performance degradation when running with FastAofTruncate and/or UseAofNullDevice, one can set the CommitFrequencyMs to -1. +- AofReplicationRefreshFrequencyMs: "AOF replication (safe tail address) refresh frequency in milliseconds. 0 = auto refresh after every enqueue. +- ReplicaDisklessSync: Enable diskless sync to avoid disk write amplification when performing full synchronization when a new replica attaches. This options uses the streaming checkpoint feature to ship the store keys to the replica when full synchronization is required. +- ReplicaDisklessSyncDelay: How long to wait between sync requests to allow for other replicas to attach in parallel hence amortize the cost of the streaming checkpoint. +- OnDemandCheckpoint: This options enables taking an on demand checkpoint when a replica attaches. It used to improve the synchronization performance in the event the AOF log has grown too large or when the AOF has been truncated and synchronization requires a fresh checkpoint to correctly synchronize with the replica. If this option is disabled and the AOF log gets truncated, not having enabled this option could break the attach process unless UseAofNullDevice or FastAofTruncate is enabled since in that case we expect data loss. +- AllowDataLoss (UseAofNullDevice || (FastAofTruncate && !OnDemandCheckpoint): This is an internal flag computed by base flags as indicated. It is used to skip AOF integrity check when an AOF sync task is started. Using this combination of options should be used with caution + +## Diskbased Attach/Sync Details + +The diskbased attach sync workflow is implemented in ```ReplicaDiskBasedSync.cs``` and is used both for attaching through CLUSTER REPLICATE, REPLICAOF and on startup recovery of a given node. +Before signaling the primary to start the attach/sync workflow, the node resets its state to prepare for synchronization. +This includes the following +1. Reset replay tasks if the node was already a replica and now is being assigned to a new primary +2. Reset replication offset value +3. Reset any active AOF sync tasks, if the node was a primary before being turned to a replica. +4. Reset/flush database to empty to avoid any synchronization conflicts +Every replica transmits its persistence data to the corresponding primary and waits for the attach process to complete. +If the connection breaks or an error is returned from the primary, the process releases any locks, resets the replica back to being a primary and responds to the caller with an error message. + +When using diskbased replication, every replica attaching to a given primary transmits its latest checkpoint information and the begin and end addresses of its own AOF log. +A checkpoint is identified by its version number, a replication-id and the minimum AOF address it covers. +The primary uses that information to decide if the replica requires partial or full synchronization by comparing it to the latest available valid checkpoint. +From the primary's perspective a valid checkpoint is one for which the current begin AOF address is less or equal to the checkpoint covered AOF address. +In addition to that, the checkpoint must be of the same version and history for the nodes involved. +When the OnDemandCheckpoint flag is used the primary might initiate the process of taking a new checkpoint. +Any on-demand checkpoint can be shared across attaching replicas if it is still valid at the moment those replicas attach. +When a new checkpoint is created, we make a best effort approach to delete older checkpoints at the primary. +This approach requires a locking mechanism to ensure that actively read checkpoints (those part of the full synchronization) will not be deleted +The locking logic is for checkpoints is implemented in ```CheckpointStore.cs```. +The attaching replica communicates to the corresponding primary which creates a ```ReplicaSyncSession.cs``` for every attaching replica. +By examining the metadata of the replica the primary decides if a full or partial synchronization is needed +If full synchronization is necessary the primary will send the latest checkpoint files to the replica in chunks. +It will then signal the replica to recover its latest checkpoint and replay the AOF log if necessary. +For partial synchronization, the primary signals the replica to recover from its local checkpoint skipping the step for sending the latest checkpoint +When the recovery is complete, the primary will initiate a permanent background AofSyncTask by establishing an iterator over its own AOF, starting from the checkpoint covered AOF address. +At startup of the AOF sync task, we validate the AOF integrity unless the nodes are configured in such a way where data loss is inevitable (see AllowDataLoss). +Integrity validation is required to ensure that the start address requested by the replica has not been truncated and the AOF sync task can start streaming the AOF records to the replica from. + +## Diskless Attach/Sync Details + +Diskless synchronization works similar to the diskbased approach. +Its major difference is that it does not require a disk checkpoint. +It leverages the Streaming Checkpoint primitive to scan and transmit the kv pairs from the underlying Tsavorite store. +The diskless attach/sync workflow is implemented at the primary within ```ReplicaSyncManager.cs```. +The replica side implementation is implemented within ```ReplicaDisklessSync.cs``` +The attaching replica transmit their persistence information (i.e. AOF start and tail address and store version). +As opposed to the disk-based approach, the attaching replicas are grouped and synced together. +There is no limit on the number of replicas that can be synced in parallel. +The only parameter that controls how many replicas are synced in parallel is ```ReplicaDisklessSyncDelay``` which delays replication sync to allow more replicas to sync together (i.e. at startup where a primary might need to be configured with few replicas). +Once the specified delay period passes the primary will examine all the metadata transmitted by the associated replicas and decide which ones require full vs partial synchronization. +Those requiring partial synchronization will be released immediately and a new AOF sync task will be created for them to start receiving the associated AOF records +The rest will be fully synchronized using the StreamingCheckpoint primitive. +Before starting the full synchronization, the primary broadcasts FLUSHDB command to cleanup the replica store so there is no conflicts with the primary store. +This streaming checkpoint primitive utilizes and iterator over the TsavoriteStore and it broadcasts batches of kv pairs to the replica. +The replica will receive those pairs and insert them into its store. +At completion the replica sets its version number to be equal the the version number of primary. +Finally, the primary executes the partial synchronization workflow which includes steps to validate the integrity of the AOF and the creation of the AOF sync tasks to start streaming the corresponding AOF records to each replica. + +# Sharded Append-Only-File Feature + +Garnet replication leverages the Append-Only-File (AOF) implementation to stream update operations to the corresponding replica. +The Garnet's AOF implementation uses a single instance of TsavoriteLog to record update operations as they occur at the primary. +Writing, streaming and replaying the AOF in order to support replication is single threaded operation +This is in contrast to Garnet's native multi-threaded architecture and does not scale well. + +This motivated the development of a sharded AOF implementation that leverages multiple physical sublogs (i.e. separate TsavoriteLog instances) to scale writes at the primary and parallel replay at the replica. +This implementation works alongside a read consistency protocol running at the replica, which is required to guarantees prefix consistent reads because sublog replay happen asynchronously potentially exposing non-prefix consistent content. +The read consistency protocol relies on virtual timestamps (i.e. sequence numbers) to indicate write order across sublogs. +These timestamps are used to ensure prefix consistency per Garnet session. + +## Sharded AOF architecture + +Garnet can be configured to use the sharded AOF implementation by adjusting the following configuration parameters; +1. AofPhysicalSublogCount: + This parameter controls the number of TsavoriteLog instances used by the GarnetAppendOnlyFile implementation. + Its value ranges between 1 and 64, with 1 being the default value which maps to the legacy single log implementation +2. AofRefreshPhysicalSublogTailFrequencyMs: + This parameter control the background refresh tail task that spawns only when the Garnet instance is configured to use more than physical sublogs. + This task is required to keep moving time forward for sublogs that are not being actively written, in order to ensure the consistency protocol works correctly. +3. AofReplayTaskCount: + This parameter controls the number of replay tasks that can used per physical sublog. + Its value ranges between 1 and 256, with the default value being 1. The combination of this default value and the AofPhysicalSublogCount default maps to the legacy single log implementation. + +### GarnetAppendOnlyFile + +This class implements Garnet's AOF offering an API to interact with the physical +sublog instances and ensure read prefix consistency. +Its most important members are +1. SequenceNumberGenerator + This class implements the API used to generate sequence numbers when the +2. ReadConsistencyManager: + Responsible for tracking the replayed key sequence numbers per virtual sublog and coordinating read operation to ensure read prefix consistency +3. GarnetLog + This class implements the API associated with operating and managing a TsavoriteLog instance but extends it to seamlessly use either a single or multiple TsavoriteLog instances depending on how the Garnet instance is configured. + +#### SequenceNumberGenerator + +The `SequenceNumberGenerator` class is implemented by using a `baseTimestamp` and a startingOffset. +Sequence numbers are generated using the difference of the baseTimestamp from the current timestamp offseted by the `startingOffset`. +The starting offset is used to eliminate clock divergence between nodes and on recovery it is initialized as the maximum sequence number calculated from the records of recovered AOF. +Note recovery can happen on startup or when a failover occurs where a replica takes over as primary making it so it needs to generate consistent sequence numbers for writes that is going to serve in the future. + +#### ReadConsistencyManager + +This `ReadConsistencyManager` class is instantiated when a node becomes a replica. +It is used to track the key sequence numbers of the replayed records. +This happens because replicas needs to ensure read prefix-consistency through tracking the maximum session sequence number seen across reads +and waiting for keys that are behind to become current through the progression of background replay functionality. +This protocol is triggered only when the Garnet cluster node are configured to use the sharded AOF (i.e. AofPhysicalSublogCount > 1 || AofReplayTaskCount > 1). +The `ReadConsistencyManager` uses the `VirtualSublogReplayState` struct to track the key sequence numbers seen for all replay records at a specific point in time. +Since it not efficient to track all keys, it uses a sketch tracking a limited amount of slots to which keys being replayed are matched through hashing. +This an approximation of the actual sequence number per key due to collisions. +However, it does not affect correctness it only incurs additional read latency when key moves ahead in time as a side-effect of overlapping key mappings to the same slot. +In addition to tracking key sequence number per fixed number of slots, each `VirtualSublogReplayState` instance tracks the maximum sequence number across slots and maintains a `TaskCompletionSource` that is signaled when replay progresses, allowing waiting readers to be awakened. + +When a `RespServerSession` processes a read command, it utilizes the `ConsistentReadGarnetApi` (through the `ConsistentReadContext` and `TransactionalConsistentReadContext` for the string and object data types respectively) to call into the `ReadConsistencyManager` and validate that it can serve the read under the prefix consistency constrains. +This happens into two phases per key: + +1. ConsistentReadKeyPrepare Phase + This phase occurs before the actual processing of the corresponding read operation in Tsavorite. + Its goal is to validate the key's freshness compared to `maximumSessionSequenceNumber` as determined by the previous read operations. + Freshness is determined by comparing the frontier sequence number (max of key specific and the virtual sublog's maximum observed value) against the `maximumSessionSequenceNumber`. + The frontier value need to be strictly behind the `maximumSessionSequenceNumber` since we cannot determine the order of writes with the same timestamp. + If this condition holds reads can proceed otherwise the read needs to wait for and advance time event to occur. + This happens every time the `VirtualSublogReplayState` gets updated, which triggers the associated `TaskCompletionSource` to allow for any waiters to re-check the aforementioned condition. +2. ConsistentReadSequenceNumberUpdate step + This phase occurs after the read has been processed. + Its goal is to update the `maximumSessionSequenceNumber` by taking the maximum of the current `maximumSessionSequenceNumber` and the corresponding key's sequence number. + This update happens after read to ensure that we associate the read with a pessimistic replayed sequence number. + +The `ReadConsistencyManager` maintains version number that gets incremented every time a new instance is created. +Creation of a new instance happens at the startup of the attach/sync workflow to ensure that subsequent reads start +with a clean slate. +The read protocol checks maintains the current version number per session and performs the appropriate checks to ensure that every read is consistent in terms of the last seen version number of the `ReadConsistencyManager`. +This happens at `ConsistentReadKeyPrepare` phase before validating the freshness of the key being read. +Read prefix consistency is guaranteed for a given batch of reads because reading a batch happens under epoch protection. +Across requests, the version of the `ReadConsistencyManager` may change with the database version change, so prefix read consistency follows the same rules as if a new database was recovered while a client was connected. + +### GarnetLog + +The `GarnetLog` instance implements an API that offers the same functionality as `TsavoriteLog` extended to support operations across multiple instances (`ShardedLog`). +The functionality that had to be extended to support multiple `TsavoriteLog` instances is as follows: +1. Metadata Operations + These include operations that retrieve `TsavoriteLog` metadata such as `BeginAddress`, `TailAddress` etc properties. For `ShardedLog`, these operations return a vector of address in a form of a struct defined as `AofAddress`. +2. Initialize Operations + These operations require initializing the state of the log and often require an input int the form of an address. For `ShardedLog`, the parameter passed is a value of `AofAddress`. +3. Commit Operations + These operations happens in unison across each sublog instance. For variants that require awaiting on a Task, the `ShardedLog` implementation uses the WheAll primitive for all tasks associated with each sublog. +4. Truncate Operations + For `ShardedLog`, truncation happens by passing an `AofAddress` parameter and applying the truncation based on the index of the sublog being truncated. +5. Enqueue Operations + Enqueue operations are performed either by providing the index of the log to enqueue to or through computation of the index by hashing the key associated with that record. + The first approach is used for coordinated operations that are applied and replayed across multiple physical logs such as transactions or checkpoint. + The second approach is used to record individual operations to the store's key value pair, specifically Upsert, RMW and Delete operations are recorded through this API for both string and object data. + +The `GarnetLog` instance offers also a locking mechanism to atomically insert record headers for coordinated operations. This locking mechanism works by preventing enqueue for coordinated operations that run in parallel. +This is used for transaction replay to avoid a deadlock when two transactions operate on overlapping sublogs and at commit the need to coordinate using a barrier to ensure prefix consistency. + + +### AOFSyncDriver + +As mentioned previously, at completion of the attach workflow, the primary will spawn background tasks responsible for shipping the AOF pages to the replica. +For every connected replica, the primary creates an instance of an `AofSyncDriver`, which manages the `AofSyncTask` responsible for shipping each physical sublog's pages. +The `ReplicationManager` manages the `AofSynDriverStore` containing all the `AofSyncDriver` instances. +The number of `AofSyncTask` instance spawned is equal to the number of physical sublogs configured on the corresponding Garnet instance. +When a Garnet instance is configured with more than one physical sublog, a refresh tail task is also created. + +#### Advance Physical Sublog Time + +When using the sharded log feature, a background task is created for each replica connection to signal time advancement for the underlying physical sublogs. This is essential when a physical sublog receives no writes—a condition that is opaque to the replica side and indistinguishable from log-shipping or replay delays. Without this signal, readers may wait indefinitely for data to arrive, significantly increasing read tail latency. + +Consider this example: given key-sequence number pairs (A,t1) and (B,t2) mapped to different physical logs with t1 < t2, if we read B first and then attempt to read A, the read session will wait for A to reach sequence number t3 > t2. + +The advance time background task periodically checks for new writes and captures a tail snapshot containing all physical sublog tails, associating them with a sequence number. It then sends this information to the replica, which updates its replayed max sequence number at the ReadConsistencyManager. + +The replica maintains its own background consumer task to process incoming signals as they arrive. The replica waits until its background replay reaches at least the provided tail snapshot before updating the max sequence number for that sublog. Since sequence number updates are monotonic, they can be applied multiple times, allowing the consumer to converge eventually toward the snapshot tail. + +# Transaction Replay + +Garnet supports two distinct types of transactions, each with different behaviors and requirements: + +1. **MULTI-EXEC transactions** — These follow the standard RESP protocol and allow users to dynamically declare operations at runtime, accumulating them within a transaction and executing them atomically when the EXEC command is issued. + +2. **Custom transaction procedures** — These are server-side procedures defined programmatically at compile time and registered with the Garnet server, providing a more structured alternative for complex multi-key operations. + +## Replay Behavior During Replication + +The replay behavior differs between these two transaction types, particularly when replicating across multiple physical sublogs. + +**For MULTI-EXEC transactions:** The replica must gather and buffer all individual operations sequentially until it encounters the associated transaction commit marker before initiating the actual replay of all buffered operations. + +**For custom transaction procedures:** Replay can begin immediately upon encountering the custom transaction body record, which itself acts as the commit marker and contains the complete transaction definition. + +## Single-Log Replay + +When operating with a single AOF log, both transaction types employ the same fundamental replay mechanism: the replay process encounters the commit marker, acquires the necessary locks for all keys specified in the transaction, and then executes the associated operations in sequence. This approach is straightforward since all operations are naturally ordered within a single sequential log. + +## Multi-Log Replay + +When operating with multiple physical sublogs, both transaction types must perform an additional synchronization step to maintain strict isolation guarantees. This is necessary because the keys involved in a transaction may be distributed across multiple physical sublogs, and each sublog is replayed independently by its own background task. The core challenge is ensuring that all transaction operations complete across all involved sublogs before allowing subsequent operations to proceed. + +The replay behavior diverges at this point: + +- **MULTI-EXEC transactions** allow for parallel replay of the operations encountered across different physical sublogs. +This method of parallel replay is subject to the constraint that the transaction cannot complete until all operations across all sublogs have finished to maintain isolation. + +- **Custom transaction procedures** replay is currently restricted by the fact that the body of the transaction is programmatically defined, disallowing parallel operation replay. +Hence, only a single designated replay background task is used to execute the entire transaction body sequentially. +The remaining participating replay tasks suspend their replay operation until the main replay tasks completes. + diff --git a/website/docs/dev/onboarding.md b/website/docs/dev/onboarding.md index b78fbe76660..b2a927c7472 100644 --- a/website/docs/dev/onboarding.md +++ b/website/docs/dev/onboarding.md @@ -126,14 +126,7 @@ Any new feature, change to existing functionality or bug fixing needs to be done ``/branch-name -3. Include Unit Tests for any new commands or feature. Allure enabled tests are required. - - Full documentation about Allure can be found [here](https://allurereport.org/docs/). - - Each test class must: - * Apply [AllureNUnit] custom attribute - * Apply [TestFixture] attribute - * Inherit from the AllureTestBase base class +3. Include Unit Tests for any new commands or feature. 4. Once it is ready for review, create a [Pull Request](https://github.com/microsoft/Garnet/pulls). Make sure to link it to your issue item in the development section. diff --git a/website/docs/dev/post_filter_design.md b/website/docs/dev/post_filter_design.md deleted file mode 100644 index 0781a02342a..00000000000 --- a/website/docs/dev/post_filter_design.md +++ /dev/null @@ -1,513 +0,0 @@ -# VSIM Post-Filter — Design Document - -## Overview - -The VSIM post-filter evaluates user-supplied filter expressions (e.g. `.year > 1980 and .genre == "action"`) against vector search candidate results. It runs in the hot path of every filtered vector similarity query. - -**Key design constraint:** Zero heap allocation in the per-candidate evaluation loop. All buffers are borrowed from the session-local `ScratchBufferBuilder` (~9 KB), a pinned `byte[]` that persists for the session's lifetime. After the first VSIM FILTER query, the buffer is already large enough — subsequent calls have zero allocation cost and zero GC pressure. - ---- - -## High-Level Architecture - -``` - VSIM SEARCH REQUEST - ┌─────────────────────────────────────────────────────────────────────┐ - │ VSIM.SEARCH myindex 3 FILTER ".year > 1980 and .genre == 'action'"│ - └──────────────────────────────────┬──────────────────────────────────┘ - │ - ▼ - ┌──────────────────────────────────────────────────────────────────────┐ - │ Vector Index (DiskANN) │ - │ Returns top-K nearest neighbors by vector distance │ - │ e.g. 100 candidates for K=3 (over-fetch to allow post-filtering) │ - └──────────────────────────────────┬──────────────────────────────────┘ - │ candidates + JSON attributes - ▼ - ┌──────────────────────────────────────────────────────────────────────┐ - │ ApplyPostFilter (this pipeline) │ - │ │ - │ Stage 1: COMPILE filter → postfix program (once per query) │ - │ Stage 2: COLLECT unique field selectors (once per query) │ - │ Stage 3: For each candidate: (×N, zero alloc) │ - │ EXTRACT fields → EVALUATE program → SET bitmap bit │ - └──────────────────────────────────┬──────────────────────────────────┘ - │ filterBitmap (packed bit array) - ▼ - ┌──────────────────────────────────────────────────────────────────────┐ - │ Return top-K results that passed the filter │ - └──────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Pipeline Stages - -### Stage 1: Compile (ExprCompiler) - -Transforms the filter expression UTF-8 bytes into a flat postfix instruction array using the shunting-yard algorithm. - -``` - Input: ".year > 1980 and .genre == \"action\"" (raw UTF-8 bytes) - - Phase 1 — Tokenize: - [SEL:year] [NUM:1980] [OP:>] [OP:and] [SEL:genre] [STR:"action"] [OP:==] - - Phase 2 — Shunting-yard (infix → postfix): - [SEL:year] [NUM:1980] [OP:Gt] [SEL:genre] [STR:"action"] [OP:Eq] [OP:And] -``` - -**Key properties:** -- Zero heap allocation — compiler uses caller-provided Span buffers (borrowed from session scratch buffer) for all scratch and output -- String/selector tokens store `(offset, length)` byte-range references into the original filter span — no copies -- Numbers parsed via `Utf8Parser.TryParse` directly from UTF-8 bytes -- Booleans normalized to `double`: `true` → 1.0, `false` → 0.0 - -### Stage 2: Selector Collection (GetSelectorRanges) - -Scans the compiled instructions to find unique field selectors. - -``` - Instructions: [SEL:year] [NUM:1980] [OP:Gt] [SEL:genre] [STR:"action"] [OP:Eq] [OP:And] - ↓ ↓ - selectorRanges: [(offset=1, len=4), (offset=23, len=5)] - "year" "genre" -``` - -Deduplication ensures each field is extracted only once, even if referenced multiple times in the filter. - -### Stage 3: Per-Candidate Evaluation (×N, zero alloc) - -For each candidate JSON document: - -``` - ┌─────────────────────────────────────────────────────────────────────┐ - │ 3a. EXTRACT — AttributeExtractor.ExtractFields │ - │ │ - │ JSON: {"year":1980,"rating":4.5,"genre":"action",...} │ - │ ↓ ↓ │ - │ extractedFields[0] = ExprToken{Num=1980.0} ← .year │ - │ extractedFields[1] = ExprToken{Str=(32,6)} ← .genre │ - │ │ - │ Single-pass scan: walks JSON once, extracts ALL needed fields. │ - │ String values are zero-copy (offset, length) into JSON bytes. │ - │ Early exit when all requested fields are found. │ - ├─────────────────────────────────────────────────────────────────────┤ - │ 3b. EVALUATE — ExprRunner.Run │ - │ │ - │ Walk postfix instructions left-to-right: │ - │ │ - │ Instruction Stack (after) │ - │ ──────────────── ────────────────────────────────── │ - │ [SEL:year] [1980.0] ← lookup in extractedFields │ - │ [NUM:1980] [1980.0, 1980.0] │ - │ [OP:Gt] [0.0] ← 1980 > 1980 = false │ - │ [SEL:genre] [0.0, "action"] │ - │ [STR:"action"] [0.0, "action", "action"] │ - │ [OP:Eq] [0.0, 1.0] ← "action"=="action"= true │ - │ [OP:And] [0.0] ← false AND true = false │ - │ │ - │ Top-of-stack = 0.0 → candidate EXCLUDED │ - ├─────────────────────────────────────────────────────────────────────┤ - │ 3c. RECORD — set filterBitmap[i/8] |= (1 << (i%8)) if passed │ - └─────────────────────────────────────────────────────────────────────┘ -``` - ---- - -## Memory Architecture - -### Design Principle: Caller Owns All Memory - -``` - VectorManager.Filter (ApplyPostFilter) - │ - │ scratch = ActiveThreadSession.scratchBufferBuilder - │ poolSlice = scratch.CreateArgSlice(560 × 16 bytes) - │ selectorSlice = scratch.CreateArgSlice(32 × 8 bytes) - │ Cast to Span and Span<(int,int)> via MemoryMarshal - │ Slice into sub-spans: - │ instrBuf, tuplePoolBuf, tokensBuf, opsStackBuf, - │ runtimePoolBuf, extractedFields, stackBuf - │ - ├──→ ExprCompiler.TryCompile(filter, instrBuf, tuplePoolBuf, - │ tokensBuf, opsStackBuf, ...) - │ │ - │ │ writes into instrBuf, tuplePoolBuf - │ │ uses tokensBuf, opsStackBuf as scratch - │ │ OWNS NOTHING — pure function over borrowed Spans - │ │ - │ ←───────┘ - │ - ├──→ ExprProgram (ref struct) - │ │ - │ │ Instructions = instrBuf[..instrCount] ← Span slice - │ │ TuplePool = tuplePoolBuf[..tupleCount]← Span slice - │ │ RuntimePool = runtimePoolBuf ← Span - │ │ OWNS NOTHING — just bundles Span references - │ │ - ├──→ AttributeExtractor.ExtractFields(...) - │ │ OWNS NOTHING - │ │ - ├──→ ExprRunner.Run(...) - │ │ OWNS NOTHING - │ │ - └──→ finally: scratch.RewindScratchBuffer(ref selectorSlice) - scratch.RewindScratchBuffer(ref poolSlice) ← LIFO -``` - -### Buffer Layout — Session Scratch Buffer - -``` - ScratchBufferBuilder (session-local pinned byte[]) - ┌──────────────────────────────────────────────────────────────┐ - │ poolSlice = CreateArgSlice(560 × 16 = 8,960 bytes): │ - │ │ - │ ┌──────────────────────────────────────────────┐ │ - │ │ instrBuf 128 × 16 B = 2,048 B │ ← compiled│ - │ │ tuplePoolBuf 64 × 16 B = 1,024 B │ output │ - │ ├──────────────────────────────────────────────┤ │ - │ │ tokensBuf 128 × 16 B = 2,048 B │ ← compiler│ - │ │ opsStackBuf 128 × 16 B = 2,048 B │ scratch │ - │ ├──────────────────────────────────────────────┤ │ - │ │ runtimePoolBuf 64 × 16 B = 1,024 B │ ← runtime │ - │ │ extractedFields 32 × 16 B = 512 B │ data │ - │ │ stackBuf 16 × 16 B = 256 B │ │ - │ └──────────────────────────────────────────────┘ │ - │ │ - │ selectorSlice = CreateArgSlice(32 × 8 = 256 bytes): │ - │ ┌──────────────────────────────────────────────┐ │ - │ │ selectorBuf 32 × 8 B = 256 B │ │ - │ └──────────────────────────────────────────────┘ │ - │ │ - │ On the stack: only ExprProgram + ExprStack ref structs │ - │ and local variables (~64 B). │ - └──────────────────────────────────────────────────────────────┘ - - Cleanup: RewindScratchBuffer in LIFO order in the finally block. - The scratch buffer is never freed — it persists for the session's lifetime. - After the first VSIM FILTER query, it's already large enough. -``` - -**Why ScratchBufferBuilder instead of ArrayPool or stackalloc?** -- **Session-local, already warm** — the pinned `byte[]` is already allocated and in cache - from RESP command parsing earlier in the same session -- **Zero contention** — no shared pool lock (even lock-free CAS has overhead under - high thread counts); scratch buffer is strictly thread-local -- **Zero stack pressure** — no risk of stack overflow from ~9 KB of stackalloc -- **Ref safety** — C# ref safety rules (CS8350/CS8352) prohibit mixing stackalloc'd - spans with heap-backed spans when passing them to methods that take `ref struct` - parameters. Using the scratch buffer avoids this entirely. -- **Consistent with Garnet idiom** — other Vector Set operations (FetchAttributes, etc.) - already use the same `ScratchBufferBuilder` pattern - -### ExprToken — The Universal Data Type (16 bytes) - -Every value in the system — numbers, strings, operators, selectors, tuples, nulls — is represented as a single 16-byte `ExprToken` struct, laid out as a tagged union: - -``` - ExprToken — [StructLayout(LayoutKind.Explicit, Size = 16)] - - Byte offset: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - ┌────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┬────┐ - │Type│ Op │Flag│ padding (5 bytes) │ Union payload (8 bytes) │ - │Code│Code│ s │ │ │ - └────┴────┴────┴────────────────────────────┴───────────────────────────────────┘ - ◄─── header (8 bytes) ──────────────────────►◄─── payload (8 bytes) ───────────► - - The payload at offset 8 is a UNION (overlapping fields): - ● double Num [8..15] — 8 bytes, for numbers and booleans - ● int Utf8Start [8..11] — 4 bytes ┐ for strings, selectors, tuples - ● int Utf8Length [12..15] — 4 bytes ┘ -``` - -#### TokenType — The Discriminator (byte 0) - -The `TokenType` at byte 0 is the **discriminator tag** that tells the system which kind of value this token represents and which payload fields are valid. Without it, the union payload is ambiguous — the same 8 bytes could be a `double` or a pair of `int`s. - -``` - ExprTokenType enum (1 byte): - - Value Name What it represents Payload used - ───── ───────── ─────────────────────────────────── ───────────────────────── - 0 None Default/uninitialized (all zeros) (none — sentinel value) - 1 Num Numeric literal or boolean double Num - 2 Str String value (byte-range reference) int Utf8Start + Utf8Length - 3 Tuple Collection of values (for IN op) int poolStart + count - 4 Selector JSON field name (e.g. .year) int Utf8Start + Utf8Length - 5 Op Operator (e.g. >, ==, and) OpCode in byte 1 - 6 Null JSON null or missing value (none) -``` - -**Why we need it:** -- The postfix instruction array is a flat `Span`. The evaluator walks it left-to-right and must know how to handle each token: push a value? look up a field? execute an operation? -- `TokenType` answers that question in a single byte comparison — no virtual dispatch, no type checks, no polymorphism overhead. -- The `None` value (0) is critical: since all stackalloc buffers start as zeros, any uninitialized slot is automatically `None`, which is detectable via `token.IsNone`. - -#### OpCode — The Operator Identity (byte 1) - -The `OpCode` at byte 1 identifies which operation to perform when `TokenType == Op`. It's only meaningful for operator tokens — for all other token types, byte 1 is unused. - -``` - OpCode enum (1 byte): - - Value Name Arity Prec Meaning Example - ───── ─────── ───── ──── ───────────────────── ──────────────────── - 0 Or 2 0 Logical OR .a > 1 or .b > 2 - 1 And 2 1 Logical AND .a > 1 and .b > 2 - 2 Gt 2 2 Greater than .year > 1980 - 3 Gte 2 2 Greater or equal .rating >= 4.0 - 4 Lt 2 2 Less than .year < 2000 - 5 Lte 2 2 Less or equal .rating <= 5.0 - 6 Eq 2 2 Equality .genre == "action" - 7 Neq 2 2 Not equal .genre != "drama" - 8 In 2 2 Membership / contains "x" in .tags - 9 Add 2 3 Addition .year + 10 - 10 Sub 2 3 Subtraction .year - 5 - 11 Mul 2 4 Multiplication .rating * 2 - 12 Div 2 4 Division .budget / 1000000 - 13 Mod 2 4 Modulo .year % 100 - 14 Pow 2 5 Power (right-assoc) (.year - 2000) ** 2 - 15 Not 1 6 Logical NOT not (.deleted) - 16 OParen 0 7 Open parenthesis ( ← compile-time only - 17 CParen 0 7 Close parenthesis ) ← compile-time only -``` - -**Why we need it:** - -1. **During compilation (shunting-yard):** The `OpCode` determines **precedence** and **associativity**. The shunting-yard algorithm uses precedence to decide when to pop operators from the stack to the output. For example, `*` (prec 4) binds tighter than `+` (prec 3), so `.rating * 2 + 1` correctly compiles to `[.rating] [2] [*] [1] [+]`. - -2. **During evaluation:** The `OpCode` tells `ExecuteInstruction` which operation to perform. The **arity** (1 or 2) determines how many operands to pop from the evaluation stack. For example, `Not` pops 1 operand, `And` pops 2. - -3. **OParen/CParen are compile-time only:** They exist in the `OpCode` enum so the shunting-yard algorithm can push/pop them, but they are never emitted to the output instruction array. If a `CParen` is found in the output, it means unbalanced parentheses — compile error. - -#### How TokenType and OpCode Work Together - -``` - Example: .year > 1980 and .genre == "action" - - Compiled postfix instruction array (7 ExprTokens, 112 bytes): - - Index TokenType OpCode Payload What happens at eval time - ───── ────────── ──────── ─────────────────── ────────────────────────── - 0 Selector (unused) Utf8Start=1, Len=4 Push extractedFields["year"] - 1 Num (unused) Num=1980.0 Push 1980.0 - 2 Op Gt (unused) Pop 2, push (1980 > 1980 = 0.0) - 3 Selector (unused) Utf8Start=22, Len=5 Push extractedFields["genre"] - 4 Str (unused) Utf8Start=31, Len=6 Push ref to "action" in filter - 5 Op Eq (unused) Pop 2, push ("action"=="action" = 1.0) - 6 Op And (unused) Pop 2, push (0.0 AND 1.0 = 0.0) - - → Top of stack = 0.0 → candidate EXCLUDED -``` - -The evaluator's dispatch is a two-level check: -1. **Check TokenType** — is this a value to push, a selector to look up, or an operator to execute? -2. **If Op, check OpCode** — which specific operation? How many operands? - -This is intentionally flat and branchless-friendly. No virtual method tables, no interface dispatch — just byte comparisons and a switch statement that the JIT can optimize into a jump table. - -**Flags byte (offset 2) — 3 bits used:** - -| Bit | Mask | Name | Meaning | -|-----|------|------|---------| -| 0 | `0x01` | HasEscape | String contains `\"`, `\\`, `\n` — needs unescape-aware comparison | -| 1 | `0x02` | FilterOrigin | Byte range references the filter buffer, not the JSON buffer | -| 2 | `0x04` | RuntimeTuple | Tuple elements are in RuntimePool (from JSON), not TuplePool (from `[...]` literal) | -| 3–7 | — | Reserved | Available for future use | - -**Payload by token type:** - -| TokenType | Payload | Source buffer | -|-----------|---------|---------------| -| Num | `double Num` (booleans: 1.0/0.0) | — | -| Str | `(Utf8Start, Utf8Length)` byte range | JSON bytes (default), or filter bytes if FilterOrigin | -| Selector | `(Utf8Start, Utf8Length)` field name | filter bytes (by convention; FilterOrigin is NOT set — identified by TokenType) | -| Tuple | `(poolStart, count)` into pool | TuplePool (compile-time) or RuntimePool (if RuntimeTuple) | -| Op | `OpCode` enum in header byte 1 | — | -| Null | (no payload) | — | -| None | all zeros — default/sentinel | — | - -**Why 16 bytes?** -- Power-of-two: array indexing is `index << 4` (single shift instruction) -- Cache-friendly: 4 tokens fit in one 64-byte cache line -- The `double` payload requires 8-byte alignment -- No managed references → safe for `stackalloc`, `Span`, and contiguous buffers - -### ExprProgram — Zero-Allocation View (ref struct) - -`ExprProgram` is a `ref struct` that bundles `Span` references into slices of the caller's pooled array. It owns no memory — it's a view. - -```csharp -internal ref struct ExprProgram -{ - public Span Instructions; // → instrBuf[..instrCount] - public int Length; - public Span TuplePool; // → tuplePoolBuf[..tupleCount] - public int TuplePoolLength; - public Span RuntimePool; // → runtimePoolBuf - public int RuntimePoolLength; -} -``` - -Because it's a `ref struct`, it can only live on the stack — the compiler enforces this. It cannot be stored in a field, captured by a lambda, or boxed. - -### ExprStack — Zero-Allocation Evaluation Stack (ref struct) - -```csharp -internal ref struct ExprStack -{ - private readonly Span _buffer; // → stackBuf - private int _count; - - public bool IsFull; // bounds check helper - public bool TryPush(ExprToken t); // bounds-checked, returns false on overflow - public ExprToken Pop(); - public ExprToken Peek(); - public void Clear(); -} -``` - -### Zero-Copy String Handling - -Strings are never allocated or copied. Instead, `ExprToken` stores `(offset, length)` byte-range references: - -``` - Filter: ".genre == \"action\"" - 0123456789... - ↑ ↑ - Selector token: Utf8Start=1, Utf8Length=5 → "genre" (into filter bytes) - String literal: Utf8Start=11, Utf8Length=6 → "action" (into filter bytes, FilterOrigin=true) - - JSON: {"genre":"action","year":1980} - 0123456789... - ↑ ↑ - Extracted string: Utf8Start=9, Utf8Length=6 → "action" (into JSON bytes, FilterOrigin=false) -``` - -When comparing, `ExprRunner.GetStrSpan` resolves the correct source buffer: - -```csharp -return t.IsFilterOrigin - ? filterBytes.Slice(t.Utf8Start, t.Utf8Length) - : json.Slice(t.Utf8Start, t.Utf8Length); -``` - -For strings without escapes: `SequenceEqual` (JIT-vectorized with SIMD in .NET 8+). -For strings with escapes (`HasEscape` flag): `UnescapedEquals` (on-the-fly unescape, still zero allocation). - ---- - -## Overflow Safety - -Every buffer write is bounds-checked. The system never crashes on pathological input — it fails gracefully: - -| Buffer | Capacity | On overflow | -|--------|----------|-------------| -| `instrBuf` | 128 tokens | `TryCompile` returns -1 → all candidates filtered out | -| `tuplePoolBuf` | 64 elements | `TryCompile` returns -1 → compile error | -| `tokensBuf` | 128 tokens | `TryCompile` returns -1 → compile error | -| `opsStackBuf` | 128 entries | `ProcessOperator` returns false → compile error | -| `runtimePoolBuf` | 64 elements | `ParseArrayToken` returns Null → array skipped | -| `selectorBuf` | 32 selectors | Extra selectors silently not collected | -| `extractedFields` | 32 fields | Sliced to actual selector count (max 32) | -| `stackBuf` | 16 depth | `TryPush` returns false → candidate excluded | - ---- - -## Component Responsibilities - -| Component | Responsibilities | Owns memory? | -|-----------|-----------------|--------------| -| `VectorManager.Filter` | borrow scratch buffer, slice into sub-spans, orchestrate pipeline, rewind on exit | **Yes** — single owner via ScratchBufferBuilder | -| `ExprCompiler` | tokenize + shunting-yard → postfix instructions | **No** — writes to caller's spans | -| `ExprProgram` | bundle Span references into a convenient struct | **No** — ref struct, just a view | -| `ExprRunner` | walk instructions, evaluate postfix program | **No** — reads program + stack spans | -| `AttributeExtractor` | single-pass JSON field extraction | **No** — writes to caller's spans | -| `ExprStack` | push/pop evaluation stack | **No** — ref struct over caller's span | -| `ExprToken` | universal 16-byte tagged union data type | **No** — blittable value type | - ---- - -## VSIM EF Parameters and How They Drive the Filter Pipeline - -Three `EF` parameters control how many candidates the post-filter processes: - -``` - VADD key FP32 element EF 200 - ^^^^^^ - Build exploration factor — controls index quality at insert time. - - VSIM key FP32 element COUNT 10 EF 100 FILTER ".year > 1980" FILTER-EF 200 - ^^^^^^ ^^^^^^^^^^^ - Search EF — beam width Filter EF — extra effort - during graph traversal. to compensate for filtering. -``` - -### Parameter Details - -| Parameter | Command | Type | Default | Range | What it controls | -|-----------|---------|------|---------|-------|-----------------| -| `EF` | `VADD` | int | index-dependent | > 0 | Build exploration factor. How many neighbors the index probes when inserting a new vector. Higher = better graph quality, slower inserts. | -| `EF` | `VSIM` | int | index-dependent | ≥ 0 | Search exploration factor (beam width). How many candidates the graph algorithm visits during search. Must be ≥ COUNT. Higher = better recall, more compute. | -| `FILTER-EF` | `VSIM` | int | COUNT×200 | ≥ 0 | How many candidates to retrieve upfront when FILTER is specified. The index over-fetches this many candidates in a single search pass, then post-filter runs once. No retry loop. | - -### How EF and FILTER-EF Affect the Post-Filter - -When a `FILTER` is specified, `FILTER-EF` controls how many candidates the index retrieves -in a **single, upfront over-fetch** — there is no retry loop. - -``` - VSIM myindex FP32 element COUNT 10 EF 50 FILTER ".genre == 'action'" FILTER-EF 200 - - Without FILTER: - retrieveCount = COUNT = 10 - effectiveEF = EF = 50 - → Index returns 10 candidates, no post-filter. - - With FILTER: - retrieveCount = FILTER-EF = 200 ← over-fetch upfront - effectiveEF = max(EF, FILTER-EF) = 200 ← widen graph search - → Index returns up to 200 candidates in ONE search. - → ApplyPostFilter runs ONCE on all 200 candidates. - → Enough pass the filter to fill COUNT=10. -``` - -``` - ┌──────────────────────────────────────────────────────────────────────┐ - │ Step 1: Graph search with effectiveEF = max(EF, FILTER-EF) = 200 │ - │ │ - │ Index traverses the graph with beam width 200. │ - │ Returns up to FILTER-EF (200) candidates sorted by distance. │ - │ │ - │ Result: e.g. 200 candidates │ - └──────────────────────┬───────────────────────────────────────────────┘ - │ - ▼ - ┌──────────────────────────────────────────────────────────────────────┐ - │ Step 2: ApplyPostFilter — single pass, zero GC allocation │ - │ │ - │ Evaluates ".genre == 'action'" against all 200 candidates. │ - │ Uses ~9 KB from session scratch buffer (contiguous, cache-warm). │ - │ │ - │ Result: e.g. 60 out of 200 pass → filterBitmap with 60 bits set │ - │ Caller picks top COUNT=10 by distance from the passing set. │ - └──────────────────────────────────────────────────────────────────────┘ - - Note: there is NO retry loop. The index search and post-filter each - run exactly once. FILTER-EF simply inflates the initial retrieval. - Default FILTER-EF = COUNT * 200 (e.g. COUNT=10 → FILTER-EF=2000). -``` - -### EF/FILTER-EF vs Post-Filter Cost - -| Scenario | Candidates evaluated | Pool usage | Heap alloc | -|----------|---------------------|------------|------------| -| COUNT=10, EF=50, no FILTER | 0 (no filter) | 0 B | 0 B | -| COUNT=10, EF=50, FILTER, FILTER-EF=100 | up to 100 | ~9 KB (pooled) | **0 B** | -| COUNT=10, EF=100, FILTER, FILTER-EF=200 | up to 200 | ~9 KB (pooled) | **0 B** | -| COUNT=10, EF=50, FILTER (default) | up to 2000 (COUNT×200) | ~9 KB (pooled) | **0 B** | -| COUNT=100, EF=500, FILTER, FILTER-EF=5000 | up to 5000 | ~9 KB (pooled) | **0 B** | - -The scratch buffer usage is constant (~9 KB) regardless of how many candidates are evaluated — the buffers are borrowed once and reused across all candidates in the loop. The session's scratch buffer persists for its lifetime, so repeated queries reuse the same physical memory. - diff --git a/website/docs/dev/range-index-resp-api.md b/website/docs/dev/range-index-resp-api.md new file mode 100644 index 00000000000..f471d65b963 --- /dev/null +++ b/website/docs/dev/range-index-resp-api.md @@ -0,0 +1,2891 @@ +# Integrating Range Index (Bf-Tree) as a Garnet Data Type + +## Summary + +Proposal for a RESP protocol API that exposes [Bf-Tree](https://github.com/microsoft/bf-tree) as a `RangeIndex` data type, analogous to how Garnet exposes SortedSets via `Z*` commands. The server hosts a key-value cache where keys are RangeIndex names (e.g., `r1`, `r2`) and values are `BfTree` instances. + +All commands follow the `RI.*` prefix convention (short for **R**ange**I**ndex). + +--- + +## Implementation Status + +> **Note:** The implementation plan sections below were written before implementation and contain +> some stale details (e.g., 51-byte stubs with `ProcessInstanceId`, `ResumePostRecovery()`). +> This section documents the **actual** implemented design. Refer to the RESP API specification +> (Section 1–5) and the code for authoritative details. + +### Stub Design (actual: 35 bytes) + +The `ProcessInstanceId` Guid was removed — stale pointers are handled by `OnDiskRead` zeroing +the `TreeHandle`. The stub is 35 bytes: + +| Offset | Size | Field | Description | +|--------|------|-------|-------------| +| 0 | 8 | `TreeHandle` | Native pointer to live BfTreeService (zeroed on disk read) | +| 8 | 8 | `CacheSize` | Circular buffer size | +| 16 | 4 | `MinRecordSize` | Min record size | +| 20 | 4 | `MaxRecordSize` | Max record size | +| 24 | 4 | `MaxKeyLen` | Max key length | +| 28 | 4 | `LeafPageSize` | Leaf page size | +| 32 | 1 | `StorageBackend` | 0=Disk, 1=Memory | +| 33 | 1 | `Flags` | bit 0: Flushed, bit 1: Recovered | +| 34 | 1 | `SerializationPhase` | Reserved | + +### IRecordTriggers Lifecycle (implemented) + +All lifecycle callbacks go through the `IRecordTriggers` interface in Tsavorite, implemented +by `GarnetRecordTriggers`: + +| Trigger | When | What it does for RangeIndex | +|---------|------|---------------------------| +| `OnFlush(addr)` | Page moves to read-only | Branch on `stub.TreeHandle`: `!=0` (live) → `BfTreeService.CprSnapshotByPtr(handle)` (concurrent-safe with workers via CPR) + `File.Copy(scratch.cpr → .flush.bftree)`; `==0` (cold, just-CAS'd at tail) → S-lock to block RestoreTree from registering mid-copy; if a live tree exists under another stub for this key → CPR snapshot via that handle, else `File.Copy(data.bftree → .flush.bftree)`. Set `IsFlushed`. **No per-key X-lock taken.** Per-tree atomic (`TreeEntry.SnapshotInProgress`) serializes against concurrent checkpoint snapshot. | +| `OnEvict` | Page evicted past HeadAddress | Remove entry from `liveIndexes` under per-key exclusive lock; **defer** `bfTree.Dispose()` via `storeEpoch.BumpCurrentEpoch(...)` so concurrent readers using `TreeHandle` complete before native free. Data files preserved for lazy restore | +| `OnDiskRead` | Record loaded from disk | Zero `TreeHandle` (native pointer is stale); no file work | +| `PostCopyToTail` | After `TryCopyToTail` CAS, before unseal | Propagate `RecordType=RangeIndexRecordType` from src to dst (CTT does not carry it). Branch on `src.TreeHandle`: `!=0` live transfer (clear src.TreeHandle); `==0` cold pre-stage (`PreStageAndRegisterPending`). Set src.`IsTransferred` so a later eviction of src does not free dst's tree / pending entry. Clear `IsFlushed` on dst. | +| `OnTruncate(newBA)` | After device truncate | Delete `..flush.bftree` files where `addr < newBA`. | +| `OnCheckpoint(VersionShift)` | PREPARE→IN_PROGRESS | Set checkpoint barrier; mark all entries (activated AND pending) `SnapshotPending=1` | +| `OnCheckpoint(FlushBegin)` | WAIT_FLUSH | Snapshot trees: activated → per-tree atomic + `BfTreeService.CprSnapshotByPtr(handle)` + `File.Copy(scratch.cpr → snapshot path)`; pending → `File.Copy(data.bftree → snapshot path)`. Clear barrier. **No per-key X-lock taken.** | +| `OnCheckpoint(CheckpointCompleted)` | REST | No-op — Tsavorite removes per-token snapshot dirs when `removeOutdated=true`; per-flush files cleaned by `OnTruncate` | +| `OnRecovery(token)` | Before snapshot file recovery | Store recovered checkpoint token (used by `RebuildFromSnapshotIfPending`) | +| `OnRecoverySnapshotRead` | Per record from snapshot file | Set `IsRecovered`; pre-stage `data.bftree` from `cpr-checkpoints//rangeindex/.bftree` and register pending entry (snapshot files may be deleted post-recovery) | +| `OnDispose(Deleted)` | DEL/UNLINK | Remove entry from `liveIndexes` under per-key exclusive lock; **defer** `bfTree.Dispose()` + `data.bftree` + `scratch.cpr` deletion via `storeEpoch.BumpCurrentEpoch(...)`. Per-flush snapshot files (`..flush.bftree`) are preserved (LOG-tied; cleaned by `OnTruncate` once BeginAddress passes their addr). | + +### File Layout (two roots) + +RangeIndex files live under **two roots**, mirroring Tsavorite's separation of log state +(`hlog`) from checkpoint state (`cpr-checkpoints`): + +**Log-tied root** — `{LogDir ?? CheckpointDir ?? cwd}/Store/rangeindex/` +Co-located with `hlog.` when storage tier is enabled. Falls back through the same chain +that Tsavorite uses for `CheckpointBaseDirectory` so RangeIndex works without storage tier. +Lifetime tracks log addresses (cleared by `OnTruncate(newBA)`). + +``` +{riLogRoot}/ + .data.bftree # bftree's working file (disk-backed) / cold-restore staging (memory-backed) + .scratch.cpr # bftree's CPR snapshot scratch path (overwritten each cpr_snapshot) + ..flush.bftree # immutable per-flush snapshot +``` + +**Checkpoint-tied root** — `{CheckpointBaseDirectory}/Store/checkpoints[_dbId]/cpr-checkpoints/` +Per-checkpoint snapshots live alongside Tsavorite's per-token files. Deleted automatically +when Tsavorite removes the parent token directory (via `removeOutdated`). + +``` +{cprDir}// + info.dat # Tsavorite (existing) + snapshot.dat # Tsavorite (existing) + snapshot.obj.dat # Tsavorite (existing) + rangeindex/ # NEW + .bftree # one per liveIndexes entry at checkpoint time +``` + +`` is a 32-character lowercase-hex `Guid("N")` derived from `XxHash128(keyBytes)` — +same scheme used to identify entries in `liveIndexes` (`KeyId(keyBytes)`). + +### liveIndexes — single dictionary, Guid-keyed + +`liveIndexes: ConcurrentDictionary` keyed by `KeyId(keyBytes) = +new Guid(XxHash128.Hash(keyBytes))`. Entries can be: + +- **Activated** (`Tree != null`): a live native BfTree exists; ops go through `stub.TreeHandle` + on the hot path (no `liveIndexes` lookup needed). +- **Pending** (`Tree == null`): `data.bftree` on disk has correct content but no native tree + has been opened yet. Awaiting `RestoreTree` activation. Tracked so checkpoint snapshots + capture the key's state. + +Pending entries are registered by: +- `PreStageAndRegisterPending` — called from `PostCopyToTail`-cold (compaction with disk + source) and RIPROMOTE `PostCopyUpdater`-cold (post-eviction promote where + `src.TreeHandle == 0`). Pre-stages `data.bftree` from `.flush.bftree` + via `File.Copy(overwrite: true)` under the per-key X-lock (which serializes against any + concurrent `RestoreTree` on the same key). +- `RebuildFromSnapshotIfPending` — called from `OnRecoverySnapshotRead` for above-FUA-at- + checkpoint stubs. Atomically pre-stages `data.bftree` from + `cpr-checkpoints//rangeindex/.bftree`. **MUST run during recovery** + because the snapshot file may be deleted post-recovery. + +### Discipline: liveIndexes is never on the hot path (except for checkpoint coord) + +`liveIndexes` is consulted on the hot path ONLY by `WaitForTreeCheckpoint`, which is gated +by the `checkpointInProgress` short-circuit (one volatile bool read in steady state). +All other code uses `stub.TreeHandle` directly. + +### Lazy Promote (Flush → Tail) + +When `ReadRangeIndex` detects `IsFlushed`: +1. Release shared lock +2. Issue `RIPROMOTE` RMW — `CopyUpdater` copies stub to tail, clears `IsFlushed` +3. `PostCopyUpdater` branches on `src.TreeHandle`: + - **`!= 0` (live transfer)**: clear src.TreeHandle (existing behavior). dst inherits the + handle via byte-copy. + - **`== 0` (cold case)**: pre-stage `data.bftree` from `.flush.bftree` + (using `rmwInfo.SourceAddress`) and register pending entry. Handles steady-state cold + restore, recovery Scenario D (below-FUA-at-checkpoint stub recovered + pure-read + access post-recovery), and any other path that promotes a flushed stub with + `TreeHandle == 0`. +4. Retry — stub is now in mutable region + +### Lazy Restore (Activation of Pending Entries) + +When `ReadRangeIndex` detects `TreeHandle == 0` (and the stub is not flushed): +1. Release shared lock +2. Acquire **exclusive** lock (prevents concurrent restores) +3. Re-read stub — if another thread already set `TreeHandle`, return +4. Recover bftree via `BfTreeService.RecoverFromCprSnapshot(data.bftree, scratch.cpr, backend)` + — pre-staging always happened earlier (`PostCopyToTail`-cold, RIPROMOTE-cold, or + `OnRecoverySnapshotRead`). Same API for disk-backed and memory-backed; bftree allocates + its own buffer for memory-backed when no `buffer_ptr` is supplied. +5. Register/activate in `liveIndexes` (upgrades a pending entry if present) +6. **Release exclusive lock**, then issue `RIRESTORE` RMW to set new `TreeHandle` + — split lock from RMW so a deferred `OnFlush` (cold-case S-lock) cannot self-deadlock + against an X-lock held over a Tsavorite operation +7. Retry + +### Checkpoint Consistency + +- At `VersionShift`: `SnapshotPending=1` set on **all** entries (activated + pending). +- At `FlushBegin`: each entry is snapshotted using the per-tree atomic + `TreeEntry.SnapshotInProgress` (no per-key X-lock — that would risk deadlock with + deferred `OnFlush` firing on the checkpoint thread): + - Activated → `BfTreeService.CprSnapshotByPtr(handle)` (concurrent-safe with workers via + CPR; serialized against concurrent `OnFlush` for the same tree by the per-tree atomic) + + `File.Copy(/.scratch.cpr → //rangeindex/.bftree)`. + - Pending → `File.Copy(/.data.bftree → //rangeindex/.bftree)`. +- Entries created during checkpoint enumeration (after the barrier) have + `SnapshotPending=0` and are skipped — they belong to v+1. +- Per-checkpoint snapshots are removed automatically when Tsavorite deletes the parent + `cpr-checkpoints//` directory (no separate `PurgeOldCheckpointSnapshots` needed). +- Per-flush snapshots are removed by `OnTruncate(newBA)` when the log advances past their + address. + +### Deadlock Safety + +The integration is built around three invariants: + +1. **OnFlush MUST NOT acquire any per-key X-lock from a code path that may run as a deferred + epoch action.** `OnFlush` may fire from `Drain` on a thread holding any RI lock (e.g., + a hot-path reader's S-lock). The per-key X-lock walks all shards via `CalculateIndex`, + which collides with any S-lock the same thread already holds → self-deadlock. The cold + case uses a per-key **shared** RI lock instead (S-vs-S compatible across threads; + blocks RestoreTree's X-lock without self-deadlock). +2. **Per-tree atomic** (`TreeEntry.SnapshotInProgress`) serializes `OnFlush` against + `SnapshotAllTreesForCheckpoint` for the same tree. Bf-tree's internal CPR coordinator + would otherwise no-op one of two concurrent calls. +3. **Deferred disposal** via `storeEpoch.BumpCurrentEpoch(...)`: `bfTree.Dispose()` and + file deletion run only after every reader that observed the tree's `TreeHandle` has + moved past. This protects readers using `stub.TreeHandle` from concurrent DEL/eviction. + +Compatibility note: `EnableRangeIndexPreview=true` is incompatible with +`CopyReadsToTail=true`. Under `CopyReadsToTail`, `ReadRangeIndex` holds a per-key shared +RI lock during `Read_MainStore`, which can synchronously trigger +`ConditionalCopyToTail → PostCopyToTail-cold → PreStageAndRegisterPending` — that path +attempts the per-key X-lock and self-deadlocks against the shared lock the same thread is +still holding. Server startup fails fast with a clear error if both are enabled. + +### Recovery Flow + +1. Recovery reads checkpoint snapshot files into the main log. +2. `OnRecovery(token)` stores `recoveredCheckpointToken` for use by step 4. +3. `OnDiskRead` zeros `TreeHandle` on every record loaded from disk (stale pointer). +4. `OnRecoverySnapshotRead` for above-FUA-at-checkpoint RI stubs: calls + `RebuildFromSnapshotIfPending` which atomically copies + `//rangeindex/.bftree` → `/.data.bftree` + and registers a pending entry. **Must happen during recovery** because the snapshot file + is removed when Tsavorite deletes the parent token directory. +5. Below-FUA-at-checkpoint stubs (`IsFlushed=1`) are NOT pre-staged at recovery; they're + handled lazily by RIPROMOTE `PostCopyUpdater`-cold on first access (which uses + `.flush.bftree`, the immutable per-flush snapshot). + +### Compaction Lifecycle + +When compaction copies an RI stub from `[BeginAddress, untilAddress)` to the tail: +1. Source record is read from memory or disk via the scan iterator. If from disk, + `OnDiskRead` invalidates `TreeHandle`. +2. `TryCopyToTail` allocates dst at the tail, byte-copies the stub, CAS-inserts. +3. `PostCopyToTail` fires post-CAS: + - Propagates `RecordType=RangeIndexRecordType` from src to dst (CTT does not carry it). + - `src.TreeHandle != 0` (live transfer): clear src.TreeHandle. liveIndexes entry exists. + - `src.TreeHandle == 0` (cold): `PreStageAndRegisterPending(dstKey, srcLogicalAddress)` + copies `.flush.bftree` → `data.bftree` under the per-key X-lock, + and registers a pending entry. + - Sets src.`IsTransferred` so a later eviction of src does not free dst's tree / pending entry. + - Clears dst.`IsFlushed`. +4. dst is unsealed by `CASRecordIntoChain`; subsequent RI ops find the new stub at the tail. + +After compaction completes, `Log.Truncate()` (or a checkpoint commit) advances `BeginAddress` +past the compacted range. `OnTruncate(newBA)` fires after device truncation completes and +deletes per-flush files whose `addr < newBA`. + +### Scratchpad: Key Plumbing for PostCopyToTail + +To support `PostCopyToTail`, the source logical address must be plumbed: +- **Compaction path**: `CompactionConditionalCopyToTail(currentAddress, ...)` sets + `pendingContext.originalAddress = currentAddress`. +- **CopyReadsToTail path**: `InternalRead.CopyFromImmutable` sets + `pendingContext.originalAddress = stackCtx.recSrc.LogicalAddress`. +- **ContinuePending path**: sets `pendingContext.originalAddress = request.logicalAddress` + (the disk-resolved source address, set by `AsyncGetFromDiskCallback`). + +`TryCopyToTail.PostCopyToTail` reads source address from +`stackCtx.recSrc.HasMainLogSrc ? stackCtx.recSrc.LogicalAddress : pendingContext.originalAddress`. + +For RIPROMOTE `PostCopyUpdater`, the source address is plumbed via +`RMWInfo.SourceAddress` (a new field) which is set in `InternalRMW` before NeedCopyUpdate +and preserved through `CopyUpdater` (since `RMWInfo.Address` is reassigned to dst by then). + +### AOF Logging + +RI.SET and RI.DEL operate on the native BfTree outside Tsavorite's RMW path. After each +successful operation, a synthetic no-op RMW is injected to trigger AOF logging. On AOF replay, +`AofProcessor.StoreRMW` detects RISET/RIDEL commands and routes them to +`RangeIndexManager.HandleRangeIndexSetReplay/DelReplay`, which re-executes the BfTree operation. + +### Type Safety (WRONGTYPE) + +- `ReadMethods`: rejects non-RI commands on RI keys and RI commands on non-RI keys +- `RMWMethods`: same bidirectional checks in `InPlaceUpdater` +- `UpsertMethods`: `InPlaceWriter` rejects SET on RI/Vector stubs (`UpsertAction.WrongType`) +- `TYPE` command returns `"rangeindex"` for RI keys + +### Implemented Commands + +| Command | Status | +|---------|--------| +| RI.CREATE | ✅ Implemented | +| RI.SET | ✅ Implemented + AOF | +| RI.GET | ✅ Implemented | +| RI.DEL | ✅ Implemented + AOF | +| RI.SCAN | ✅ Implemented | +| RI.RANGE | ✅ Implemented | +| RI.EXISTS | ✅ Implemented | +| RI.CONFIG | ✅ Implemented | +| RI.METRICS | ✅ Implemented | +| DEL/UNLINK | ✅ Works via OnDispose | +| TYPE | ✅ Returns "rangeindex" | +| RI.MSET / RI.MGET / RI.MDEL | ❌ Not yet implemented | +| RI.KEYS | ❌ Not yet implemented | +| Cluster replication | ❌ Future work | +| Key migration | ❌ Future work | + +--- + +### 1. Lifecycle / Management Commands + +| Command | Syntax | Description | Maps to | +|---|---|---|---| +| **RI.CREATE** | `RI.CREATE key [options...]` | Create a new RangeIndex. Options allow tuning the underlying BfTree config | `BfTree::new()` / `BfTree::with_config()` | +| **RI.EXISTS** | `RI.EXISTS key` | Check if a RangeIndex exists. Returns `1` or `0` | Cache lookup | +| **RI.CONFIG** | `RI.CONFIG key` | Return current config of the RangeIndex as key-value pairs | `BfTree::config()` | +| **RI.METRICS** | `RI.METRICS key` | Return buffer/tree metrics (JSON) | `BfTree::get_buffer_metrics()` / `get_metrics()` | + +Deletion of RangeIndex keys uses the standard `DEL` / `UNLINK` commands. The store's `DisposeRecord` callback detects the RangeIndex `RecordType`, snapshots the BfTree pointer from the stub, and frees it — no special drop command is needed. + +Snapshot and restore are handled automatically by the cache checkpointing mechanism. + +#### `RI.CREATE` options + +``` +RI.CREATE myindex + [DISK | MEMORY] + [CACHESIZE bytes] + [MINRECORD bytes] + [MAXRECORD bytes] + [MAXKEYLEN bytes] + [PAGESIZE bytes] +``` + +**Storage backends:** + +- **`DISK`** (default) — Disk-backed tree. Base pages are stored in a data file on + disk at a **deterministic path** derived from the key bytes: + `{dataDir}/rangeindex/{XxHash128(key)}/data.bftree`. The circular buffer (`CACHESIZE`) + acts as a hot-data cache. No data loss on eviction. Total capacity is limited by disk + space. Supports all operations including scan. Snapshot and recovery use the tree's + own data file. +- **`MEMORY`** — Memory-only tree (maps to bf-tree's `cache_only` mode). All data + lives in the circular buffer. Total capacity is bounded by `CACHESIZE`. Scan + operations are **not supported**. Snapshot and recovery will be supported in a + future bf-tree release; Garnet will snapshot/recover memory-only trees the same way + as disk-backed trees once bf-tree adds this capability. + +**Examples:** +``` +RI.CREATE r1 DISK +RI.CREATE r1 DISK CACHESIZE 67108864 MAXKEYLEN 64 +RI.CREATE r1 MEMORY CACHESIZE 16777216 MINRECORD 8 MAXRECORD 4096 +``` + +**Reply:** `+OK` or `-ERR ` + +--- + +### 2. Write Commands + +| Command | Syntax | Description | Maps to | +|---|---|---|---| +| **RI.SET** | `RI.SET key field value` | Insert or update a key-value entry in the RangeIndex | `BfTree::insert(key, value)` | +| **RI.DEL** | `RI.DEL key field` | Delete an entry from the RangeIndex | `BfTree::delete(key)` | +| **RI.MSET** | `RI.MSET key field1 value1 [field2 value2 ...]` | Batch insert multiple entries | Multiple `BfTree::insert()` | +| **RI.MDEL** | `RI.MDEL key field1 [field2 ...]` | Batch delete multiple entries | Multiple `BfTree::delete()` | + +**Terminology:** `key` is the RangeIndex name; `field` is the entry key within the BfTree; `value` is the entry value. + +#### `RI.SET` +``` +RI.SET r1 "user:1001" "Alice" +``` +**Reply:** `+OK` on success, `-ERR ` on `InvalidKV` + +#### `RI.DEL` +``` +RI.DEL r1 "user:1001" +``` +**Reply:** `:1` (deleted) or `:0` (not found) + +#### `RI.MSET` +``` +RI.MSET r1 "user:1001" "Alice" "user:1002" "Bob" "user:1003" "Charlie" +``` +**Reply:** `:3` (number of entries inserted) + +#### `RI.MDEL` +``` +RI.MDEL r1 "user:1001" "user:1002" +``` +**Reply:** `:2` (number of entries deleted) + +--- + +### 3. Read Commands + +| Command | Syntax | Description | Maps to | +|---|---|---|---| +| **RI.GET** | `RI.GET key field` | Read a single entry | `BfTree::read(key, buffer)` | +| **RI.MGET** | `RI.MGET key field1 [field2 ...]` | Read multiple entries | Multiple `BfTree::read()` | + +#### `RI.GET` +``` +RI.GET r1 "user:1001" +``` +**Reply:** `$5\r\nAlice\r\n` (bulk string) or `$-1` (nil, if not found/deleted) + +#### `RI.MGET` +``` +RI.MGET r1 "user:1001" "user:1002" "user:9999" +``` +**Reply:** Array of bulk strings (nil for missing entries): +``` +*3\r\n$5\r\nAlice\r\n$3\r\nBob\r\n$-1\r\n +``` + +--- + +### 4. Scan / Range Query Commands + +These are the core differentiating commands that leverage Bf-Tree's range scan capability. + +| Command | Syntax | Description | Maps to | +|---|---|---|---| +| **RI.SCAN** | `RI.SCAN key start COUNT n [FIELDS KEY\|VALUE\|BOTH]` | Scan `n` entries starting from `start` key | `BfTree::scan_with_count()` | +| **RI.RANGE** | `RI.RANGE key start end [FIELDS KEY\|VALUE\|BOTH]` | Scan entries in `[start, end]` range | `BfTree::scan_with_end_key()` | + +#### `RI.SCAN` + +Scans `n` entries starting at `start` key (inclusive), ordered by key bytes. + +``` +RI.SCAN r1 "user:1000" COUNT 10 +RI.SCAN r1 "user:1000" COUNT 10 FIELDS KEY +RI.SCAN r1 "user:1000" COUNT 10 FIELDS VALUE +RI.SCAN r1 "user:1000" COUNT 10 FIELDS BOTH +``` + +**Default `FIELDS`:** `BOTH` (returns key-value pairs) + +**Reply** (with `FIELDS BOTH`, default): +``` +*4 +*2 +$9 +user:1000 +$5 +Alice +*2 +$9 +user:1001 +$3 +Bob +... +``` +Each element is a 2-element array `[key, value]`. + +**Reply** (with `FIELDS KEY`): +Array of bulk strings (keys only). + +**Reply** (with `FIELDS VALUE`): +Array of bulk strings (values only). + +**Errors:** +- `-ERR invalid count` if count is 0 +- `-ERR invalid start key` if key is empty or too long +- `-ERR memory-only mode does not support scan` if the index is memory-only + +#### `RI.RANGE` + +Scans all entries in the closed range `[start, end]`. + +``` +RI.RANGE r1 "user:1000" "user:2000" +RI.RANGE r1 "user:1000" "user:2000" FIELDS KEY +RI.RANGE r1 "a" "z" FIELDS BOTH +``` + +Same reply format as `RI.SCAN`. + +**Errors:** +- `-ERR invalid start key` +- `-ERR invalid end key` +- `-ERR start key must be <= end key` + +--- + +### 5. Utility Commands + +| Command | Syntax | Description | +|---|---|---| +| **RI.KEYS** | `RI.KEYS [pattern]` | List all RangeIndex names (optionally filtered by glob pattern) | +| **RI.PING** | `RI.PING [message]` | Health check, returns `PONG` or echoes message | + +--- + +## RESP Protocol Details + +All commands follow [RESP3](https://redis.io/docs/reference/protocol-spec/) conventions: + +- **Simple Strings:** `+OK\r\n` +- **Errors:** `-ERR message\r\n` +- **Integers:** `:N\r\n` +- **Bulk Strings:** `$len\r\ndata\r\n` (nil = `$-1\r\n`) +- **Arrays:** `*count\r\n...` + +Keys and values are transmitted as raw bytes (bulk strings), matching Bf-Tree's `&[u8]` interface. + +--- + +## Mapping Summary: Bf-Tree API → RESP Commands + +| Bf-Tree Method | RESP Command | +|---|---| +| `BfTree::new()` | `RI.CREATE` | +| `BfTree::with_config()` | `RI.CREATE ... [options]` | +| `drop(BfTree)` | `DEL` / `UNLINK` | +| `BfTree::insert(key, value)` | `RI.SET` | +| `BfTree::read(key, buf)` | `RI.GET` | +| `BfTree::delete(key)` | `RI.DEL` | +| `BfTree::scan_with_count()` | `RI.SCAN` | +| `BfTree::scan_with_end_key()` | `RI.RANGE` | +| `BfTree::config()` | `RI.CONFIG` | +| `BfTree::get_buffer_metrics()` | `RI.METRICS` | + +--- + +## Example Session + +``` +> RI.CREATE r1 DISK CACHESIZE 33554432 ++OK + +> RI.SET r1 "emp:001" "Alice,Engineering,L5" ++OK + +> RI.SET r1 "emp:002" "Bob,Sales,L3" ++OK + +> RI.SET r1 "emp:010" "Charlie,Engineering,L7" ++OK + +> RI.GET r1 "emp:002" +$14 +Bob,Sales,L3 + +> RI.SCAN r1 "emp:001" COUNT 2 FIELDS BOTH +*2 +*2 +$7 +emp:001 +$20 +Alice,Engineering,L5 +*2 +$7 +emp:002 +$14 +Bob,Sales,L3 + +> RI.RANGE r1 "emp:001" "emp:010" FIELDS KEY +*3 +$7 +emp:001 +$7 +emp:002 +$7 +emp:010 + +> RI.DEL r1 "emp:002" +:1 + +> DEL r1 +:1 +``` + +--- + +# Implementation Plan: RangeIndex in Garnet + +This plan is designed to be self-contained: it includes enough context, file paths, code +patterns, and reference pointers so that an implementer can work through each step without +needing to rediscover the Garnet architecture. + +## Overview + +RangeIndex is implemented as a **built-in Garnet type stored in the unified store**, +accessed via the **string context**. A small fixed-size struct ("stub") is stored as a +raw-byte value (not a heap object) in Tsavorite's unified store. The stub contains BfTree +configuration metadata, a native pointer (`nint`) to the live BfTree instance, and a +`Guid` process-instance-id for stale-pointer detection after restart. A +`RangeIndexManager` (partial class) owns the BfTree lifecycle outside of Tsavorite. + +RangeIndex supports two **storage backends**, configurable per index via `RI.CREATE`: + +- **Disk-backed** (default) — Bf-Tree stores leaf pages in a data file on disk, with a + circular buffer in memory as a hot-data cache. This is the primary mode for production + use. No data loss on eviction — evicted pages are written to disk. Total capacity is + limited by disk space. Snapshot uses `BfTree::snapshot()` which drains the circular + buffer and writes the index structure to the tree's own data file. Recovery uses + `BfTree::new_from_snapshot(config)` which opens the existing data file and resumes. +- **Memory-only** — Bf-Tree uses a bounded in-memory circular buffer (bf-tree's + `cache_only` mode). Evicted pages are nullified. Total capacity is bounded by + `CACHESIZE`. Scan operations are not supported. Snapshot and recovery are planned + for a future bf-tree release; Garnet will treat memory-only trees identically to + disk-backed trees for persistence once bf-tree adds this capability. Until then, + snapshot/recovery calls throw `NotSupportedException` at the FFI boundary. + +This follows the same "stub-in-store with external data manager" pattern used by +**VectorManager** on the +[`dev`](https://github.com/microsoft/garnet/tree/dev) +dev branch. Implementers should cross-reference that branch for working examples +of each pattern described below. The key difference: VectorSet stores element data inside +Tsavorite (via a separate `VectorSessionFunctions` context), while BfTree manages **all** +its data externally (circular buffer + disk files), so RangeIndex only needs the string +context for the stub — no additional Tsavorite context type is required. + +**Why the string context and not the object context?** +- The Bf-Tree manages its own memory (circular buffer, leaf pages) and disk storage — it + cannot be inlined into Tsavorite's log the way a `HashObject` or `SortedSetObject` can. +- The stub pattern cleanly separates metadata persistence (Tsavorite checkpoint) from + index operations (Bf-Tree). +- A 51-byte fixed-size stub is a natural fit for the string context's inline byte values, + avoiding the overhead of `GarnetObjectBase` serialization. +- The `RecordInfo.ValueIsObject` bit remains `false` for RangeIndex records, distinguishing + them from collection objects. + +--- + +## Proposed File Layout + +The RangeIndex implementation is organized as a partial class (`RangeIndexManager`) split +across multiple files, plus supporting files for RESP handlers, storage session wrappers, +and native interop. Each file mirrors a corresponding VectorManager file on the +[`dev`](https://github.com/microsoft/garnet/tree/dev) +dev branch. + +| # | New File | Reference (`dev`) | Role | +|---|---|---|---| +| 1 | `libs/server/Resp/RangeIndex/RangeIndexManager.cs` | [`VectorManager.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.cs) | Main class: constants, `processInstanceId`, `IsEnabled`, initialization, `TryInsert`/`TryRead`/`TryScan`/`TryRange` methods, `ResumePostRecovery()` | +| 2 | `libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs` | [`VectorManager.Index.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Index.cs) | Stub struct definition, `CreateIndex()`, `ReadIndex()`, `RecreateIndex()` | +| 3 | `libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs` | [`VectorManager.Locking.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Locking.cs) | `ReadRangeIndexLock` ref struct, `ReadRangeIndex()`, `ReadOrCreateRangeIndex()` — shared/exclusive lock management via `ReadOptimizedLock` | +| 4 | `libs/server/Resp/RangeIndex/RangeIndexManager.Cleanup.cs` | [`VectorManager.Cleanup.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Cleanup.cs) | Post-drop async cleanup: background task that scans and removes orphaned data | +| 5 | `libs/server/Resp/RangeIndex/RangeIndexManager.Migration.cs` | [`VectorManager.Migration.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Migration.cs) | *(future)* Replication/migration support | +| 6 | `libs/server/Resp/RangeIndex/RangeIndexManager.Replication.cs` | [`VectorManager.Replication.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Replication.cs) | *(future)* Primary→replica replication | +| 7 | `libs/native/bftree-garnet/BfTreeInterop.csproj` | — | C# interop project: MSBuild cargo target + `ContentWithTargetPath` for native libs | +| 8 | `libs/native/bftree-garnet/BfTreeService.cs` | [`DiskANNService.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/DiskANNService.cs) | High-level managed wrapper for native BfTree library | +| 9 | `libs/native/bftree-garnet/NativeBfTreeMethods.cs` | — | `[LibraryImport]` P/Invoke declarations for `bftree_garnet` native library | +| 10 | `libs/native/bftree-garnet/Cargo.toml` + `src/lib.rs` | [`diskann-garnet`](https://github.com/microsoft/DiskANN/tree/main/diskann-garnet) | Rust FFI wrapper crate: `#[no_mangle] extern "C"` exports over `bf-tree` crate | +| 11 | `libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs` | [`RespServerSessionVectors.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/RespServerSessionVectors.cs) | RESP command handlers (`NetworkRISET`, `NetworkRIGET`, etc.) | +| 12 | `libs/server/Storage/Session/MainStore/RangeIndexOps.cs` | *(inline in VectorManager methods)* | Storage session wrappers that acquire locks via manager and call `Try*` methods | + +Additionally, several existing files are modified (see [Complete File Inventory](#complete-file-inventory)). + +--- + +## Architecture (data flow) + +``` +RESP Client ("RI.SET r1 mykey myval") + │ + ▼ +┌───────────────────────────────────────────────────────────────────────┐ +│ 1. RESP Parser (Resp/Parser/RespCommand.cs) │ +│ Tokenizes "RI.SET" → RespCommand.RISET enum value │ +│ Read/write classification by enum position (reads < APPEND) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 2. Command Dispatch (Resp/RespServerSession.cs) │ +│ Switch on RespCommand → calls NetworkRISET(ref api) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 3. RESP Handler (Resp/RangeIndex/RespServerSessionRangeIndex.cs) │ +│ Parses args from parseState, calls storageApi.RangeIndexSet(...) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 4. IGarnetApi / GarnetApi (API/IGarnetApi.cs, API/GarnetApi.cs) │ +│ Thin delegation: PinnedSpanByte → ReadOnlySpan, forwards to │ +│ storageSession.RangeIndexSet(...) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 5. Storage Session (Storage/Session/MainStore/RangeIndexOps.cs) │ +│ Acquires index lock via rangeIndexManager.ReadOrCreateRangeIndex() │ +│ Calls rangeIndexManager.TryInsert(indexSpan, field, value) │ +│ Replicates on success via rangeIndexManager.Replicate*(...) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 6. RangeIndexManager (Resp/RangeIndex/RangeIndexManager.cs) │ +│ TryInsert: ReadIndex(stub) → extract TreePtr → BfTreeService.Insert│ +├───────────────────────────────────────────────────────────────────────┤ +│ 7. BfTreeService (Resp/RangeIndex/BfTreeService.cs) │ +│ P/Invoke call: bftree_insert(treePtr, key, keyLen, val, valLen) │ +├───────────────────────────────────────────────────────────────────────┤ +│ 8. Bf-Tree Rust library (bftree.dll / libbftree.so) │ +│ BfTree::insert(key, value) → LeafInsertResult::Success │ +└───────────────────────────────────────────────────────────────────────┘ +``` + +**On first write (key doesn't exist yet):** +- Step 5 calls `ReadOrCreateRangeIndex()` which fails to find the key +- It promotes the shared lock to exclusive, creates a new BfTree via + `BfTreeService.CreateIndex()`, and issues an RMW with the new stub +- The RMW hits `InitialUpdater` in `RMWMethods.cs` which writes the stub and + sets the `RecordType` to `RangeIndexManager.RangeIndexRecordType` on the record +- Lock is released, then re-acquired as shared for the actual operation + +--- + +## The Stub (RangeIndexManager.Index.cs) + +> **Reference:** [`VectorManager.Index.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Index.cs) — +> the `Index` struct, `CreateIndex()`, `ReadIndex()`, `RecreateIndex()`. + +A fixed-size struct stored as a raw-byte (non-object) value in the unified store, accessed +via the string context. Since `RecordInfo.ValueIsObject` is `false` for these records, the +string context's `MainSessionFunctions` handles the RMW/Read/Delete callbacks. + +```csharp +[StructLayout(LayoutKind.Explicit, Size = 35)] +private struct RangeIndexStub +{ + [FieldOffset(0)] public nint TreeHandle; // Pointer to live BfTree instance + [FieldOffset(8)] public ulong CacheSize; // BfTree cb_size_byte + [FieldOffset(16)] public uint MinRecordSize; // BfTree cb_min_record_size + [FieldOffset(20)] public uint MaxRecordSize; // BfTree cb_max_record_size + [FieldOffset(24)] public uint MaxKeyLen; // BfTree cb_max_key_len + [FieldOffset(28)] public uint LeafPageSize; // BfTree leaf_page_size + [FieldOffset(32)] public byte StorageBackend; // 0=Disk, 1=Memory + [FieldOffset(33)] public byte Flags; // bit 0: Flushed (needs promote to tail) + [FieldOffset(34)] public byte SerializationPhase; // checkpoint coordination (future) +} +``` + +**Key fields explained:** + +- `TreeHandle` — Native pointer to a live `BfTree` instance. Zeroed by `OnDiskRead` + when the record is loaded from disk (recovery, pending read, etc.). A zero TreeHandle + signals "needs lazy restore" — the first subsequent operation restores the BfTree from + `data.bftree` via `BfTreeService.RecoverFromCprSnapshot` and updates this field via + RIRESTORE RMW. +- `Flags` — Bit 0 (`IsFlushed`): Set by `OnFlush` when the page moves to read-only. + The next RI operation detects this flag, issues RIPROMOTE RMW to copy the stub to the + mutable region (tail), and clears the flag. This ensures the stub will be re-flushed + (with up-to-date BfTree snapshot) on the next checkpoint or ReadOnly transition. + Bit 1 (`IsRecovered`): Set by `OnRecoverySnapshotRead` so the first promote after + recovery routes through `RecreateIndex` rather than the steady-state restore path. + Bit 2 (`IsTransferred`): Set on the source record by `PostCopyToTail` (CTT) and by + RIPROMOTE `PostCopyUpdater` after the CAS succeeds, so a later eviction/dispose of the + source record does not free the tree (live case) or the pending entry (cold case) that + the destination now owns. +- `SerializationPhase` — Reserved for checkpoint coordination (future work). +- Config fields (`CacheSize`, `MinRecordSize`, etc.) — Persisted so recovery can + reconstruct the BfTree with identical configuration. + +--- + +## Step-by-Step Implementation + +### Step 1: Type discrimination for RangeIndex records + +> **Reference:** `libs/storage/Tsavorite/cs/src/core/Allocator/LogRecord.cs` — +> `RecordType` is a `byte` field in the `RecordDataHeader` (offset 2 in the header), +> accessible via `LogRecord.RecordType` and `srcLogRecord.RecordType` in session function +> callbacks. Currently `LogRecord.InitializeRecord()` hardcodes `recordType: 0` (with +> a TODO to pass in the actual type). +> +> See also: `RecordDataHeader.cs` — `RecordTypeOffsetInHeader = 2`, +> `ISourceLogRecord.RecordType` property. + +RangeIndex records need to be distinguishable from regular string records so that: +- **ReadMethods.cs** — RI commands are rejected on non-RI keys, and non-RI commands + are rejected on RI keys (type safety) +- **DeleteMethods.cs** — deletion is blocked while stub is non-zero (BfTree still alive) +- **RMWMethods.cs** — no guard needed; the command itself determines behavior + +**Approach:** Use the `RecordType` byte on each `LogRecord`. Define a constant +`RangeIndexRecordType` (e.g., `2`, reserving `1` for VectorSet). This byte is set when +the stub record is first created and checked in `Reader`/`Deleter` callbacks for type +safety. + +```csharp +// In RangeIndexManager.cs: +internal const byte RangeIndexRecordType = 2; +``` + +**Tsavorite plumbing required:** The `RecordDataHeader.Initialize()` method and +`LogRecord.InitializeRecord()` currently hardcode `recordType: 0`. These need to be +updated to accept and propagate a `recordType` parameter from the session function +callbacks (e.g., `InitialUpdater` sets the record type when creating a new record). +This is a one-time infrastructure change that also unblocks VectorSet. + +Also add a helper in `RespCommand` extensions: + +```csharp +// In RespCommandExtensions.cs: +public static bool IsLegalOnRangeIndex(this RespCommand cmd) + => cmd is RespCommand.RISET or RespCommand.RIDEL or RespCommand.RIGET + or RespCommand.RIMSET or RespCommand.RIMDEL or RespCommand.RIMGET + or RespCommand.RISCAN or RespCommand.RIRANGE + or RespCommand.RICREATE + or RespCommand.RIEXISTS or RespCommand.RICONFIG or RespCommand.RIMETRICS; +``` + +--- + +### Step 2: Add RESP commands to the parser + +> **Reference:** `libs/server/Resp/Parser/RespCommand.cs` +> - Read commands are defined before `APPEND` (the last read command is just before `APPEND`) +> - Write commands are defined starting at `APPEND` +> - Read/write classification uses enum ordering: `cmd <= LastReadCommand` ⟹ read-only +> - Fast parsing: `FastParseArrayCommand()` uses `ulong` pointer comparisons for short +> fixed-length commands +> - Longer/unusual commands fall through to `SlowParseCommand()` + +**Add enum values:** + +```csharp +// --- Read commands (insert before APPEND) --- +RIGET, // RI.GET key field +RIMGET, // RI.MGET key field1 [field2 ...] +RISCAN, // RI.SCAN key start COUNT n [FIELDS KEY|VALUE|BOTH] +RIRANGE, // RI.RANGE key start end [FIELDS KEY|VALUE|BOTH] +RIEXISTS, // RI.EXISTS key +RICONFIG, // RI.CONFIG key +RIMETRICS, // RI.METRICS key +RIKEYS, // RI.KEYS [pattern] + +// --- Write commands (insert after APPEND, before boundary) --- +RISET, // RI.SET key field value +RIDEL, // RI.DEL key field +RIMSET, // RI.MSET key f1 v1 [f2 v2 ...] +RIMDEL, // RI.MDEL key f1 [f2 ...] +RICREATE, // RI.CREATE key [options] +``` + +**Parsing:** RI commands are dot-prefixed (`RI.SET`), so they won't fit the 4-char fast +path. Add a branch in `SlowParseCommand()` or a dedicated `ParseRangeIndexCommand()`: + +```csharp +// In SlowParseCommand or equivalent: +if (length >= 4 && ptr[0] == 'R' && ptr[1] == 'I' && ptr[2] == '.') +{ + return ParseRangeIndexCommand(ptr + 3, length - 3); +} + +private static RespCommand ParseRangeIndexCommand(byte* ptr, int length) +{ + // Match remaining bytes: "SET", "GET", "DEL", "SCAN", "RANGE", etc. + return length switch + { + 3 when *(ushort*)ptr == MemoryMarshal.Read("SE"u8) + && ptr[2] == (byte)'T' => RespCommand.RISET, + 3 when *(ushort*)ptr == MemoryMarshal.Read("GE"u8) + && ptr[2] == (byte)'T' => RespCommand.RIGET, + 3 when *(ushort*)ptr == MemoryMarshal.Read("DE"u8) + && ptr[2] == (byte)'L' => RespCommand.RIDEL, + 4 when *(uint*)ptr == MemoryMarshal.Read("SCAN"u8) => RespCommand.RISCAN, + 4 when *(uint*)ptr == MemoryMarshal.Read("MSET"u8) => RespCommand.RIMSET, + 4 when *(uint*)ptr == MemoryMarshal.Read("MDEL"u8) => RespCommand.RIMDEL, + 4 when *(uint*)ptr == MemoryMarshal.Read("MGET"u8) => RespCommand.RIMGET, + 4 when *(uint*)ptr == MemoryMarshal.Read("KEYS"u8) => RespCommand.RIKEYS, + 5 when *(uint*)ptr == MemoryMarshal.Read("RANG"u8) + && ptr[4] == (byte)'E' => RespCommand.RIRANGE, + 6 when *(uint*)ptr == MemoryMarshal.Read("CREA"u8) => RespCommand.RICREATE, + 6 when *(uint*)ptr == MemoryMarshal.Read("CONF"u8) => RespCommand.RICONFIG, + 6 when *(uint*)ptr == MemoryMarshal.Read("EXIS"u8) => RespCommand.RIEXISTS, + 7 when *(uint*)ptr == MemoryMarshal.Read("METR"u8) => RespCommand.RIMETRICS, + _ => RespCommand.NONE, + }; +} +``` + +--- + +### Step 3: Add command dispatch in `RespServerSession` + +> **Reference:** `libs/server/Resp/RespServerSession.cs` +> Commands are dispatched via switch expressions in `ProcessBasicCommands` and +> `ProcessArrayCommands`. Each maps a `RespCommand` enum to a handler method. +> Example: `RespCommand.GET => NetworkGET(ref storageApi),` + +Add RangeIndex dispatch entries: + +```csharp +// In the command dispatch switch: +RespCommand.RISET => NetworkRISET(ref storageApi), +RespCommand.RIDEL => NetworkRIDEL(ref storageApi), +RespCommand.RIGET => NetworkRIGET(ref storageApi), +RespCommand.RISCAN => NetworkRISCAN(ref storageApi), +RespCommand.RIRANGE => NetworkRIRANGE(ref storageApi), +RespCommand.RIMSET => NetworkRIMSET(ref storageApi), +RespCommand.RIMDEL => NetworkRIMDEL(ref storageApi), +RespCommand.RIMGET => NetworkRIMGET(ref storageApi), +RespCommand.RICREATE => NetworkRICREATE(ref storageApi), +RespCommand.RIEXISTS => NetworkRIEXISTS(ref storageApi), +RespCommand.RICONFIG => NetworkRICONFIG(ref storageApi), +RespCommand.RIMETRICS => NetworkRIMETRICS(ref storageApi), +RespCommand.RIKEYS => NetworkRIKEYS(ref storageApi), +``` + +--- + +### Step 4: Implement RESP command handlers + +> **Reference:** RESP handler pattern used throughout `libs/server/Resp/` (e.g., +> `BasicCommands.cs`, `Objects/HashCommands.cs`). +> **Reference:** [`RespServerSessionVectors.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/RespServerSessionVectors.cs) — +> vector RESP handlers (`NetworkVADD`, `NetworkVGET`, etc.) follow the same pattern. +> Pattern: `private bool NetworkXXX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi` +> - Parse arguments from `parseState.GetArgSliceByRef(idx)` +> - Call `storageApi.RangeIndex*(...)` with parsed args +> - Write RESP response via `RespWriteUtils.TryWrite*(..., ref dcurr, dend)` +> - Handle `GarnetStatus.OK`, `NOTFOUND`, `WRONGTYPE` + +**File:** `libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs` (new) + +
    +RESP command handler implementations (click to expand) + +```csharp +internal sealed unsafe partial class RespServerSession +{ + /// RI.SET key field value + /// Inserts or updates a field in the RangeIndex. Auto-creates the index if needed. + private bool NetworkRISET(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + // Expect 3 args: key, field, value + if (parseState.Count != 3) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RISET)); + } + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + var value = parseState.GetArgSliceByRef(2); + + var res = storageApi.RangeIndexSet(key, field, value, + out var result, out var errorMsg); + + if (res == GarnetStatus.WRONGTYPE) + // Key exists but is not a RangeIndex + return AbortRangeIndexWrongType(); + + if (result == RangeIndexResult.OK) + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_OK, ref dcurr, dend)) + SendAndReset(); + } + else + { + while (!RespWriteUtils.WriteError(errorMsg, ref dcurr, dend)) + SendAndReset(); + } + return true; + } + + /// RI.GET key field + /// Returns the value for a field, or nil if not found. + private bool NetworkRIGET(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 2) + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RIGET)); + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + var output = new SpanByteAndMemory(); + + var res = storageApi.RangeIndexGet(key, field, ref output, out var result); + + if (res == GarnetStatus.WRONGTYPE) + return AbortRangeIndexWrongType(); + if (res == GarnetStatus.NOTFOUND || result == RangeIndexResult.NotFound) + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend)) + SendAndReset(); + } + else + { + // Write bulk string from output + // ... (use output.SpanByte or output.Memory) + } + return true; + } + + /// RI.DEL key field + private bool NetworkRIDEL(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 2) + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RIDEL)); + + var key = parseState.GetArgSliceByRef(0); + var field = parseState.GetArgSliceByRef(1); + + var res = storageApi.RangeIndexDel(key, field); + + if (res == GarnetStatus.WRONGTYPE) + return AbortRangeIndexWrongType(); + + while (!RespWriteUtils.TryWriteInt32(res == GarnetStatus.OK ? 1 : 0, + ref dcurr, dend)) + SendAndReset(); + return true; + } + + /// RI.SCAN key start COUNT n [FIELDS KEY|VALUE|BOTH] + private bool NetworkRISCAN(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + // Minimum 4 args: key, start, "COUNT", n + // Optional: "FIELDS", KEY|VALUE|BOTH + if (parseState.Count < 4) + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RISCAN)); + + var key = parseState.GetArgSliceByRef(0); + var startKey = parseState.GetArgSliceByRef(1); + + // Parse COUNT + // parseState[2] must be "COUNT" + var countArg = parseState.GetArgSliceByRef(3); + if (!NumUtils.TryParse(countArg.ReadOnlySpan, out int count) || count <= 0) + { + while (!RespWriteUtils.WriteError("ERR invalid count"u8, ref dcurr, dend)) + SendAndReset(); + return true; + } + + // Parse optional FIELDS + byte returnField = 2; // 0=Key, 1=Value, 2=KeyAndValue (default BOTH) + if (parseState.Count >= 6) + { + // parseState[4] = "FIELDS", parseState[5] = KEY|VALUE|BOTH + var fieldsVal = parseState.GetArgSliceByRef(5).ReadOnlySpan; + if (fieldsVal.SequenceEqual("KEY"u8)) returnField = 0; + else if (fieldsVal.SequenceEqual("VALUE"u8)) returnField = 1; + else returnField = 2; + } + + var output = new SpanByteAndMemory(); + var res = storageApi.RangeIndexScan(key, startKey, count, returnField, + ref output, out var resultCount); + + // Write RESP array from output + // ... + return true; + } + + /// RI.RANGE key start end [FIELDS KEY|VALUE|BOTH] + private bool NetworkRIRANGE(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 3) + return AbortWithWrongNumberOfArguments(nameof(RespCommand.RIRANGE)); + + var key = parseState.GetArgSliceByRef(0); + var startKey = parseState.GetArgSliceByRef(1); + var endKey = parseState.GetArgSliceByRef(2); + + byte returnField = 2; // default BOTH + if (parseState.Count >= 5) + { + var fieldsVal = parseState.GetArgSliceByRef(4).ReadOnlySpan; + if (fieldsVal.SequenceEqual("KEY"u8)) returnField = 0; + else if (fieldsVal.SequenceEqual("VALUE"u8)) returnField = 1; + else returnField = 2; + } + + var output = new SpanByteAndMemory(); + var res = storageApi.RangeIndexRange(key, startKey, endKey, returnField, + ref output, out var resultCount); + + // Write RESP array from output + // ... + return true; + } + + // NetworkRICREATE, NetworkRIMSET, NetworkRIMDEL, + // NetworkRIMGET, NetworkRIEXISTS, NetworkRICONFIG, NetworkRIMETRICS, + // NetworkRIKEYS follow the same pattern. +} +``` + +
    + +--- + +### Step 5: Add `IGarnetApi` and `GarnetApi` interface methods + +> **Reference:** `libs/server/API/IGarnetApi.cs` — declares storage API methods. +> `libs/server/API/GarnetApi.cs` — delegation: `PinnedSpanByte` → `.ReadOnlySpan` +> then forward to `storageSession.RangeIndex*()`. + +**IGarnetApi.cs — add:** + +
    +IGarnetApi method declarations (click to expand) + +```csharp +// --- RangeIndex operations --- + +GarnetStatus RangeIndexSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg); + +GarnetStatus RangeIndexDel(PinnedSpanByte key, PinnedSpanByte field); + +GarnetStatus RangeIndexGet(PinnedSpanByte key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result); + +GarnetStatus RangeIndexScan(PinnedSpanByte key, PinnedSpanByte startKey, + int count, byte returnField, + ref StringOutput output, out int resultCount); + +GarnetStatus RangeIndexRange(PinnedSpanByte key, PinnedSpanByte startKey, PinnedSpanByte endKey, + byte returnField, + ref StringOutput output, out int resultCount); + +GarnetStatus RangeIndexCreate(PinnedSpanByte key, + ulong cacheSize, uint minRecord, uint maxRecord, + uint maxKeyLen, uint leafPageSize, byte storageBackend, + out RangeIndexResult result, out ReadOnlySpan errorMsg); + +GarnetStatus RangeIndexExists(PinnedSpanByte key, out bool exists); + +GarnetStatus RangeIndexConfig(PinnedSpanByte key, + ref StringOutput output); + +GarnetStatus RangeIndexMetrics(PinnedSpanByte key, + ref StringOutput output); +``` + +
    + +**GarnetApi.cs — add delegation** (expression-bodied): + +
    +GarnetApi delegation methods (click to expand) + +```csharp +public GarnetStatus RangeIndexSet(PinnedSpanByte key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg) + => storageSession.RangeIndexSet( + key.ReadOnlySpan, + field, value, out result, out errorMsg); + +public GarnetStatus RangeIndexGet(PinnedSpanByte key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result) + => storageSession.RangeIndexGet( + key.ReadOnlySpan, + field, ref output, out result); + +// ... same pattern for all methods +``` + +
    + +--- + +### Step 6: Implement Storage Session layer (`RangeIndexOps.cs`) + +> **Reference:** `libs/server/Storage/Session/MainStore/` — contains string-context +> storage operations (e.g., `MainStoreOps.cs`). RangeIndex follows the same pattern +> with a new file `RangeIndexOps.cs`. +> - Write ops: marshal args → `ReadOrCreateRangeIndex()` → `TryInsert()` → replicate +> - Read ops: `ReadRangeIndex()` → `TryRead()` / `TryScan()` / `TryRange()` +> - Lock pattern: `using (rangeIndexManager.ReadOrCreateRangeIndex(this, ...))` — the +> `using` block holds a shared lock via `ReadRangeIndexLock` ref struct +> - On first access, `ReadOrCreateRangeIndex` promotes to exclusive lock, creates BfTree +> via `BfTreeService.CreateIndex()`, issues RMW to persist stub, releases exclusive, +> re-acquires shared lock, then returns + +**File:** `libs/server/Storage/Session/MainStore/RangeIndexOps.cs` (new) + +
    +StorageSession RangeIndexOps implementation (click to expand) + +```csharp +internal sealed unsafe partial class StorageSession +{ + /// RI.SET — insert/update a field in the range index + public GarnetStatus RangeIndexSet(ReadOnlySpan key, PinnedSpanByte field, PinnedSpanByte value, + out RangeIndexResult result, out ReadOnlySpan errorMsg) + { + errorMsg = default; + + // Marshal field + value into parseState for replication log + parseState.InitializeWithArguments(field, value); + var input = new StringInput(RespCommand.RISET, ref parseState); + + // Acquire lock + create-if-needed + Span indexSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + using (rangeIndexManager.ReadOrCreateRangeIndex( + this, key, ref input, indexSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.Error; + return status; + } + + // Dispatch to BfTree while holding shared lock + result = rangeIndexManager.TryInsert( + indexSpan, field.ReadOnlySpan, value.ReadOnlySpan, + out errorMsg); + + if (result == RangeIndexResult.OK) + rangeIndexManager.ReplicateRangeIndexSet( + key, ref input, ref stringBasicContext); + + return GarnetStatus.OK; + } + } + + /// RI.GET — point read + public GarnetStatus RangeIndexGet(ReadOnlySpan key, PinnedSpanByte field, + ref StringOutput output, out RangeIndexResult result) + { + parseState.InitializeWithArgument(field); + var input = new StringInput(RespCommand.RIGET, ref parseState); + + Span indexSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + using (rangeIndexManager.ReadRangeIndex( + this, key, ref input, indexSpan, out var status)) + { + if (status != GarnetStatus.OK) + { + result = RangeIndexResult.NotFound; + return status; + } + + result = rangeIndexManager.TryRead( + indexSpan, field.ReadOnlySpan, ref output); + return GarnetStatus.OK; + } + } + + /// RI.DEL — delete a field + public GarnetStatus RangeIndexDel(ReadOnlySpan key, PinnedSpanByte field) + { + parseState.InitializeWithArgument(field); + var input = new StringInput(RespCommand.RIDEL, ref parseState); + + Span indexSpan = stackalloc byte[RangeIndexManager.IndexSizeBytes]; + using (rangeIndexManager.ReadRangeIndex( + this, key, ref input, indexSpan, out var status)) + { + if (status != GarnetStatus.OK) return status; + + rangeIndexManager.TryDelete(indexSpan, field.ReadOnlySpan); + rangeIndexManager.ReplicateRangeIndexDel( + key, ref input, ref stringBasicContext); + return GarnetStatus.OK; + } + } + + /// RI.SCAN — range scan with count + public GarnetStatus RangeIndexScan(ReadOnlySpan key, PinnedSpanByte startKey, + int count, byte returnField, + ref StringOutput output, out int resultCount) + { + // ... same lock pattern, then: + // rangeIndexManager.TryScanWithCount(indexSpan, startKey, count, returnField, ref output, out resultCount) + } + + /// RI.RANGE — range scan with end key + public GarnetStatus RangeIndexRange(ReadOnlySpan key, PinnedSpanByte startKey, + PinnedSpanByte endKey, byte returnField, + ref StringOutput output, out int resultCount) + { + // ... same lock pattern, then: + // rangeIndexManager.TryScanWithEndKey(indexSpan, startKey, endKey, returnField, ref output, out resultCount) + } +} +``` + +
    + +--- + +### Step 7: Implement `RangeIndexManager` (partial class) + +> `RangeIndexManager` is a new partial class split across multiple files. +> We start with 4 core files; migration and replication are deferred. + +#### 7a. `RangeIndexManager.cs` — Main class + +> **Reference:** [`VectorManager.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.cs) — +> constants, `processInstanceId`, `TryAdd`/`TryRemove`, `Initialize()`, `ResumePostRecovery()`, `Dispose()`. + +
    +RangeIndexManager main class (click to expand) + +```csharp +public sealed partial class RangeIndexManager : IDisposable +{ + // --- Constants --- + internal const int IndexSizeBytes = 51; // sizeof(RangeIndexStub) + internal const long RISetAppendLogArg = long.MinValue; + internal const long RecreateIndexArg = RISetAppendLogArg + 1; + internal const long RIDelAppendLogArg = RecreateIndexArg + 1; + + // --- Fields --- + private readonly Guid processInstanceId = Guid.NewGuid(); + private readonly BfTreeService service = new(); + public bool IsEnabled { get; } + private readonly ILogger logger; + + // --- Core operation methods --- + + /// Insert a field-value pair into the BfTree identified by the stub + internal RangeIndexResult TryInsert(ReadOnlySpan indexValue, + ReadOnlySpan field, ReadOnlySpan value, + out ReadOnlySpan errorMsg) + { + errorMsg = default; + ReadIndex(indexValue, out var treePtr, out _, out _, out _, + out _, out _, out _, out _, out _); + var insertResult = service.Insert(treePtr, field, value); + if (insertResult == BfTreeInsertResult.InvalidKV) + { + errorMsg = "ERR invalid key or value size"u8; + return RangeIndexResult.Error; + } + return RangeIndexResult.OK; + } + + /// Read a single field from the BfTree + internal RangeIndexResult TryRead(ReadOnlySpan indexValue, + ReadOnlySpan field, ref SpanByteAndMemory output) + { + ReadIndex(indexValue, out var treePtr, out _, out var maxRecordSize, + out _, out _, out _, out _, out _, out _); + var result = service.Read(treePtr, field, maxRecordSize, ref output); + return result; + } + + /// Delete a field from the BfTree + internal void TryDelete(ReadOnlySpan indexValue, + ReadOnlySpan field) + { + ReadIndex(indexValue, out var treePtr, out _, out _, out _, + out _, out _, out _, out _, out _); + service.Delete(treePtr, field); + } + + /// Scan with count + internal RangeIndexResult TryScanWithCount(ReadOnlySpan indexValue, + ReadOnlySpan startKey, int count, byte returnField, + ref SpanByteAndMemory output, out int resultCount) + { + ReadIndex(indexValue, out var treePtr, out _, out _, out _, + out _, out _, out _, out _, out _); + return service.ScanWithCount(treePtr, startKey, count, returnField, + ref output, out resultCount); + } + + /// Scan with end key + internal RangeIndexResult TryScanWithEndKey(ReadOnlySpan indexValue, + ReadOnlySpan startKey, ReadOnlySpan endKey, + byte returnField, + ref SpanByteAndMemory output, out int resultCount) + { + ReadIndex(indexValue, out var treePtr, out _, out _, out _, + out _, out _, out _, out _, out _); + return service.ScanWithEndKey(treePtr, startKey, endKey, returnField, + ref output, out resultCount); + } + + /// Recovery is lazy — no proactive scan needed. ReadRangeIndex detects stale stubs. + internal void ResumePostRecovery() { /* no-op; lazy recovery via ReadRangeIndex */ } + + public void Dispose() { service.Dispose(); } +} +``` + +
    + +#### 7b. `RangeIndexManager.Index.cs` — Stub struct + serialization + +> **Reference:** [`VectorManager.Index.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/VectorManager.Index.cs) — +> 51-byte `Index` struct with `CreateIndex()`, `ReadIndex()`, `RecreateIndex()`, `SetContextForMigration()`. + +
    +RangeIndexStub struct and serialization methods (click to expand) + +```csharp +public sealed partial class RangeIndexManager +{ + [StructLayout(LayoutKind.Explicit, Size = Size)] + private struct RangeIndexStub + { + internal const int Size = 51; + + [FieldOffset(0)] public nint TreePtr; + [FieldOffset(8)] public ulong CacheSize; + [FieldOffset(16)] public uint MinRecordSize; + [FieldOffset(20)] public uint MaxRecordSize; + [FieldOffset(24)] public uint MaxKeyLen; + [FieldOffset(28)] public uint LeafPageSize; + [FieldOffset(32)] public byte StorageBackend; + [FieldOffset(33)] public byte Flags; + [FieldOffset(34)] public byte SerializationPhase; + [FieldOffset(35)] public Guid ProcessInstanceId; + } + + /// Write a new stub into the value span of a LogRecord. + /// Called from InitialUpdater via RMWMethods.cs. + internal void CreateIndex(uint cacheSize, uint minRecordSize, + uint maxRecordSize, uint maxKeyLen, uint leafPageSize, + byte storageBackend, nint treePtr, Span valueSpan) + { + Debug.Assert(valueSpan.Length >= RangeIndexStub.Size); + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.TreePtr = treePtr; + stub.CacheSize = cacheSize; + stub.MinRecordSize = minRecordSize; + stub.MaxRecordSize = maxRecordSize; + stub.MaxKeyLen = maxKeyLen; + stub.LeafPageSize = leafPageSize; + stub.StorageBackend = storageBackend; + stub.ProcessInstanceId = processInstanceId; + } + + /// Deserialize stub from main store value. + internal static void ReadIndex(ReadOnlySpan value, + out nint treePtr, out ulong cacheSize, + out uint minRecordSize, out uint maxRecordSize, + out uint maxKeyLen, out uint leafPageSize, + out byte storageBackend, out byte flags, out Guid pid) + { + ref readonly var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(value)); + treePtr = stub.TreePtr; + cacheSize = stub.CacheSize; + minRecordSize = stub.MinRecordSize; + maxRecordSize = stub.MaxRecordSize; + maxKeyLen = stub.MaxKeyLen; + leafPageSize = stub.LeafPageSize; + storageBackend = stub.StorageBackend; + flags = stub.Flags; + pid = stub.ProcessInstanceId; + } + + /// Update TreePtr after recovery (old pointer is stale). + internal void RecreateIndex(nint newTreePtr, Span valueSpan) + { + ReadIndex(valueSpan, out _, out _, out _, out _, + out _, out _, out _, out _, out var indexPid); + Debug.Assert(processInstanceId != indexPid, + "Shouldn't recreate an index from the same process instance"); + ref var stub = ref Unsafe.As( + ref MemoryMarshal.GetReference(valueSpan)); + stub.TreePtr = newTreePtr; + stub.ProcessInstanceId = processInstanceId; + } + +} +``` + +
    + +#### 7c. `RangeIndexManager.Locking.cs` — Lock management + +> The locking pattern uses a striped `ReadOptimizedLock` (from `Garnet.common`) for +> concurrent access, with stripe count based on `Environment.ProcessorCount`. Key hash +> (via `unifiedBasicContext.GetKeyHash(key)`) selects the stripe. +> `ReadRangeIndex()` acquires a shared lock; `ReadOrCreateRangeIndex()` promotes to +> exclusive if the key doesn't exist, creates the BfTree, then downgrades to shared. +> +> `ReadOptimizedLock` is defined in `libs/common/Synchronization/ReadOptimizedLock.cs` +> (shared with VectorSet). It provides shared/exclusive locking with +> `AcquireSharedLock(keyHash, out token)`, +> `TryPromoteSharedLock(keyHash, sharedToken, out exclusiveToken)`, +> `ReleaseSharedLock(token)`, and `ReleaseExclusiveLock(token)` methods. +> Tests: `test/Garnet.test/ReadOptimizedLockTests.cs`. + +
    +ReadRangeIndexLock and locking methods (click to expand) + +```csharp +public sealed partial class RangeIndexManager +{ + private readonly ReadOptimizedLock rangeIndexLocks; // Striped lock (ProcessorCount stripes) + + /// RAII lock holder — disposed at end of `using` block + internal readonly ref struct ReadRangeIndexLock : IDisposable + { + private readonly ref readonly ReadOptimizedLock lockRef; + private readonly int lockToken; + + internal ReadRangeIndexLock(in ReadOptimizedLock lockRef, int token) + { + this.lockRef = ref lockRef; + this.lockToken = token; + } + public void Dispose() => lockRef.ReleaseSharedLock(lockToken); + } + + /// Acquire shared lock on an EXISTING range index. + /// Returns NOTFOUND if key doesn't exist. + /// If ProcessInstanceId mismatches (evicted index whose stub was cleared by the + /// deserialization observer, or stale pointer after process restart), promotes to + /// exclusive, restores BfTree from snapshot, releases exclusive, re-acquires shared. + internal ReadRangeIndexLock ReadRangeIndex( + StorageSession session, PinnedSpanByte key, ref StringInput input, + Span indexSpan, out GarnetStatus status) + { + // 1. var keyHash = session.unifiedBasicContext.GetKeyHash(key); + // 2. rangeIndexLocks.AcquireSharedLock(keyHash, out var sharedLockToken); + // 3. var indexOutput = StringOutput.FromPinnedSpan(indexSpan); + // session.Read_MainStore(key.ReadOnlySpan, ref input, ref indexOutput, ref session.stringBasicContext); + // 4. If not found: status = NOTFOUND, return default lock + // 5. ReadIndex → extract TreePtr, ProcessInstanceId + // 6. If ProcessInstanceId != this.processInstanceId: + // TryPromoteSharedLock → restore from snapshot → RMW update → release exclusive, retry + // 7. Return ReadRangeIndexLock holding shared lock + } + + /// Acquire shared lock, CREATE the range index if key doesn't exist. + /// Used by RI.SET and RI.CREATE. + internal ReadRangeIndexLock ReadOrCreateRangeIndex( + StorageSession session, PinnedSpanByte key, ref StringInput input, + Span indexSpan, out GarnetStatus status) + { + // 1. Same as ReadRangeIndex + // 2. If not found: TryPromoteSharedLock to exclusive + // 3. Create BfTree: var treePtr = service.CreateIndex(config...) + // 4. Inject treePtr into input.parseState (arg slot for InitialUpdater) + // 5. Issue RMW_MainStore → hits InitialUpdater which writes stub to logRecord + // 6. Release exclusive lock, re-acquire shared + // 7. Read the stub into indexSpan + // 8. Return ReadRangeIndexLock holding shared lock + } +} +``` + +
    + +--- + +### Step 8: Wire into Main Store RMW callbacks + +> **Reference:** `libs/server/Storage/Functions/MainStore/RMWMethods.cs` +> - `InitialUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo)` +> - `InPlaceUpdater(ref LogRecord logRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo)` +> - `CopyUpdater(in TSourceLogRecord srcLogRecord, ref LogRecord dstLogRecord, in RecordSizeInfo sizeInfo, ref StringInput input, ref StringOutput output, ref RMWInfo rmwInfo)` +> Add new cases for `RespCommand.RICREATE`, `RISET`, and `RIDEL` in each method. +> Access command via `input.header.cmd`. Access/modify value via `logRecord.ValueSpan` / `logRecord.TrySetValueSpan(...)`. + +#### InitialUpdater + +
    +InitialUpdater case (click to expand) + +```csharp +case RespCommand.RICREATE: +case RespCommand.RISET: +{ + if (input.arg1 is RangeIndexManager.RISetAppendLogArg) + { + break; // Synthetic replication op, do nothing + } + + // Extract config + treePtr from parseState + // (injected by ReadOrCreateRangeIndex before issuing RMW) + var cacheSize = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(0).ReadOnlySpan); + var minRecordSize = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(1).ReadOnlySpan); + var maxRecordSize = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(2).ReadOnlySpan); + var maxKeyLen = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(3).ReadOnlySpan); + var leafPageSize = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(4).ReadOnlySpan); + var storageBackend = input.parseState.GetArgSliceByRef(5).ReadOnlySpan[0]; + var treePtr = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(6).ReadOnlySpan); + + // Set RecordType to identify this as a RangeIndex stub + // logRecord.RecordType = RangeIndexManager.RangeIndexRecordType; + functionsState.rangeIndexManager.CreateIndex( + cacheSize, minRecordSize, maxRecordSize, + maxKeyLen, leafPageSize, storageBackend, + treePtr, logRecord.ValueSpan); +} +break; +``` + +
    + +#### InPlaceUpdater + +
    +InPlaceUpdater case (click to expand) + +```csharp +case RespCommand.RISET: +case RespCommand.RIDEL: +case RespCommand.RICREATE: + if (input.arg1 == RangeIndexManager.RecreateIndexArg) + { + var newTreePtr = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(6).ReadOnlySpan); + functionsState.rangeIndexManager.RecreateIndex( + newTreePtr, logRecord.ValueSpan); + } + // All other operations (insert/delete/scan) are handled OUTSIDE + // of Tsavorite's RMW — they happen in the StorageSession layer + // while holding the shared lock. The RMW path here is only for + // stub lifecycle (create/recreate). + return true; +``` + +
    + +#### CopyUpdater + +After copying the stub to the new record, we must handle the old record's BfTree +carefully. This is the inline-bytes analogue of `CacheSerializedObjectData` in +`HeapObjectBase.cs`, which uses a `SerializationPhase` state machine (`REST` → +`SERIALIZING` → `SERIALIZED`) to coordinate between CopyUpdater and concurrent +snapshot flush. + +**The problem:** During a checkpoint, the snapshot flush callback needs a consistent +BfTree snapshot. But once CopyUpdater completes and the new record becomes visible, +concurrent operations will modify the BfTree through the new record — racing with the +snapshot flush trying to read the same BfTree for the old (checkpoint-version) record. + +**The solution:** The CopyUpdater itself snapshots the BfTree (under the exclusive RMW +lock, before the new record is visible), producing a stable file. The snapshot flush +callback then uses the already-written file instead of the live BfTree. + +A `SerializationPhase` state machine on the RangeIndexManager (per-index, keyed by +`TreePtr`) coordinates this: +- **`REST`** — no snapshot in progress +- **`SERIALIZING`** — CopyUpdater or snapshot flush is writing the BfTree to disk +- **`SERIALIZED`** — a stable snapshot file exists for this checkpoint version + +**Two cases in CopyUpdater:** + +1. **No checkpoint in progress** (`!srcLogRecord.Info.IsInNewVersion`): + Zero `TreePtr` in the old stub. Eviction sees `TreePtr == 0` and skips. + +2. **Checkpoint in progress** (`srcLogRecord.Info.IsInNewVersion`): + The old record is part of the checkpoint. Snapshot the BfTree now (transition + `REST` → `SERIALIZING` → `SERIALIZED`), then zero `TreePtr` in the old stub. + The snapshot flush callback sees `SERIALIZED` and uses the file, or sees + `SERIALIZING` and waits (spin-yield), matching the `CacheSerializedObjectData` + pattern. + +
    +CopyUpdater case (click to expand) + +```csharp +case RespCommand.RISET: +case RespCommand.RIDEL: +case RespCommand.RICREATE: + if (input.arg1 == RangeIndexManager.RecreateIndexArg) + { + var newTreePtr = MemoryMarshal.Read( + input.parseState.GetArgSliceByRef(6).ReadOnlySpan); + srcLogRecord.ValueSpan.CopyTo(dstLogRecord.ValueSpan); + functionsState.rangeIndexManager.RecreateIndex( + newTreePtr, dstLogRecord.ValueSpan); + } + else + { + srcLogRecord.ValueSpan.CopyTo(dstLogRecord.ValueSpan); + } + + // Handle old record's BfTree — analogous to CacheSerializedObjectData. + ReadIndex(srcLogRecord.ValueSpan, out var treePtr, ...); + if (treePtr != nint.Zero) + { + if (srcLogRecord.Info.IsInNewVersion) + { + // Checkpoint in progress: snapshot the BfTree NOW, before the new + // record becomes visible and concurrent ops modify the tree. + // Uses SerializationPhase state machine to coordinate with flush callback. + functionsState.rangeIndexManager.SnapshotForCheckpoint( + treePtr, srcLogRecord.Key); + } + + // Zero TreePtr in old record — safe now because either: + // (a) no checkpoint → eviction will skip, or + // (b) checkpoint → snapshot file already written above + ref var oldStub = ref Unsafe.As( + ref MemoryMarshal.GetReference(srcLogRecord.ValueSpan)); + oldStub.TreePtr = nint.Zero; + } + break; +``` + +
    + +--- + +### Step 9: Wire into Main Store Read Methods + +> **Reference:** `libs/server/Storage/Functions/MainStore/ReadMethods.cs` +> - `SingleReader()` — validates record type before allowing reads +> - `ConcurrentReader()` — same pattern for concurrent access +> Currently, `ReadMethods.cs` checks `ValueIsObject` to reject string commands on +> object records. Add analogous guards using the `RecordType` byte. + +Add type-safety guards in both `SingleReader` and `ConcurrentReader`: + +```csharp +// Add RangeIndex type-safety checks: +if (srcLogRecord.RecordType == RangeIndexManager.RangeIndexRecordType && !cmd.IsLegalOnRangeIndex()) +{ + readInfo.Action = ReadAction.CancelOperation; + return false; +} +else if (srcLogRecord.RecordType != RangeIndexManager.RangeIndexRecordType && cmd.IsLegalOnRangeIndex()) +{ + readInfo.Action = ReadAction.CancelOperation; + return false; +} +``` + +**Note:** Actual read operations (RI.GET, RI.SCAN, RI.RANGE) do NOT go through +Tsavorite's `Read()` path for data access. The storage session reads the stub via +`Read_MainStore()` only to get the `TreePtr`, then calls BfTree directly via +`RangeIndexManager.TryRead()` etc. The type-safety guards above prevent misuse +(e.g., `GET` on a RangeIndex key). + +--- + +### Step 10: Delete handling via `DisposeRecord` + +> **Reference:** `libs/server/Storage/Functions/MainStore/DisposeMethods.cs` +> The `DisposeRecord` callback is invoked by Tsavorite when a record is deleted (`DEL` / `UNLINK`) +> or evicted from the log. It handles freeing the BfTree for RangeIndex keys. + +No special `RI.DROP` command is needed. Standard `DEL` / `UNLINK` commands delete +RangeIndex keys. The store's `DisposeRecord(DisposeReason.PageEviction)` and delete +path handles BfTree cleanup: + +```csharp +// In DisposeRecord: +if (logRecord.RecordType == RangeIndexManager.RangeIndexRecordType) +{ + var indexSpan = logRecord.ValueSpan; + functionsState.rangeIndexManager.ReadIndex(indexSpan, + out var treePtr, out _, out _, out _, + out _, out _, out _, out _, out var pid); + if (pid == functionsState.rangeIndexManager.ProcessInstanceId && treePtr != 0) + { + // Snapshot the BfTree pointer and free it + functionsState.rangeIndexManager.Service.Drop(treePtr); + } +} +``` + +This approach is simpler and safer than a dedicated drop command: +- No two-phase zeroing+delete dance is needed +- `DisposeRecord` is guaranteed to be called for every deleted or evicted record +- Works for both explicit `DEL`/`UNLINK` and page eviction scenarios + +--- + +### Step 11: Build and integrate the native Bf-Tree library + +The Bf-Tree is a Rust library published on crates.io as +[`bf-tree`](https://crates.io/crates/bf-tree) (source: +[`microsoft/bf-tree`](https://github.com/microsoft/bf-tree)). It has no C FFI layer — +that is provided by a thin **wrapper crate** in the Garnet repo. Unlike the +[`diskann-garnet`](https://github.com/microsoft/DiskANN/tree/main/diskann-garnet) +approach (which publishes a separate NuGet from the DiskANN repo), the `bftree-garnet` +crate and its C# interop wrapper live **inside the Garnet repo** and the native binaries +ship inside the existing `Microsoft.Garnet` NuGet package. This avoids the need for a +separate signing pipeline in the `bf-tree` repo, keeps versioning unified with Garnet, +and follows the same pattern used by `native_device` (Tsavorite's native storage driver). + +#### 11a. Project structure + +**Location:** `libs/native/bftree-garnet/` in the Garnet repo. + +``` +libs/native/bftree-garnet/ +├── Cargo.toml # Rust cdylib crate, depends on bf-tree from crates.io +├── src/ +│ └── lib.rs # #[no_mangle] extern "C" fn FFI exports +└── BfTreeInterop.csproj # C# project with MSBuild cargo target + interop code + ├── NativeBfTreeMethods.cs # [LibraryImport] P/Invoke declarations + └── BfTreeService.cs # High-level managed wrapper +``` + +**`Cargo.toml`:** +```toml +[package] +name = "bftree-garnet" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +bf-tree = "0.4" +``` + +**`src/lib.rs`** — contains all `#[no_mangle] pub extern "C" fn` exports that wrap +bf-tree's Rust API for C/P/Invoke consumption. See the Rust FFI code below. + +**`BfTreeInterop.csproj`** — a C# class library that: +1. Contains the managed interop code (`NativeBfTreeMethods.cs`, `BfTreeService.cs`) +2. Has an MSBuild `` target that runs `cargo build --release` for local development +3. Copies the native library to `$(OutDir)` for the current platform +4. Declares `ContentWithTargetPath` items to include the native library under + `runtimes/{rid}/native/` for NuGet packaging + +```xml + + + true + Garnet.server.BfTreeInterop + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +**`Garnet.server.csproj`** references this project: +```xml + +``` + +#### 11b. Pipeline build and signing plan + +The native library is built as part of Garnet's existing release pipeline +(`azure-pipelines-external-release.yml`). A new **Stage 1** builds the Rust crate on +each target platform, then the existing .NET build/sign/pack stages consume the outputs. + +**Stage 1: Build Native (new, matrix job)** + +Two parallel agents build the Rust crate for their respective platforms: + +| Agent | Rust target | Output | +|---|---|---| +| Linux (ubuntu) | `x86_64-unknown-linux-gnu` | `libbftree_garnet.so` | +| Windows | `x86_64-pc-windows-msvc` | `bftree_garnet.dll` | + +Each agent: +1. Installs the Rust toolchain (e.g., via `rustup`) +2. Runs `cargo build --release --manifest-path libs/native/bftree-garnet/Cargo.toml` +3. Uploads the native library as a pipeline artifact + +```yaml +# Pseudocode for azure-pipelines-external-release.yml additions: +- stage: BuildNative + jobs: + - job: BuildNativeLinux + pool: + vmImage: 'ubuntu-latest' + steps: + - script: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + cargo build --release --manifest-path libs/native/bftree-garnet/Cargo.toml + - publish: libs/native/bftree-garnet/target/release/libbftree_garnet.so + artifact: native-linux-x64 + + - job: BuildNativeWindows + pool: + vmImage: 'windows-latest' + steps: + - script: | + rustup default stable + cargo build --release --manifest-path libs/native/bftree-garnet/Cargo.toml + - publish: libs/native/bftree-garnet/target/release/bftree_garnet.dll + artifact: native-win-x64 +``` + +**Stage 2: Build .NET + Sign + Pack (existing stage, modified)** + +Before the existing `dotnet build` step, download the native artifacts from Stage 1 and +place them in the expected `runtimes/` layout: + +```yaml +# Download and place native binaries +- download: current + artifact: native-linux-x64 +- download: current + artifact: native-win-x64 +- script: | + mkdir -p libs/native/bftree-garnet/runtimes/linux-x64/native + mkdir -p libs/native/bftree-garnet/runtimes/win-x64/native + cp $(Pipeline.Workspace)/native-linux-x64/libbftree_garnet.so \ + libs/native/bftree-garnet/runtimes/linux-x64/native/ + cp $(Pipeline.Workspace)/native-win-x64/bftree_garnet.dll \ + libs/native/bftree-garnet/runtimes/win-x64/native/ +``` + +The rest of the existing pipeline continues unchanged, with two modifications: + +1. **ESRP binary signing** — extend the file pattern to include the new native DLL: + ``` + Pattern: Garnet*.dll,Tsavorite*.dll,Garnet*.exe,HdrHistogram.dll,native_device.dll,bftree_garnet.dll,*Lua.dll + ``` + +2. **`dotnet pack`** — no changes needed. The `ContentWithTargetPath` items in + `BfTreeInterop.csproj` automatically include the native binaries in the NuGet. + +**Signing summary:** + +| Artifact | Signed? | Method | +|---|---|---| +| `bftree_garnet.dll` (Windows) | ✅ Yes | ESRP Authenticode signing (`CP-230012`), added to existing binary signing glob | +| `libbftree_garnet.so` (Linux) | ❌ No | Linux shared libraries are not Authenticode-signed (same as `libnative_device.so`) | +| `Microsoft.Garnet.*.nupkg` | ✅ Yes | Existing ESRP NuGet signing step (`CP-401405`), no changes needed | +| `garnet-server.*.nupkg` | ✅ Yes | Same existing step, no changes needed | + +**Local development experience:** + +- `dotnet build` triggers `cargo build --release` via the MSBuild target → copies the + current-platform native lib to the output directory → everything works out of the box +- The `Condition="'$(CI)' != 'true'"` guard on the MSBuild target prevents the local + cargo build from running in CI (where pre-built binaries are provided by Stage 1) +- No Rust toolchain is needed if you're not modifying FFI code — NuGet restore from + nuget.org brings pre-built binaries for all platforms in the published package + +**Version management:** + +- The `bf-tree` crate version is pinned in `libs/native/bftree-garnet/Cargo.toml` + (e.g., `bf-tree = "0.4"`) +- To update: bump the version in `Cargo.toml`; the next pipeline run picks it up +- No separate NuGet versioning — the native lib ships inside the existing + `Microsoft.Garnet` NuGet, versioned together with Garnet via `Version.props` + +#### 11c. Implement `BfTreeService` (C# interop wrapper) + +> **Reference:** [`DiskANNService.cs`](https://github.com/microsoft/garnet/blob/dev/libs/server/Resp/Vector/DiskANNService.cs) — +> wraps the unmanaged DiskANN library. BfTreeService follows the same pattern with +> P/Invoke to the Rust shared library. + +**File:** `libs/native/bftree-garnet/BfTreeService.cs` (new, inside BfTreeInterop project) + +
    +BfTreeService implementation (click to expand) + +```csharp +/// Wraps the native Bf-Tree library (bftree.dll / libbftree.so). +/// Provides managed C# interface for BfTree lifecycle and operations. +internal sealed class BfTreeService : IDisposable +{ + /// Create a new BfTree instance. Returns native pointer. + internal nint CreateIndex(ulong cacheSize, uint minRecordSize, + uint maxRecordSize, uint maxKeyLen, uint leafPageSize, + byte storageBackend, string filePath) + { + fixed (byte* pathPtr = Encoding.UTF8.GetBytes(filePath)) + return NativeBfTreeMethods.bftree_create( + cacheSize, minRecordSize, maxRecordSize, + maxKeyLen, leafPageSize, storageBackend, + pathPtr, filePath.Length); + } + + /// Insert a key-value pair. Returns result code. + internal BfTreeInsertResult Insert(nint tree, + ReadOnlySpan key, ReadOnlySpan value) + { + fixed (byte* kp = key, vp = value) + return (BfTreeInsertResult)NativeBfTreeMethods.bftree_insert( + tree, kp, key.Length, vp, value.Length); + } + + /// Point read. Writes value into output. + internal RangeIndexResult Read(nint tree, + ReadOnlySpan key, uint maxRecordSize, + ref SpanByteAndMemory output) + { + Span buffer = stackalloc byte[(int)maxRecordSize]; + fixed (byte* kp = key, bp = buffer) + { + var result = NativeBfTreeMethods.bftree_read( + tree, kp, key.Length, bp, buffer.Length); + if (result < 0) return RangeIndexResult.NotFound; + // Copy result into output SpanByteAndMemory + // ... + return RangeIndexResult.OK; + } + } + + /// Delete a key. + internal void Delete(nint tree, ReadOnlySpan key) + { + fixed (byte* kp = key) + NativeBfTreeMethods.bftree_delete(tree, kp, key.Length); + } + + /// Scan with count. Iterates and writes RESP-formatted results into output. + internal RangeIndexResult ScanWithCount(nint tree, + ReadOnlySpan startKey, int count, byte returnField, + ref SpanByteAndMemory output, out int resultCount) + { + // 1. Call bftree_scan_with_count → get iterator pointer + // 2. Loop: bftree_scan_next → append to output + // 3. bftree_scan_drop → free iterator + // returnField: 0=Key, 1=Value, 2=KeyAndValue (maps to ScanReturnField enum) + } + + /// Scan with end key. Same pattern as ScanWithCount. + internal RangeIndexResult ScanWithEndKey(nint tree, + ReadOnlySpan startKey, ReadOnlySpan endKey, + byte returnField, + ref SpanByteAndMemory output, out int resultCount) + { /* same iterator pattern */ } + + /// Scan all entries in the tree, ordered by key. + /// Useful for streaming the full tree state to a replica. + /// Internally calls ScanWithCount with start_key=\x00 and count=int.MaxValue. + /// Only supported for disk-backed trees (memory-only do not support scan). + internal List ScanAll(ScanReturnField returnField = ScanReturnField.KeyAndValue) + => ScanWithCount(new byte[] { 0 }, int.MaxValue, returnField); + + /// Snapshot a disk-backed BfTree in place. + internal void Snapshot(nint tree) + { + int result = NativeBfTreeMethods.bftree_snapshot(tree); + if (result != 0) + throw new InvalidOperationException("Failed to snapshot BfTree."); + } + + /// Recover a disk-backed BfTree from its data file. + internal nint RecoverFromSnapshot(string filePath, + ulong cacheSize, uint minRecordSize, uint maxRecordSize, + uint maxKeyLen, uint leafPageSize) + { + var pathBytes = Encoding.UTF8.GetBytes(filePath); + fixed (byte* pp = pathBytes) + return NativeBfTreeMethods.bftree_new_from_snapshot( + pp, pathBytes.Length, + cacheSize, minRecordSize, maxRecordSize, + maxKeyLen, leafPageSize); + } + + /// Drop/free a BfTree instance. + internal void Drop(nint tree) + => NativeBfTreeMethods.bftree_drop(tree); + + public void Dispose() { /* cleanup any global state */ } +} +``` + +
    + +
    +P/Invoke declarations (click to expand) + +```csharp +/// P/Invoke declarations for the native Bf-Tree library. +/// Uses LibraryImport (source-generated, zero-overhead) matching the DiskANN pattern. +internal static unsafe partial class NativeBfTreeMethods +{ + private const string LibName = "bftree_garnet"; + + [LibraryImport(LibName)] internal static partial nint bftree_create( + ulong cacheSize, uint minRecord, uint maxRecord, + uint maxKeyLen, uint leafPageSize, byte storageBackend, + byte* filePath, int filePathLen); + + [LibraryImport(LibName)] internal static partial int bftree_insert( + nint tree, byte* key, int keyLen, byte* value, int valueLen); + + [LibraryImport(LibName)] internal static partial int bftree_read( + nint tree, byte* key, int keyLen, byte* outBuffer, int outBufferLen, + int* outValueLen); + + [LibraryImport(LibName)] internal static partial void bftree_delete( + nint tree, byte* key, int keyLen); + + [LibraryImport(LibName)] internal static partial nint bftree_scan_with_count( + nint tree, byte* startKey, int startKeyLen, int count, byte returnField); + + [LibraryImport(LibName)] internal static partial nint bftree_scan_with_end_key( + nint tree, byte* startKey, int startKeyLen, + byte* endKey, int endKeyLen, byte returnField); + + [LibraryImport(LibName)] internal static partial int bftree_scan_next( + nint iter, byte* outBuffer, int outBufferLen, + out int keyLen, out int valueLen); + + [LibraryImport(LibName)] internal static partial void bftree_scan_drop(nint iter); + + [LibraryImport(LibName)] internal static partial int bftree_snapshot(nint tree); + + [LibraryImport(LibName)] internal static partial nint bftree_new_from_snapshot( + byte* filePath, int filePathLen, + ulong cacheSize, uint minRecord, uint maxRecord, + uint maxKeyLen, uint leafPageSize); + + [LibraryImport(LibName)] internal static partial void bftree_drop(nint tree); +} + +/// Result codes from BfTree insert operations +internal enum BfTreeInsertResult +{ + Success = 0, + InvalidKV = 1, +} +``` + +
    + +**Rust FFI side** (`libs/native/bftree-garnet/src/lib.rs`): + +
    +Rust FFI exports (click to expand) + +```rust +use bf_tree::{BfTree, Config, LeafInsertResult, LeafReadResult, ScanReturnField, + ScanIter, StorageBackend}; +use std::path::Path; +use std::slice; + +// Storage backend constants (matches C# StorageBackendType enum) +const STORAGE_MEMORY: u8 = 1; + +#[no_mangle] +pub unsafe extern "C" fn bftree_create( + cb_size_byte: u64, cb_min_record_size: u32, cb_max_record_size: u32, + cb_max_key_len: u32, leaf_page_size: u32, + storage_backend: u8, file_path: *const u8, file_path_len: i32, +) -> *mut BfTree { + let mut config = Config::default(); + // ... apply non-zero config fields ... + if storage_backend == STORAGE_MEMORY { + config.cache_only(true); + } else { + // Disk-backed (default) + let path_str = /* UTF-8 from file_path */; + config.storage_backend(StorageBackend::Std); + config.file_path(Path::new(path_str)); + } + match BfTree::with_config(config, None) { + Ok(tree) => Box::into_raw(Box::new(tree)), + Err(_) => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub unsafe extern "C" fn bftree_insert( + tree: *mut BfTree, key: *const u8, key_len: i32, + value: *const u8, value_len: i32, +) -> i32 { /* ... 0=Success, 1=InvalidKV */ } + +#[no_mangle] +pub unsafe extern "C" fn bftree_read( + tree: *mut BfTree, key: *const u8, key_len: i32, + out_buffer: *mut u8, out_buffer_len: i32, out_value_len: *mut i32, +) -> i32 { /* ... 0=Found, -1=NotFound, -2=Deleted, -3=InvalidKey */ } + +#[no_mangle] +pub unsafe extern "C" fn bftree_delete( + tree: *mut BfTree, key: *const u8, key_len: i32, +) { /* ... */ } + +#[no_mangle] +pub unsafe extern "C" fn bftree_drop(tree: *mut BfTree) { + if !tree.is_null() { drop(Box::from_raw(tree)); } +} + +/// Snapshot a disk-backed BfTree in place. Returns 0 on success, -1 on failure. +#[no_mangle] +pub unsafe extern "C" fn bftree_snapshot(tree: *mut BfTree) -> i32 { /* ... */ } + +/// Recover a disk-backed BfTree from its data file. Returns tree ptr or null. +#[no_mangle] +pub unsafe extern "C" fn bftree_new_from_snapshot( + file_path: *const u8, file_path_len: i32, + cb_size_byte: u64, cb_min_record_size: u32, cb_max_record_size: u32, + cb_max_key_len: u32, leaf_page_size: u32, +) -> *mut BfTree { /* ... BfTree::new_from_snapshot(config, None) */ } + +/// STUB: Snapshot a memory-only (cache_only) BfTree to disk. Returns -1 (not yet implemented). +#[no_mangle] +pub unsafe extern "C" fn bftree_snapshot_memory( + _tree: *mut BfTree, _path: *const u8, _path_len: i32, +) -> i32 { -1 /* TODO: implement when bf-tree adds cache_only snapshot */ } + +/// STUB: Recover a memory-only (cache_only) BfTree from disk. Returns null (not yet implemented). +#[no_mangle] +pub unsafe extern "C" fn bftree_recover_memory( + _path: *const u8, _path_len: i32, + _cb_size_byte: u64, _cb_min_record_size: u32, _cb_max_record_size: u32, + _cb_max_key_len: u32, _leaf_page_size: u32, +) -> *mut BfTree { std::ptr::null_mut() /* TODO: implement when bf-tree adds cache_only recovery */ } + +// scan_with_count, scan_with_end_key, scan_next, scan_drop follow similar patterns. +// See libs/native/bftree-garnet/src/lib.rs for the full implementation. +``` + +
    + +--- + +### Step 12: Add `RangeIndexManager` to `FunctionsState` + +> **Reference:** `libs/server/Storage/Functions/FunctionsState.cs` +> This class holds shared state accessible from Tsavorite session function callbacks +> (e.g., `appendOnlyFile`, `watchVersionMap`, `cacheSizeTracker`). +> +> **Reference:** On `dev`, `FunctionsState` has a +> [`vectorManager` field](https://github.com/microsoft/garnet/blob/dev/libs/server/Storage/Functions/FunctionsState.cs) +> and [`StorageSession`](https://github.com/microsoft/garnet/blob/dev/libs/server/Storage/Session/StorageSession.cs) +> has a matching `vectorManager` field — follow the same pattern for `rangeIndexManager`. + +```csharp +// Add field: +internal readonly RangeIndexManager rangeIndexManager; + +// Initialize in constructor: +this.rangeIndexManager = new RangeIndexManager(logger); +``` + +Also add `rangeIndexManager` to `StorageSession` so that `RangeIndexOps.cs` can access it: + +> **Reference:** `libs/server/Storage/Session/StorageSession.cs` +> The session holds context fields (`stringBasicContext`, `objectBasicContext`, +> `unifiedBasicContext`) and shared state. Add the manager as a field. + +```csharp +internal readonly RangeIndexManager rangeIndexManager; +``` + +--- + +### Step 13: Checkpoint & Recovery + +> The stub in the store is automatically serialized by Tsavorite during checkpoint +> (as part of the hybrid log). On recovery, BfTrees are **not** proactively recreated. +> Instead, `RangeIndexManager` generates a fresh `processInstanceId` at startup, and +> `ReadRangeIndex()` lazily detects stale stubs (via `ProcessInstanceId` mismatch) on +> first access, restoring from snapshot at that point. This matches how VectorSet +> handles recovery on the dev branch. + +**Checkpoint:** No separate pre-checkpoint scan. BfTrees are serialized per-record +during the snapshot page flush via the `OnSnapshotRecord` callback (see section B below). + +**Recovery:** No proactive store scan needed. After Tsavorite recovery, the stubs contain +`ProcessInstanceId` values from the prior process. Since `RangeIndexManager` generates a +fresh `Guid` at startup, every stub will mismatch on first access. The existing +`ReadRangeIndex()` flow handles this lazily: + +1. First `RI.*` command on a recovered key → `Read_MainStore` returns stub from store +2. `ProcessInstanceId != this.processInstanceId` → stale +3. Promote to exclusive lock → restore BfTree from snapshot → update stub via RMW +4. Proceed with the requested operation + +This avoids a potentially expensive full-store scan at startup, and only pays the +restore cost for indexes that are actually accessed. + +--- + +## Result Enum + +```csharp +public enum RangeIndexResult +{ + OK, + NotFound, + Deleted, + InvalidKey, + Error, +} +``` + +--- + +## Complete File Inventory + +### New Files (12 files) + +| # | File Path | Purpose | Lines (est.) | +|---|---|---|---| +| 1 | `libs/server/Resp/RangeIndex/RangeIndexManager.cs` | Main class: constants, Try* methods, recovery | ~200 | +| 2 | `libs/server/Resp/RangeIndex/RangeIndexManager.Index.cs` | Stub struct, Create/Read/RecreateIndex | ~120 | +| 3 | `libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs` | ReadRangeIndexLock, ReadRangeIndex, ReadOrCreateRangeIndex | ~250 | +| 4 | `libs/server/Resp/RangeIndex/RangeIndexManager.Cleanup.cs` | Post-drop async cleanup | ~100 | +| 5 | `libs/server/Resp/RangeIndex/RespServerSessionRangeIndex.cs` | RESP command handlers | ~400 | +| 6 | `libs/server/Storage/Session/MainStore/RangeIndexOps.cs` | Storage session wrappers | ~200 | +| 7 | `libs/native/bftree-garnet/BfTreeInterop.csproj` | C# interop project with MSBuild cargo target | ~50 | +| 8 | `libs/native/bftree-garnet/NativeBfTreeMethods.cs` | [LibraryImport] P/Invoke declarations | ~60 | +| 9 | `libs/native/bftree-garnet/BfTreeService.cs` | High-level managed BfTree operations wrapper | ~150 | +| 10 | `libs/native/bftree-garnet/Cargo.toml` | Rust crate config, depends on bf-tree from crates.io | ~15 | +| 11 | `libs/native/bftree-garnet/src/lib.rs` | Rust #[no_mangle] extern "C" FFI exports | ~200 | +| 12 | `test/Garnet.test/RangeIndexTests.cs` | Integration tests | ~300 | + +### Modified Files (10 files) + +| # | File Path | Change | Scope | +|---|---|---|---| +| 1 | `libs/storage/Tsavorite/.../LogRecord.cs` | Wire `RecordType` propagation through `InitializeRecord()` | ~15 lines | +| 2 | `libs/server/Resp/Parser/RespCommand.cs` | Add RI* enum values + `ParseRangeIndexCommand()` | ~50 lines | +| 3 | `libs/server/Resp/RespServerSession.cs` | Add RI* command dispatch entries | ~15 lines | +| 4 | `libs/server/Storage/Functions/MainStore/RMWMethods.cs` | Add RICREATE/RISET/RIDEL cases in InitialUpdater, InPlaceUpdater, CopyUpdater | ~60 lines | +| 5 | `libs/server/Storage/Functions/MainStore/ReadMethods.cs` | Add `RecordType` type guards in Reader | ~20 lines | +| 6 | `libs/server/Storage/Functions/MainStore/DeleteMethods.cs` | Add `RecordType` deletion guard | ~10 lines | +| 7 | `libs/server/Storage/Functions/FunctionsState.cs` | Add `rangeIndexManager` field | ~5 lines | +| 8 | `libs/server/Storage/Session/StorageSession.cs` | Add `rangeIndexManager` field | ~5 lines | +| 9 | `libs/server/API/IGarnetApi.cs` + `GarnetApi.cs` | Add RangeIndex* method declarations + delegation | ~60 lines | +| 10 | `libs/server/Garnet.server.csproj` | Add `` to `BfTreeInterop.csproj` | ~2 lines | + +--- + +## Persistence, Checkpoint, Migration, and Replication + +This section covers how BfTree data survives page flush, checkpoint, recovery, replica +sync, and key migration. The **critical design consideration** is that BfTree stores its +data entirely **outside** Tsavorite (in its own circular buffer + disk files). The stub in +Tsavorite is just a pointer + config — the actual data must be managed separately at every +persistence boundary. This is fundamentally different from built-in object types (Hash, +List, Set, SortedSet) which serialize their data directly into Tsavorite's log and benefit +from automatic checkpoint/migration handling. + +### Background: How Tsavorite Persists Records + +> **Reference:** `libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs` + +- Tsavorite writes hybrid log pages **verbatim to disk** — raw memory bytes, no per-record + transformation or callback. +- `RecordInfo` flags and `RecordType` byte are part of the record header + and survive flush as-is. +- The `TreePtr` field in the stub is a raw `nint` — it is written to disk as a raw + 64-bit value. After the BfTree is evicted and freed (see Page Flush below), this + pointer becomes stale. On re-read from disk, `ProcessInstanceId` mismatch (set to a + sentinel during eviction) triggers `RecreateIndex()` to restore the BfTree from its + snapshot. +- There is an **observer pattern** via `store.Log.SubscribeEvictions()` that fires per-page, + which we use to snapshot and free BfTrees whose stubs are being evicted. + +### The Four Persistence Boundaries + +Both storage backends support snapshot and recovery at the Garnet level. For +**disk-backed** trees, snapshot and recovery are fully implemented via `BfTree::snapshot()` +and `BfTree::new_from_snapshot(config)`. For **memory-only** trees, bf-tree's `cache_only` +mode does not yet implement snapshot/recovery — Garnet will throw `NotSupportedException` +at the FFI boundary until bf-tree adds this capability, at which point memory-only trees +will be snapshotted/recovered identically to disk-backed trees. + +For disk-backed trees: +- **Snapshot**: `BfTree::snapshot()` drains the circular buffer and writes the index + structure to the tree's own data file. +- **Recovery**: `BfTree::new_from_snapshot(config)` reopens the existing data file and + resumes operations. + +The `StorageBackend` byte in the stub determines the behavior. + +| Boundary | What happens | RangeIndex action required | +|---|---|---| +| **Page flush** | Tsavorite moves pages to read-only, then flushes to disk. | `OnFlushRecord` snapshots BfTree + sets Flushed flag. Next access promotes stub to tail via RIPROMOTE. `DisposeRecord(PageEviction)` frees native instance. `OnDiskReadRecord` zeros TreeHandle on disk reads. Lazy restore via `RestoreTreeFromFlush`. | +| **Checkpoint** | Tsavorite takes a full checkpoint of the hybrid log. All stubs are included. | Stubs in mutable region are flushed (triggering `OnFlushRecord`). Promoted stubs capture latest BfTree state. | +| **Recovery** | Tsavorite recovers from checkpoint. Stubs loaded from disk. | `OnDiskReadRecord` zeros TreeHandle. First access promotes (IsFlushed) → restores BfTree from flush file via `RestoreTreeFromFlush`. | +| **Key migration** | Individual keys are transferred to another node during cluster slot migration. | For disk-backed trees: serialize the BfTree snapshot alongside the stub. Memory-only trees: same approach once snapshot is supported. | +| **Replica sync** | Full checkpoint is sent to a replica. | For disk-backed trees: send BfTree data files alongside checkpoint, or use `ScanAll()` to stream the full tree state record-by-record. Memory-only trees: send snapshot once supported. | + +### Design: Snapshot File Management + +Each BfTree instance has a **deterministic file path** derived from its Garnet key bytes +and the server's data directory. + +**Working copy** (where the live BfTree stores its data): +``` +{dataDir}/rangeindex/{key_hash}/data.bftree +``` + +**Flush snapshot** (point-in-time copy for cold-read recovery after page eviction): +``` +{dataDir}/rangeindex/{key_hash}/flush.bftree +``` + +`key_hash` is the XxHash128 of the key bytes, formatted as a 32-character hex string +(`Guid.ToString("N")`). All paths are derived deterministically — no in-memory registry +or per-stub file paths are needed. The user does not specify any paths; `RI.CREATE DISK` +derives them automatically. + +### Design: Deterministic Path Derivation (Implemented) + +The `RangeIndexManager` derives paths using `XxHash128` hashing of the key bytes: + +```csharp +// In RangeIndexManager.cs +internal string DeriveWorkingPath(ReadOnlySpan keyBytes) + => Path.Combine(dataDir, "rangeindex", HashKeyToDirectoryName(keyBytes), "data.bftree"); + +internal string DeriveFlushPath(ReadOnlySpan keyBytes) + => Path.Combine(dataDir, "rangeindex", HashKeyToDirectoryName(keyBytes), "flush.bftree"); + +internal static string HashKeyToDirectoryName(ReadOnlySpan keyBytes) +{ + var hash = XxHash128.Hash(keyBytes); + return new Guid(hash).ToString("N"); +} +``` + +--- + +### A. Page Flush (Hybrid Log Eviction) — Implemented + +> **Reference:** +> - `libs/storage/Tsavorite/cs/src/core/Allocator/ObjectAllocatorImpl.cs` — +> `OnFlushRecordsInRange()` calls `storeFunctions.OnFlushRecord(ref logRecord)` per +> valid record on original in-memory pages before flush, gated by `CallOnFlush`. +> - `libs/storage/Tsavorite/cs/src/core/Index/StoreFunctions/IRecordTrigger.cs` — +> `OnFlushRecord`, `OnDiskReadRecord`, `DisposeRecord` callbacks. +> - `libs/server/Storage/Functions/GarnetRecordTrigger.cs` — Garnet implementation. +> - `libs/server/Resp/RangeIndex/RangeIndexManager.Locking.cs` — Lazy promote + restore. + +**Three-phase lifecycle for page eviction:** + +1. **`OnFlushRecord` — snapshot + set flushed flag:** Called per valid record in + `OnPagesMarkedReadOnlyWorker()` on the **original in-memory records** (not a copy), + before the page is flushed to disk. For RangeIndex records: + - Calls `BfTreeService.SnapshotToFile(flushPath)`: disk-backed trees do in-place + `snapshot()` then `File.Copy` to `{dataDir}/rangeindex/{key_hash}/flush.bftree`. + - Sets the `Flushed` flag on the in-memory stub so the next operation promotes it. + +2. **Lazy promote on next access:** `ReadRangeIndex()` detects `IsFlushed` → releases + shared lock → issues `RIPROMOTE` RMW → `CopyUpdater` copies stub to tail (mutable + region), clears flushed flag, transfers tree ownership by clearing source TreeHandle. + This ensures the stub will be re-flushed (with latest BfTree state) on the next + checkpoint or ReadOnly transition. + +3. **`DisposeRecord(PageEviction)` — free native instance:** Called when the page is + evicted past HeadAddress. `GarnetRecordTrigger` calls `DisposeTreeIfOwned()` which + unregisters and frees the native BfTree. The source TreeHandle was already cleared + by RIPROMOTE's CopyUpdater (if the stub was promoted), so this is a no-op for + promoted records. + +**`OnDiskReadRecord` — invalidate stale handles:** Called per record loaded from disk +(recovery, pending reads, push scans). Zeros `TreeHandle` on RangeIndex +stubs so operations detect the stub as "needs lazy restore." + +**On cold read (after eviction past HeadAddress):** +1. Tsavorite issues pending read from disk → `OnDiskReadRecord` zeros TreeHandle +2. `ReadRangeIndex()` detects `IsFlushed` → promotes stub to tail via RIPROMOTE RMW +3. CopyUpdater from disk source: stub already has TreeHandle=0 (from OnDiskReadRecord) +4. After promote, `ReadRangeIndex()` detects `TreeHandle == 0` → calls `RestoreTreeFromFlush` +5. Derives flush path from key bytes → `RecoverFromSnapshot(flushPath, config...)` +6. Registers restored BfTree → issues RIRESTORE RMW to set TreeHandle on mutable stub +7. Re-reads stub → TreeHandle is valid → returns data from restored BfTree + +**Internal RMW commands:** +- `RIPROMOTE` — Copies stub to tail, clears flushed flag, transfers tree ownership. + IPU: asserts flushed flag is never set on mutable records. NeedCopyUpdate: always true. +- `RIRESTORE` — Sets TreeHandle on a mutable stub after lazy restore from flush file. + IPU: sets TreeHandle from `input.arg1`. CopyUpdater: copies stub + sets TreeHandle. + +--- + +### B. Checkpoint + +**No separate pre-checkpoint scan needed.** During a snapshot checkpoint, Tsavorite +flushes pages to the snapshot file. A per-record callback is invoked for each record +being written to the snapshot. For RangeIndex stubs, this callback ensures a stable +BfTree snapshot file exists for the checkpoint. + +**Important: snapshot consistency.** The bf-tree `snapshot()` method flushes state to +the **same SSD backend file** the tree uses for operations. It does not create a separate +point-in-time copy. Subsequent writes to the BfTree modify the same file, overwriting the +serialized state. For checkpoint correctness, we need a point-in-time copy. + +**Approach: reflink (copy-on-write) file clone.** After calling `snapshot()` to flush the +BfTree state, create a **reflink copy** of the backend file to the checkpoint directory. +On filesystems that support it (btrfs, XFS, bcachefs, ZFS), this is an instant O(1) +operation that shares data blocks via copy-on-write — subsequent BfTree writes allocate +new blocks without affecting the checkpoint copy. On filesystems without reflink support +(ext4, NTFS), this falls back to a regular file copy. + +```csharp +private static void ReflinkCopy(string source, string destination) +{ + Directory.CreateDirectory(Path.GetDirectoryName(destination)); + // Try reflink first (Linux: ioctl FICLONE), fall back to regular copy + if (!TryReflinkClone(source, destination)) + File.Copy(source, destination, overwrite: true); +} +``` + +**Coordination with CopyUpdater via `SerializationPhase` state machine:** + +The CopyUpdater (Step 8) may have already serialized the BfTree for this checkpoint +version (when `IsInNewVersion` was true). The `SerializationPhase` state machine +(`REST` → `SERIALIZING` → `SERIALIZED`) coordinates this: + +- If `SERIALIZED`: CopyUpdater already wrote the file — the snapshot callback just + reflink-copies it to the checkpoint directory. No need to touch the live BfTree. +- If `SERIALIZING`: Another thread is writing — spin-yield until `SERIALIZED`. +- If `REST`: No CopyUpdate happened — the snapshot callback snapshots the BfTree itself + (under exclusive lock) and reflink-copies. + +> **Tsavorite change required:** Invoke a per-record callback during snapshot page flush, +> analogous to `DisposeRecord` on eviction. This could be a new `DisposeReason` value +> (e.g., `DisposeReason.SnapshotCheckpoint`) or a separate `OnSnapshotToDisk` callback. +> The implementation snapshots the BfTree but does **not** free it (unlike eviction) — +> the BfTree remains live in memory. + +```csharp +// In Garnet's record disposer / callback: +// Called per-record during snapshot page flush +public void OnSnapshotRecord(ref LogRecord logRecord, Guid checkpointToken, string checkpointDir) +{ + if (logRecord.RecordType != RangeIndexManager.RangeIndexRecordType) + return; + + ReadIndex(logRecord.ValueSpan, out var treePtr, ...); + if (treePtr == nint.Zero) return; + + var keyBytes = logRecord.Key; + var backendPath = DeriveBackendPath(keyBytes, rangeIndexManager.dataDir); + var checkpointPath = DeriveSnapshotPath(keyBytes, checkpointDir, checkpointToken); + + // Check SerializationPhase — CopyUpdater may have already serialized + while (true) + { + var phase = rangeIndexManager.GetSerializationPhase(treePtr); + if (phase == SerializationPhase.SERIALIZED) + { + // CopyUpdater already wrote the snapshot — just reflink-copy + ReflinkCopy(backendPath, checkpointPath); + return; + } + if (phase == SerializationPhase.SERIALIZING) + { + // Another thread is writing — wait + Thread.Yield(); + continue; + } + + // REST — we need to snapshot the BfTree ourselves + if (rangeIndexManager.TryTransitionSerializationPhase(treePtr, + SerializationPhase.REST, SerializationPhase.SERIALIZING)) + break; + } + + // Acquire exclusive lock, snapshot, reflink-copy + var keyHash = HashKeyToGuid(keyBytes); + rangeIndexManager.rangeIndexLocks.AcquireExclusiveLock(keyHash, out var lockToken); + try + { + rangeIndexManager.service.Snapshot(treePtr); + rangeIndexManager.SetSerializationPhase(treePtr, SerializationPhase.SERIALIZED); + ReflinkCopy(backendPath, checkpointPath); + } + finally + { + rangeIndexManager.rangeIndexLocks.ReleaseExclusiveLock(lockToken); + } +} +``` + +**After checkpoint:** Reset all `SerializationPhase` states to `REST`. Optionally clean up +old snapshot files from previous checkpoints. + +--- + +### C. Recovery + +> **Reference:** `libs/server/Databases/SingleDatabaseManager.cs` — `RecoverCheckpoint()` +> restores the store from a checkpoint. + +**No proactive store scan.** After Tsavorite recovery, all stubs are loaded with stale +`ProcessInstanceId` values (from the prior process). `RangeIndexManager` generates a fresh +`processInstanceId = Guid.NewGuid()` at construction, so every stub will mismatch. + +BfTrees are restored **lazily** on first access via the existing `ReadRangeIndex()` flow: + +1. First `RI.*` command on a recovered key → `Read_MainStore` returns the stub +2. `stub.ProcessInstanceId != this.processInstanceId` → stale pointer detected +3. Promote shared lock to exclusive +4. Derive checkpoint snapshot path from key bytes + checkpoint directory + checkpoint token +5. `newTreePtr = service.RestoreFromSnapshot(snapshotPath, config...)` +6. Issue RMW with `RecreateIndexArg` → updates `TreePtr` and `ProcessInstanceId` in stub +7. Release exclusive, re-acquire shared, proceed with operation + +This avoids a full-store scan at startup. Only indexes that are actually accessed pay +the restore cost. This is the same approach used by VectorManager on the dev branch +(where `ResumePostRecovery()` is a no-op TODO). + +--- + +### D. Replica Sync (Sending Checkpoint to Replica) + +> **Reference:** `libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSession.cs` — +> `SendCheckpoint()` sends checkpoint files categorized by `CheckpointFileType` enum. +> `libs/cluster/Server/Replication/CheckpointFileType.cs` — currently defines types for +> `STORE_HLOG`, `STORE_HLOG_OBJ`, `STORE_INDEX`, `STORE_SNAPSHOT`, +> `STORE_SNAPSHOT_OBJ`. A new `RANGEINDEX_SNAPSHOT` type must be added. + +**Problem:** The Tsavorite checkpoint files contain stubs with stale `TreePtr` values. The +actual BfTree data is in separate snapshot files that are not part of Tsavorite's file set. +When sending a checkpoint to a replica, we must send these additional files. + +**Solution:** + +#### 1. Add a new `CheckpointFileType` + +```csharp +// In CheckpointFileType.cs, add: +RANGEINDEX_SNAPSHOT = 11, // BfTree snapshot file +``` + +#### 2. Extend `SendCheckpoint()` to send BfTree snapshots + +```csharp +// In ReplicaSyncSession.cs, inside SendCheckpoint(), after sending store files: + +// Send RangeIndex snapshot files +var riSnapshotDir = Path.Combine(checkpointDir, "rangeindex"); +if (Directory.Exists(riSnapshotDir)) +{ + foreach (var snapshotFile in Directory.EnumerateFiles( + riSnapshotDir, "*.bftree", SearchOption.AllDirectories)) + { + // Derive a unique segment ID from the file path + var relativePath = Path.GetRelativePath(checkpointDir, snapshotFile); + SendFileByPath(snapshotFile, CheckpointFileType.RANGEINDEX_SNAPSHOT, + relativePath, fileToken); + } +} +``` + +#### 3. Extend the replica receiver to handle BfTree snapshots + +> **Reference:** `libs/cluster/Session/RespClusterMigrateCommands.cs` — processes +> incoming checkpoint file segments. + +```csharp +// On the replica side, when receiving RANGEINDEX_SNAPSHOT file type: +case CheckpointFileType.RANGEINDEX_SNAPSHOT: + // Write snapshot file to local checkpoint directory + var localPath = Path.Combine(localCheckpointDir, "rangeindex", relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(localPath)); + WriteSegmentToFile(localPath, segment); + break; +``` + +After all files are received and Tsavorite recovery completes, the replica's +`RangeIndexManager` has a fresh `processInstanceId`. BfTrees are restored lazily on +first access — `ReadRangeIndex()` detects the `ProcessInstanceId` mismatch in each stub +and restores from the snapshot files at the expected paths. + +--- + +### E. Key Migration (Cluster Slot Migration) + +> **Reference:** `libs/cluster/Server/Migration/MigrateSessionKeys.cs` — +> migrates individual keys during cluster slot migration via `MigrateKeysFromStore()`. +> Key scanning: `MigrateScanFunctions.cs` — iterates store records for slot matching. +> Receiver: `libs/cluster/Session/RespClusterMigrateCommands.cs`. +> Currently, migration handles only standard string and object records. RangeIndex +> keys require special 2-phase migration since BfTree data lives outside Tsavorite. + +**Problem:** During slot migration, individual keys are transferred to the target node. +For a RangeIndex key, we can't just send the 51-byte stub — we must also send the +entire BfTree data (all entries in the index). The target node must recreate the BfTree +from this data. + +**Solution: 2-Phase Migration** + +#### Phase 1: Serialize BfTree data + +When the migration scanner encounters a RangeIndex record (identified by `RecordType`): + +```csharp +// In MigrateScanFunctions.cs (add RangeIndex detection): +if (srcLogRecord.RecordType == RangeIndexManager.RangeIndexRecordType) +{ + mss.EncounteredRangeIndex(ref key, ref value); + return; // Don't transmit the stub yet — Phase 2 +} +``` + +```csharp +// In MigrateSessionKeys.cs (new method): +internal void EncounteredRangeIndex(ReadOnlySpan key, ReadOnlySpan value) +{ + // Save key + value for Phase 2 + rangeIndexKeysToMigrate.Add((key.ToByteArray(), value.ToByteArray())); +} +``` + +#### Phase 2: Transmit BfTree snapshot + stub + +```csharp +// In MigrateSessionKeys.cs, TransmitRangeIndexKeys() (new method): +internal bool TransmitRangeIndexKeys() +{ + foreach (var (keyBytes, stubBytes) in rangeIndexKeysToMigrate) + { + // 1. Read stub to get TreePtr + RangeIndexManager.ReadIndex(stubBytes, out var treePtr, out var cacheSize, + out var minRecordSize, out var maxRecordSize, + out var maxKeyLen, out var leafPageSize, + out var storageBackend, ...); + + // 2. Snapshot BfTree to a temporary file + var tempSnapshotPath = Path.GetTempFileName(); + rangeIndexManager.SnapshotToPath(treePtr, tempSnapshotPath); + + // 3. Read snapshot file bytes + var snapshotBytes = File.ReadAllBytes(tempSnapshotPath); + + // 4. Send to target: [stub (51 bytes)] + [snapshot_length (4 bytes)] + // + [snapshot_bytes (N bytes)] + var payload = new byte[stubBytes.Length + 4 + snapshotBytes.Length]; + Buffer.BlockCopy(stubBytes, 0, payload, 0, stubBytes.Length); + BitConverter.TryWriteBytes(payload.AsSpan(stubBytes.Length), snapshotBytes.Length); + Buffer.BlockCopy(snapshotBytes, 0, payload, stubBytes.Length + 4, + snapshotBytes.Length); + + // 5. Transmit as a special store type "RISTORE" + gcs.TryWriteRangeIndexMigration(keyBytes, payload); + + // 6. Cleanup temp file + File.Delete(tempSnapshotPath); + + // 7. Delete local key (if not COPY) + if (!isCopy) DeleteLocalKey(keyBytes); + } + return true; +} +``` + +#### Receiver side + +```csharp +// In RespClusterMigrateCommands.cs, add case for RISTORE: +case "RISTORE": + // 1. Parse payload: stub + snapshot bytes + var stub = payload.AsSpan(0, RangeIndexManager.IndexSizeBytes); + var snapshotLen = BitConverter.ToInt32( + payload.AsSpan(RangeIndexManager.IndexSizeBytes)); + var snapshotBytes = payload.AsSpan( + RangeIndexManager.IndexSizeBytes + 4, snapshotLen); + + // 2. Write snapshot to local file + var localSnapshotPath = DeriveSnapshotPath(key); + File.WriteAllBytes(localSnapshotPath, snapshotBytes.ToArray()); + + // 3. Restore BfTree from snapshot + RangeIndexManager.ReadIndex(stub, out _, out var cacheSize, ...); + var newTreePtr = rangeIndexManager.RestoreFromSnapshot( + localSnapshotPath, cacheSize, minRecordSize, maxRecordSize, + maxKeyLen, leafPageSize); + + // 4. Build new stub with updated TreePtr + ProcessInstanceId + var newStubBytes = new byte[RangeIndexManager.IndexSizeBytes]; + stub.CopyTo(newStubBytes); + rangeIndexManager.UpdateStubPointer(newStubBytes, newTreePtr); + + // 5. Insert into local main store with RangeIndex RecordType + InsertRangeIndexKey(keyBytes, newStubBytes); + break; +``` + +--- + +### F. Summary: FFI Functions for Persistence + +The core FFI functions for disk-backed tree persistence are implemented in +`libs/native/bftree-garnet/src/lib.rs`: + +- **`bftree_snapshot(tree)`** — Snapshots a disk-backed tree in place (drains circular + buffer, writes index to data file). Returns 0 on success. +- **`bftree_new_from_snapshot(file_path, ...config)`** — Recovers a disk-backed tree + from its data file. Returns tree pointer or null. +- **`bftree_create(..., storage_backend, file_path, ...)`** — Creates a new tree with + the specified backend (0=Disk, 1=Memory). +- **`bftree_drop(tree)`** — Frees a tree instance. + +**Tsavorite `IRecordTrigger` callbacks (implemented):** + +| Callback | Gating Property | When Called | RangeIndex Action | +|---|---|---|---| +| `OnFlushRecord(ref LogRecord)` | `CallOnFlush` | Per valid record on original in-memory page in `OnPagesMarkedReadOnlyWorker`, before flush | Snapshot BfTree to flush file, set Flushed flag | +| `OnDiskReadRecord(ref LogRecord)` | `CallOnDiskRead` | Per record loaded from disk (recovery, pending reads, push scans) | Zero TreeHandle (invalidate stale pointer) | +| `DisposeRecord(ref LogRecord, reason)` | `DisposeOnPageEviction` | Per record on page eviction, delete | Free native BfTree via `DisposeTreeIfOwned` | + +**Memory-only trees:** bf-tree's `snapshot_memory_to_disk` panics for `cache_only` +trees (scan not supported). Future fix: use `StorageBackend::Memory` with +`cache_only=false` instead, which supports scan and snapshot. + +**Future:** For key migration, additional buffer-based serialization functions may be +needed to avoid temp files: + +```rust +/// Serialize BfTree to a byte buffer (for migration without temp files). +#[no_mangle] +pub extern "C" fn bftree_serialize_to_buffer( + tree: *mut BfTree, + buffer: *mut u8, buffer_len: i32, +) -> i64; // bytes written, or -1 if buffer too small + +/// Get the serialized size of a BfTree snapshot (for pre-allocating buffer). +#[no_mangle] +pub extern "C" fn bftree_serialized_size(tree: *mut BfTree) -> i64; + +/// Restore from a byte buffer (received from migration). +#[no_mangle] +pub extern "C" fn bftree_deserialize_from_buffer( + buffer: *const u8, buffer_len: i32, + cb_size_byte: u64, cb_min_record_size: u32, cb_max_record_size: u32, + cb_max_key_len: u32, leaf_page_size: u32, +) -> *mut BfTree; // null on failure +``` + +These are not yet implemented and will be added when migration support is built. + +--- + +### G. Updated File Inventory (Persistence-Related Additions) + +| # | File Path | Purpose | +|---|---|---| +| NEW | `libs/server/Resp/RangeIndex/RangeIndexManager.Persistence.cs` | `DisposeRecord` handler, `OnSnapshotRecord` handler, snapshot path derivation | +| NEW | `libs/server/Resp/RangeIndex/RangeIndexManager.Migration.cs` | `HandleMigratedRangeIndexKey()`, migration serialization/deserialization | +| MOD | `libs/server/Databases/DatabaseManagerBase.cs` | *(no RangeIndex-specific changes needed — checkpoint handled via per-record callback)* | +| MOD | `libs/server/Databases/SingleDatabaseManager.cs` | *(no RangeIndex-specific changes needed — recovery is lazy)* | +| MOD | `libs/cluster/Server/Replication/CheckpointFileType.cs` | Add `RANGEINDEX_SNAPSHOT` enum value | +| MOD | `libs/cluster/Server/Replication/PrimaryOps/ReplicaSyncSession.cs` | Send BfTree snapshot files during replica sync | +| MOD | `libs/cluster/Session/RespClusterMigrateCommands.cs` | Handle `RISTORE` type during key migration | +| MOD | `libs/cluster/Server/Migration/MigrateSessionKeys.cs` | Detect RangeIndex `RecordType`, serialize BfTree, 2-phase transmit | +| MOD | `libs/cluster/Server/Migration/MigrateScanFunctions.cs` | Check `RecordType` during slot scan | +| MOD | `libs/native/bftree-garnet/src/lib.rs` | *(future)* Add `bftree_serialize_to_buffer`, `bftree_deserialize_from_buffer` for migration | + +--- + +### H. Persistence Testing Plan + +1. **Page flush round-trip** — Insert data, force page eviction → BfTree freed; read cold key back → BfTree restored from snapshot, data intact +2. **Checkpoint + recovery** — Insert data, checkpoint, restart process, verify all RI.GET/RI.SCAN return correct results +3. **Multiple checkpoints** — Take multiple checkpoints, verify only latest snapshot files are used +4. **Replica sync** — Primary creates RangeIndex, inserts data, replica connects and receives checkpoint, verify RI.GET on replica returns correct data +5. **Key migration** — Create RangeIndex in slot N, migrate slot N to another node, verify RI.GET on target returns correct data, verify source no longer has the key +6. **Migration + concurrent writes** — Migrate a slot while writes are happening, verify no data loss +7. **Recovery with missing snapshot** — Delete snapshot file, attempt recovery, verify graceful error handling +8. **Large BfTree migration** — Create RangeIndex with 100K entries, migrate, verify correctness and timing + +--- + +## Testing Plan + +### Unit Tests + +1. **Stub round-trip** — `CreateIndex` → `ReadIndex` → verify all fields +2. **RecreateIndex** — verify `TreePtr` and `ProcessInstanceId` update +3. **BfTreeService** — insert/read/delete/scan via P/Invoke against live BfTree +4. **Result codes** — verify `NotFound`, `Deleted`, `InvalidKey` mapping + +### Integration Tests (RESP client → Garnet server) + +1. **RI.CREATE + RI.SET + RI.GET** — basic CRUD +2. **RI.SET auto-create** — `RI.SET` on non-existent key creates the index +3. **RI.MSET + RI.MGET** — batch operations +4. **RI.SCAN** — scan with count, verify ordering and count limit +5. **RI.RANGE** — scan with end key, verify inclusive bounds +6. **RI.SCAN FIELDS KEY/VALUE/BOTH** — verify ScanReturnField behavior +7. **RI.DEL + RI.GET** — delete then read returns nil +8. **DEL/UNLINK** — delete RangeIndex key, verify BfTree freed +9. **RI.EXISTS** — returns 1/0 correctly +10. **RI.CONFIG / RI.METRICS** — return valid data +11. **WRONGTYPE** — `GET` on RI key returns WRONGTYPE; `RI.SET` on string key returns WRONGTYPE +12. **Large scan** — insert 10K entries, scan all, verify ordering + +### Checkpoint/Recovery Tests + +1. Create index → insert data → checkpoint → restart → RI.GET verifies data +2. Multiple indexes → checkpoint → restart → all indexes restored +3. RI.SCAN after recovery returns same results as before checkpoint + +### Concurrency Tests + +1. Multiple threads doing RI.SET/RI.GET simultaneously (BfTree is thread-safe) +2. RI.SCAN concurrent with RI.SET (scan consistency) +3. DEL during concurrent RI.SET (proper locking) diff --git a/website/docs/dev/tsavorite/epochprotection.md b/website/docs/dev/tsavorite/epochprotection.md index 8db3babe0a5..4e81519e29d 100644 --- a/website/docs/dev/tsavorite/epochprotection.md +++ b/website/docs/dev/tsavorite/epochprotection.md @@ -1,5 +1,5 @@ --- -id: epoch +id: epochprotection sidebar_label: EpochProtection title: Epoch Protection Framework @@ -8,7 +8,7 @@ title: Epoch Protection Framework ## Context -We need to ensure shared variables are not being read and mutated simultaneously without determinisitic orderings. Commonly used concurreny primitives such as Mutexes and sempahores provided by the language require threads to synchronize frequently with each other. This Synchronization across threads is expensive; Epoch protection **reduces the frequency of synchronization across threads**. +We need to ensure shared variables are not being read and mutated simultaneously without deterministic orderings. Commonly used concurreny primitives such as Mutexes and semaphores provided by the language require threads to synchronize frequently with each other. This Synchronization across threads is expensive; Epoch protection **reduces the frequency of synchronization across threads**. ## Epoch Protection (10,000-foot view) diff --git a/website/docs/dev/tsavorite/locking.md b/website/docs/dev/tsavorite/locking.md index fc7af97e04d..7b6e0e501de 100644 --- a/website/docs/dev/tsavorite/locking.md +++ b/website/docs/dev/tsavorite/locking.md @@ -6,19 +6,17 @@ title: Locking # Locking -There are three modes of locking in Tsavorite, set by a `ConcurrencyControlMode` value on the Tsavorite constructor: - - `LockTable`: Tsavorite's hash index buckets are used to hold the lock state. Locktable locking is either manual or transient: - - **Manual**: Garnet calls a `Lock` method on `LockableContext` or `LockableUnsafeContext` (hereafter referred to collectively as `Lockable*Context`) at the beginning of a transaction, passing an ordered array of keys, and must call `Unlock` when the transaction is complete. Tsavorite does not try to lock during individual operations on these session contexts. - - **Transient**: Tsavorite acquires and releases locks for individual keys for the duration of a data operation: Upsert, RMW, Read, or Delete. Collectively, these are referred to here as `InternalXxx` for the internal methods that implement them. - - `None`: No locking is done by Tsavorite. +Locking is always on in Tsavorite. It is done by locking the HashIndex bucket. There are two modes of locking; these are automatic based on what sessions the caller uses: + - **Manual**: In this mode, the Garnet processing layer calls a `Lock` method on `TransactionalContext` or `TransactionalUnsafeContext` (hereafter referred to collectively as `Transactional*Context`) at the beginning of a transaction, passing an ordered array of keys, and must call `Unlock` when the transaction is complete. Tsavorite does not try to lock during individual operations on these session contexts. + - **Transient**: Tsavorite acquires and releases locks for individual keys for the duration of a data operation: Upsert, RMW, Read, or Delete. Collectively, these are referred to here as `InternalRUMD` for the internal methods that implement them: Read, Upsert, rMw, and Delete. All locks are obtained via spinning on `Interlocked.CompareExchange` and `Thread.Yield()` and have limited spin count, to avoid deadlocks; if they fail to acquire the desired lock in this time, the operation retries. -As noted above, manual locking is done by obtaining the `Lockable*Context` instance from a `ClientSession`. There are currently 4 `*Context` implementations; all are `struct` for inlining. All `*Context` are obtained as properties on the `ClientSession` named for the type (e.g. `clientSession.LockableContext`). The characteristics of each `*Context` are: +As noted above, manual locking is done by obtaining the `Transactional*Context` instance from a `ClientSession`. There are currently 4 `*Context` implementations; all are `struct` for inlining. All `*Context` are obtained as properties on the `ClientSession` named for the type (e.g. `clientSession.TransactionalContext`). The characteristics of each `*Context` are: - **`BasicContext`**: This is exactly the same as `ClientSession`, internally calling directly through to `ClientSession`'s methods and reusing `ClientSession`'s `TsavoriteSession`. It provides safe epoch management (acquiring and releasing the epoch on each call) and Transient locking. - **`UnsafeContext : IUnsafeContext`**: This provides Transient locking, but rather than safe epoch management handled per-operation by Tsavorite, this supports "unsafe" manual epoch management controlled by the client via `BeginUnsafe()` and `EndUnsafe()`; it is the client's responsibility to make these calls correctly. `UnsafeContext` API methods call the internal ContextRead etc. methods without doing the Resume and Suspend (within try/finally) of epoch protection as is done by the "Safe" API methods. -- **`LockableContext : ILockableContext`**: This provides safe epoch management, but rather than Transient locking, this requires Manual locks via `BeginLockable` and `EndLockable`. This requirement ensures that all locks are acquired before any methods accessing those keys are called. -- **`LockableUnsafeContext : ILockableContext, IUnsafeContext`**: This combines manual epoch management and manual locking, exposing both sets of methods. +- **`TransactionalContext : ITransactionalContext`**: This provides safe epoch management, but rather than Transient locking, this requires Manual locks via `BeginTransactional` and `EndTransactional`. This requirement ensures that all locks are acquired before any methods accessing those keys are called. +- **`TransactionalUnsafeContext : ITransactionalContext, IUnsafeContext`**: This combines manual epoch management and manual locking, exposing both sets of methods. In addition to the `Lock` methods, Tsavorite supports: - `TryLock`: Accepts an array of keys and returns true if all locks were acquired, else false (and any locks that were acquired are released) @@ -29,28 +27,28 @@ In addition to the `Lock` methods, Tsavorite supports: All manual locking of keys must lock the keys in a deterministic order, and unlock in the reverse order, to avoid deadlocks. Lock spinning is limited in order to avoid deadlocks such as the following: - - `Lockable*Context` LC1 exclusively locks k1 + - `Transactional*Context` LC1 exclusively locks k1 - `BasicContext` BC1 tries to acquire an exclusive Transient lock on k1, and spins while holding the epoch - LC1 does an RMW on k1 resulting in a CopyUpdate; this does a BlockAllocate that finds it must flush pages from the head of the log in order to make room at the tail. - LC1 therefore calls BumpCurrentEpoch(... OnPagesClosed) - Because BC1 holds the epoch, the OnPagesClosed() call is never drained, so we have deadlock By ensuring that locks are limited in spins, we force one or both of the above sessions to release any locks it has already aquired and return up the callstack to retry the operation via RETRY_LATER (which refreshes the epoch, allowing other operations such as the OnPagesClosed() mentioned above to complete). -Transient locks are never held across pending I/O or other Wait operations. All the data operations' low-level implementors (`InternalRead`, `InternalUpsert`, `InternalRMW`, and `InternalDelete`--collectively known as `InternalXxx`) release these locks when the call is exited; if the operations must be retried, the locks are reacquired as part of the normal operation there. +Transient locks are never held across pending I/O or other Wait operations. All the data operations' low-level implementors (`InternalRead`, `InternalUpsert`, `InternalRMW`, and `InternalDelete`--collectively known as `InternalRUMD`) release these locks when the call is exited; if the operations must be retried, the locks are reacquired as part of the normal operation there. ## Example -Here is an example of the above two use cases, condensed from the unit tests in `LockableUnsafeContextTests.cs`: +Here is an example of the above two use cases, condensed from the unit tests in `TransactionalUnsafeContextTests.cs`: ```cs - var luContext = session.GetLockableUnsafeContext(); + var luContext = session.GetTransactionalUnsafeContext(); luContext.BeginUnsafe(); - luContext.BeginLockable(); + luContext.BeginTransaction(); var keys = new[] { - new FixedLengthLockableKeyStruct(readKey24, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(readKey51, LockType.Shared, luContext), // Source, shared - new FixedLengthLockableKeyStruct(resultKey, LockType.Exclusive, luContext), // Destination, exclusive + new FixedLengthTransactionalKeyStruct(readKey24, LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(readKey51, LockType.Shared, luContext), // Source, shared + new FixedLengthTransactionalKeyStruct(resultKey, LockType.Exclusive, luContext), // Destination, exclusive }; // Sort the keys to guard against deadlock @@ -64,7 +62,7 @@ Here is an example of the above two use cases, condensed from the unit tests in luContext.Unlock(keys); - luContext.EndLockable(); + luContext.EndTransaction(); luContext.EndUnsafe(); ``` @@ -73,7 +71,7 @@ Here is an example of the above two use cases, condensed from the unit tests in This section covers the internal design and implementation of Tsavorite's locking. ### Operation Data Structures -There are a number of variables necessary to track the main hash table entry information, the 'source' record as defined above, and other stack-based data relevant to the operation. These variables are placed within structs that live on the stack at the `InternalXxx` level. +There are a number of variables necessary to track the main hash table entry information, the 'source' record as defined above, and other stack-based data relevant to the operation. These variables are placed within structs that live on the stack at the `InternalRUMD` level. #### HashEntryInfo This is used for hash-chain traversal and CAS updates. It consists primarily of: @@ -134,9 +132,7 @@ Some relevant `RecordInfo` bits: ### Locking Flow -When `Internalxxx` when `ConcurrencyControlMode` is `LockTable`: - -We obtain the key hash at the start of the operation, so we lock its bucket if we are not in a `Lockable*Context` (if we are, we later Assert that the key is already locked). +We obtain the key hash at the start of the operation, so we lock its bucket if we are not in a `Transactional*Context` (if we are, we later Assert that the key is already locked). Following this, the requested operation is performed within a try/finally block whose 'finally' releases the lock. @@ -161,4 +157,4 @@ Using the above example and assuming an update of r8000, the resulting chain wou - `HashTable` -> r8000 (invalid) -> r7000 -> mxxxx (new) -> m4000 -> m3000 -> m... - In this example, note that the record address spaces are totally different between the main log and readcache; "xxxx" is used as the "new address" to symbolize this. -This splicing operation requires that we deal with updates at the tail of the tag chain (in the `HashEntryInfo`) as well as at the splice point. This cannot be done as a single atomic operation. To handle this, we detach the readcache prefix chain, insert the new record at the tail, and then reattach the detached records. See `DetachAndReattachReadCacheChain` for specifics. We may fail the reattach, but this is acceptable (versus more complicated and expensive locking) because such failures should be rare and the readcache is just a performance optimization. +This splicing operation requires that we deal with updates at the tail of the tag chain (in the `HashEntryInfo`) as well as at the splice point. This cannot be done as a single atomic operation. To handle this, we check if another session added a readcache entry from a pending read while we were inserting a record at the tail of the log. If so, the new readcache record must be invalidated (see `ReadCacheCheckTailAfterSplice`). diff --git a/website/docs/dev/tsavorite/logrecord.md b/website/docs/dev/tsavorite/logrecord.md new file mode 100644 index 00000000000..966a2bddf37 --- /dev/null +++ b/website/docs/dev/tsavorite/logrecord.md @@ -0,0 +1,183 @@ +--- +id: logrecord +sidebar_label: LogRecord +title: LogRecord +--- + +# `LogRecord` + +The `LogRecord` struct is a major revision in the Tsavorite `ISessionFunctions` design. It replaces individual `ref key` and `ref value` parameters in the `ISessionFunctions` methods (as well as endoding optional `ETag` and `Expiration` into the Value) with a single `LogRecord`, which may be either `LogRecord` for in-memory log records, or `DiskLogRecord` for on-disk records. These `LogRecord` have properties for `Key` and `Value` as well as making `Etag` and `Expiration` first-class properties. There are a number of additional changes in this design as well, as shown in the following sections. + +Much of the record-related logic of the allocators (e.g. `SpanByteAllocator`) has been moved into the `LogRecord` structs. + +See [RecordDataHeader](#recorddataheader) for details of the layout, including `RecordType`, `Namespace`, and the ObjectLogPosition ulong if the record is not inline (has an Overflow Key and/or an Overflow or Object value). + +## `SpanByte` and `ArgSlice` are now `PinnedSpanByte` or `ReadOnlySpan` + +To clarify that the element must be a pointer to a pinned span of bytes, the `SpanByte` and `ArgSlice` types have been replaced with `PinnedSpanByte` and `ReadOnlySpan`. The `PinnedSpanByte` is similar to the earlier `SpanByte`; a struct that wraps a pointer to a pinned span of bytes. Its construction has been changed from direct constructor calls to static `FromPinned*` calls, e.g. `FromLengthPrefixedPinnedPointer`. This is mostly used for cases where `(ReadOnly)Span` are not possible due to restrictions on their use; further work could reduce these areas. + +There are still areas where direct `byte*` are used, such as number parsing. Later work can revisit this to use `(ReadOnly)Span` instead if there is no performance impact. + +The `SpanByte` class now exists only as a static utility class that provides extension functions `(ReadOnly)Span`. + +## All Keys are now `ReadOnlySpan` at the Tsavorite Level + +Originally, Tsavorite was templated on the `TKey` generic type, which was either `SpanByte` for the string store or `byte[]` for the object store. In this revision, all keys at the Tsavorite level are now `ReadOnlySpan`. At the Garnet processing level, they may be `PinnedSpanByte` at the `GarnetApi` layer and above. Any key structure must be converted to a stream of bytes and a `ReadOnlySpan` or `PinnedSpanByte` created over this. This can be a stack variable (which is not subject to GC) or a pinned pointer. + +This has simplified the signature and internal implementation of TsavoriteKV itself, the sessions, allocators, ISessionFunctions, Compaction, Iterators, and so on. And we now have only two allocators, `SpanByteAllocator` and the new `ObjectAllocator`. + +## Removal of `BlittableAllocator` + +As part of the migration to `SpanByte`-only keys, `BlittableAllocator` has been removed. Tsavorite Unit Tests such as `BasicTests` and the YCSB benchmark's fixed-length test illustrate simple ways to use stack-based 'long' keys and values with `SpanByte`. This does incur some log record space overhead for the key's or value's length bytes, described below under `LogRecord`. + +A reduced form of `BlittableAllocator`, renamed `TsavoriteLogAllocator`, is still used by `TsavoriteLog`. + +## Replace `GenericAllocator` with `ObjectAllocator` + +With the move to `SpanByte`-only keys we also created a new `ObjectAllocator` for a store that uses an object value type. `GenericAllocator` is not able to take SpanByte keys, and stored both key and value in a separate managed array; `ObjectAllocator` uses native allocations, the same as `SpanByteAllocator`. + +### IHeapObject + +An object field's object must inherit from `IHeapObject`. The Garnet processing layer uses `IGarnetObject`, which inherits from `IHeapObject`). The Tsavorite Unit Tests use object types that implement `IHeapObject`. + +`IHeapObject` provides methods for object management by core Tsavorite and Garnet processing. One significant property is `MemorySize`, the size the object takes in memory. This includes .NET object overhead as well as the size of the actual data. It is used in object size tracking. + +There are a number of other methods on IHeapObject, mostly to handle serialization. + +### `ObjectIdMap` +In `ObjectAllocator` we have an `ObjectIdMap` that provides a GC root for objects (and overflows, as discussed next). In the log record itself, there is a 4-byte `ObjectId` that is an index into the `ObjectIdMap`. + +The `ObjectIdMap` manages the lifetime of .NET objects for the `ObjectAllocator`. Because we cannot store objects in the unmanaged log (or IntPtrs, as they will become obsolete), we store a 4-byte `ObjectId` in the log record; this is an index into the `ObjectIdMap`. These IDs are simply integer indices into the `MultiLevelPageArray` `objectArray` of the `ObjectIdMap`, so it is not truly a "map". + +The `ObjectIdMap` has a freeList in a `SimpleConcurrentStack` called `freeSlots`. When an object is deallocated, its slot is nulled and the slot is added to the free list. When an object is allocated, it looks for a free slot in the free list, and if it finds one, it returns that slot. If it does not find one, it allocates a new slot and returns that. This freelist keeps the `ObjectIdMap` from having to grow more than necessary. + +#### MultiLevelPageArray + +The `MultiLevelPageArray` is a data structure to provide efficiently growable arrays with direct indexing. It is a managed structure, as it is used by the `ObjectIdMap` to root .NET objects. It is also used to provide a structure for simple stacks such as for free lists. + +This `MultiLevelPageArray` is a 3-d array of page vectors. Because `NativePageAllocator` allocates pages for caller use, this can be envisioned as a book, where the first two dimensions are infrastructure, and the third is where the user-visible allocations are created. + - The first dimension is the "book", which is a collection of "chapters". Think of the book as a spine, which can be reallocated--but when it is reallocated, the individual chapters, and references within them, are not moved, so may be accessed by other threads. + - The second dimension is the "chapters", which are collections of pages. + - The third dimension is the actual pages of data which are returned to the user. "Page" is somewhat of a misnomer, as the purpose has changed slightly from its initial intent; currently these are object slots for `IHeapObject` and `byte[]`, as well as integer indexes for freelists. + +This structure is chosen so that only the "book" is grown; individual chapters are allocated as a fixed size. This means that getting and clearing items in the chapter does not have to take a latch to prevent a lost update as the array is grown, as would be necessary if there was only a single level of infrastructure (i.e. a growable single vector). + +The `MultiLevelPageArray` is a managed structure, because it is also used to hold the .NET objects for the `ObjectAllocator` to manage their lifetimes. + +In the initial implementation `MultiLevelPageArray` has fixed-size book (1024) and chapters (64k), but this can be made configurable. + +#### `SimpleConcurrentStack` + +The `SimpleConcurrentStack` sits on top of the `MultiLevelPageArray` and provides a simple stack interface for the free lists. + +## Overflow Keys and Values + +To keep the size of the main log record tractable, we provide an option for `ObjectAllocator` to have large `Span` keys and values "overflow"; rather than being stored inline in the log record, they are allocated separately as `byte[]`, and an integer `ObjectId` is stored in the log record (with the key or value having no explicit length, and the data being the `ObjectId` of size `sizeof(int)`). This redirection does incur some performance overhead. The initial reason for this was to keep `ObjectAllocator` pages small enough that the page-level object size tracking would be sufficiently granular; if those pages are large enough to support large keys, then it is possible there are a large number of records with small keys and large objects, making it impossible to control object space budgets with sufficient granularity. By providing this for `Span` values as well, it allows similar control of the number of records per page. + +## `ISourceLogRecord` + +In this revision of Tsavorite, the individual "ref key" and "ref value" (as well as "ref recordInfo") parameters to `ISessionFunctions` methods have been replaced by a single `LogRecord` parameter. Not only does this consolidate those record attributes, it also encapsulate the "optional" record attributes of `ETag` and `Expiration`, as well as managing the `FillerLength` that allows records to shrink and re-expand in place. Previously the `ISessionFunctions` implementation had to manage the "extra" length; that is now automatically handled by the `LogRecord`. Similarly, `ETag` and `Expiration` previously were encoded into the Value `SpanByte` or a field of the object and this required tracking additional metadata and shifting when these values were added/removed; these too are now managed by the `LogRecord` as first-class properties. + +As part of this change, keys are now always `ReadOnlySpan` at the Tsavorite level. At the processing layer, they are initially `PinnedSpanBytes`; these have a `ReadOnlySpan` property that is called to convert them to `ReadOnlySpan` at the GarnetApi/StorageApi boundary. + +Although we have two allocators, there is only one `LogRecord` family; we do not have separate `StringLogRecord` and `ObjectLogRecord`. There are a couple reasons for this: + - It would be more complex to maintain them, especially as we have multiple implementations of `ISourceLogRecord`. + - Iterators would no longer be able to iterate both stores. + - The `ObjectAllocator` can have `SpanByte`, overflow `byte[]`, or `IHeapObject` values, so the `LogRecord` must be able to handle both. +This decision may be revisited in the future; for example, `SpanByteAllocator` currently cannot have overflow keys or values, so a much leaner implementation could be used for that case. This would require a `TLogRecord` generic type in place of the earlier `TKey` and `TValue` types that have been removed in this revision. + +`ISourceLogRecord` defines the common operations among a number of `LogRecord` implementations. These common operations are summarized here, and the implementations are described below. + - Obtaining the RecordInfo header. There is both a "ref" (mutable) and non-"ref" (readonly) form. + - Obtaining the `ReadOnlySpan` of the Key. + - Obtaining the ValueSpan `Span` (for both `SpanByteAllocator` and `ObjectAllocator`; this may be either inline or overflow). + - Obtaining the ValueObject (for `ObjectAllocator` only. This is intended to be an `IHeapObject`. + - Obtaining the "optionals": ETag and Expiration. Note that while `FillerLength` is also optional in the record (it may or may not be present), it is now completely handled by the `LogRecord` and thus is unknown to the caller. + - Setting a new Value Span or Object. + - Setting the Value field's length. + - This is done automatically when setting the Value Span or Object. + - It may also be called directly and then the ValueSpan obtained and operated on directly, rather than creating a separate `Span` and copying. + - Utilities such as clearing the Value Object, obtaining record size info, and so on. + +For operations that take an input log record, such as `ISessionFunctions.CopyUpdater`, the input log record is of type `TSourceLogRecord`, which may be either `LogRecord` or `DiskLogRecord`. No code outside Tsavorite should need to know the actual type. Within Tsavorite, it is sometimes useful to call `AsLogRecord` or `AsDiskLogRecord` and operate accordingly. + +### `LogRecord` struct + +This is the primary implementation which wraps a log record. It carries the log record's physical address and, if this is an `ObjectAllocator` record, an `ObjectIdMap` for that log record's log page. See `LogRecord.cs` for more details, including the record layout and comments. + +For `ObjectAllocator` records, `TrySetContentLengthsAndPrepareOptionals` also manages conversion between the three Value "styles". Both Keys and Values may be inline or overflow, and values additionally may be object. Keys are not mutable, so there is no `LogRecord` method to change them. Values, however, may move between any of the three: + - Initially, a Value in the `ObjectAllocator` may be a small inline value, such as the first couple strings of a list. This is stored as a byte stream "inline" in the record. + - Depending on the inline size limit, such a value may overflow, and become a pointer to an `OverflowAllocator` allocation. In this case, `TrySetContentLengthsAndPrepareOptionals` will handle converting the inline field value to an overflow pointer, shrinking the record, moving the optionals, and adjusting the `FillerLength` as needed. The value length becomes `ObjectIdMap.ObjectIdSize`. + - Finally, the value may be "promoted" to an actual object; e.g., allocating a `ListObject` and populating it from the `ValueSpan`. Again, `TrySetContentLengthsAndPrepareOptionals` will handle this conversion, resizing the Value, moving the optionals, and adjusting the `FillerLength` as needed. The value becomes a 4-byte int containing the `ObjectId` for the `ObjectIdMap` and its length becomes `ObjectIdMap.ObjectIdSize`. + + `TrySetContentLengthsAndPrepareOptionals` handles this switching between inline, overflow, and object values automatically, based upon settings in the `RecordSizeInfo` that is also passed to `ISessionFunctions` methods and then to the `LogRecord`. When `LogRecord` converts between these styles, it handles all the necessary freeing and allocations. For example, when growing a Value causes allows it to move from inline to overflow, the `byte[]` slot is allocated in `ObjectIdMap`; if it shrinks enough to return to inline, the `ObjectIdMap` slot element is nulled and the slot is added to the freelist, and the record is resized to inline. + + Although `TrySetContentLengthsAndPrepareOptionals` allocates the `ObjectId` slot, it does not know the actual object, so the `ISessionFunctions` implementation must create the object and call `TrySetValueObject`. + + Performance note: `TrySetContentLengthsAndPrepareOptionals` handles all conversions. It should be beneficial to provide some leaner versions, for example string-only when lengths are unlikely to change. + +#### RecordDataHeader + +This is a struct wrapper to manage the variable-length record lengths. At a high level, it manages an indicator byte for lengths, as well as fixed information such as Namespace and RecordType. +For details of the layout, see RecordDataHeader.cs; in summary, the bytes of this header are laid out in sequence as: +- Indicator byte: flags, number of bytes in record length (immutable), number of bytes in key length (also immutable) +- Namespace byte (immutable): indicates the namespace value (if 127 or less) or the number of bytes in the ExtendedNamespace field +- RecordType byte (immutable): indicates the type of the record +- RecordLength (variable # of bytes; immutable): The allocated size of the record +- KeyLength: (variable # of bytes; immutable until revivification): The size of the key +- ExtendedNamespace (variable # of bytes; immutable until revivification): The bytes of the namespace if it is longer than 127 bytes +- Key bytes (immutable until revivification) +- Value bytes, or Overflow or Object id for the `ObjectIdMap` +- Optionals, if present, in this order: + - ETag + - Expiration + - ObjectLogPosition (pseudo-optional; always present if the record Key or Value is Overflow, or the Value is an Object; used only for serialization) +- Filler (the amount of extra space in the record, e.g. if the initial length was less than the "rounded to record alignment" or if the value shrank) + +Note that we do not store Value length directly. Because we must ensure that the Record length is available for Scan even when the record is actively being edited, +we store the complete record length; for space efficiency, we then calculate the Value length from the immutable other fields rather than storing it explicitly. + +#### RecordSizeInfo + +This structure is populated prior to record allocation (it is necessary to know what size to allocate from the log), and then is passed through to `ISessionFunctions` implementation and subsequently to the `LogRecord`. The flow is: +- The `RecordSizeInfo` is populated prior to record allocation: + - This is done by calling the appropriate `IVariableLengthInput` method to populate the `RecordFieldInfo` part of the structure with Key and Value sizes, whether the Value is to be an object, and whether there are optionals present: + - `GetRMWModifiedFieldInfo`: For in-place or copy updates via RMW + - `GetRMWInitialFieldInfo`: For initial update via RMW + - `GetUpsertFieldInfo`: For writing via Upsert. There are three overloads of this, because Upsert takes a Value which may be one of `ReadOnlySpan`, `IHeapObject`, or a source `TSourceLogRecord`. + - The allocator's `PopulateRecordSizeInfo` method is called to fill in the `RecordSizeInfo` fields based upon the `RecordFieldInfo` fields and other information such as maximum inline size and so on: + - Whether the Key or Value are inline or overflow + - Utility methods to make it easy for the Tsavorite allocators to calculate the allocation size + - Other utility methods to allow `LogRecord.TrySetContentLengthsAndPrepareOptionals` to operate efficiently. + +#### LogField + +This is a static class that provides utility functions for `LogRecord` to operate on a Key or Value field at a certain address. + +As a terminology note, `LogField` (and `RecordSizeInfo` and `LogRecord`) use the following terms for field layout. In all cases, the field length is stored separately from the data, in the [RecordDataHeader](#recorddataheader): +- Inline: The field is stored inline in the record. +- Overflow: The field is a byte stream that exceeds the limit to remain inline, so is stored in as an `OverflowByteArray` wrapping a `byte[]`. The "Inline size" is `sizeof(ObjectId)`, which is an int. The "Data size" is the length of the byte stream. +- Object: The field is an object implementing `IHeapObject`. As with overflow, the "Inline size" is `sizeof(ObjectId)`, which is an int. The "Data size" is only relevant during serialization; however, the `HeapMemorySize` property of the `IHeapObject` is used in object size tracking. + +### DiskLogRecord struct + +The DiskLogRecord is an `ISourceLogRecord` that is backed by a `LogRecord`. See `DiskLogRecord.cs` for more details. Its main purpose is to act as a container for a `LogRecord` and the `SectorAlignedMemory` buffer and value-object disposer associated with that `LogRecord`. + +### PendingContext + +`PendingContext` implements `ISourceLogRecord` because it carries a information through the IO process and provides the source record for RMW copy updates. + +Previously `PendingContext` had separate `HeapContainers` for keys and values. However, for operations such as conditional insert for Copy-To-Tail or Compaction, we need to carry through the entire log record (including optionals). In the case of records read from disk (e.g. Compaction), it is easiest to pass the `LogRecord` in its entirety, including its `SectorAlignedMemory` buffer, in the `DiskLogRecord`. So now PendingContext will also serialize the Key passed to Upsert or RMW, and the value passed to Upsert, as a `DiskLogRecord`. `PendingContext` still carries the `HeapContainer` for Input, and `CompletedOutputs` must still retain the Key's `HeapContainer`. + +For Compaction or other operations that must carry an in-memory record's data through the pending process, `PendingContext` serializes that in-memory `LogRecord` to its `DiskLogRecord`. + +### ObjectScanIterator + +`ObjectScanIterator` must copy in-memory source records for Pull iterations, so it implements `ISourceLogRecord` by delegating to a `DiskLogRecord` that is instantiated over its copy buffer. + +### TsavoriteKVIterator + +`TsavoriteKVIterator` must copy in-memory source records for Pull iterations, so it implements `ISourceLogRecord` by delegating its internal `ITsavoriteScanIterator`s. + +## Migration and Replication + +Key migration and diskless Replication have been converted to serialize the record to a `DiskLogRecord` on the sending side, and on the receiving side call one of the new `Upsert` overloads that take a `TSourceLogRecord` as the Value. This serialization mimics the writing to disk, but instead of a separate file or memory allocation, it allocates one chunk large enough for the entire inline portion followed by the out-of-line portions appended after the inline portion. Note that this limits the capacity of out-of-line allocations to a single network buffer; there is a pending work item to provide "chunked" output to (and read from) the network buffer. \ No newline at end of file diff --git a/website/docs/dev/tsavorite/object-allocator.md b/website/docs/dev/tsavorite/object-allocator.md new file mode 100644 index 00000000000..84f2a477b02 --- /dev/null +++ b/website/docs/dev/tsavorite/object-allocator.md @@ -0,0 +1,37 @@ +--- +id: object-allocator +sidebar_label: ObjectAllocator +title: ObjectAllocator +--- + +# ObjectAllocator + +The `ObjectAllocator` replaces the `GenericAllocator` to provide two important improvements: +- It supports `ReadOnlySpan` keys. With this change Tsavorite now uses only `ReadOnlySpan` keys; the Garnet processing layer uses `PinnedSpanByte` keys initially, which are converted to `ReadOnlySpan` at the GarnetApi/StorageSession boundary. (The `GenericAllocator` did not support `SpanByte` keys.) +- It replaces the separate managed array of `GenericAllocator` with native allocations for the log pages, exactly like `SpanByteAllocator`. + +Tsavorite provides two primary allocators: +- Strings are stored in a version of `TsavoriteKV` that uses a `SpanByteAllocator`. +- Objects are stored in a version of `TsavoriteKV` that uses an `ObjectAllocator`. This is also the "unified allocator" used by Garnet. + +The former `BlittableAllocator` is now the `TsavoriteLogAllocator`, used only by the `TsavoriteLog`. + +One big difference between the two is that `SpanByteAllocator` allows larger pages for strings, while `ObjectAllocator` allows using a smaller page size because objects use only 4 byte identifiers in the inline log record. Thus, the page size can be much smaller, allowing finer control over Object size tracking and memory-budget enforcement. Either allocator can also set the Key and Value max inline sizes to cause the field to be stored in an overflow allocation, although this is less performant. + +## Two Log Files + +ObjectAllocator supports two log files simlarly to `GenericAllocator`. However, `GenericAllocator` was limited to a fixed 100MB buffer size for object serialization. `ObjectAllocator` uses a circular buffer system, currently 4 buffers of 4MB each, and writes one to disk while populating the next (or reads the following buffers from disk while processing the current one, e.g. deserializing objects. + +The key points in the code here are AsyncWrite, AsyncRead, AsyncGetFromDisk, and iterators. The basic flow is: +- Flush: + - This is AsyncFlushPagesForReadOnly and Checkpointing. AsyncFlushPagesForReadOnly is a loop on partial or complete pages; it creates a CircularDiskWriteBuffer that is reused throughout the entire call to AsyncFlushPagesForReadOnly, for all the partial ranges. The main driver is an ObjectLogWriter, which handles writing for Overflow (large, out-of-line allocations that are byte[]; this writing is optimized to minimize copies) strings and serialization of objects. The basic operation of the circular buffer is to track the current DiskWriteBuffer's current position, last flushed position, and end position (e.g. if it ends in the middle of a segment). +- Read: + - This may be either AsyncGetFromDisk for RUMD operations or Scan, either for iteration or recovery. Similar to Flush, it creates a CircularDiskReadBuffer composed of multiple DiskReadBuffers, and issues disk reads ahead of the operation on the buffer (such as object deserialization). + +### Adaptive Fields in LogRecord +When we serialize a LogRecord to disk, as we write the object log file we also modify the disk-image of the LogRecord (but, importantly, do not modify the length of the logRecord, to keep the LogicalAddress space consistent) to maintain a "pointer" and length in the ObjectLog file: +- We record the position in the Log file in the ObjectLogPosition ulong that is allocated for the record when we specify non-inline. +- We reuse the ObjectId space for the length of that field. For Value objects, we also use the top byte of the `ObjectLogPosition` field, giving a total of 40 bits or 1 TB of address for a single object, and 72 PB for the entire object log. + +### ObjectIdMap Remapping +The ObjectAllocator carries a second ObjectIdMap, a transient one intended for IO and iterator and pending operations. (These are the same scenarios that embed a LogRecord into a DiskLogRecord.) In these cases we remap the ObjectIds in the temporary image to use the TransientObjectIdMap, so pages can be evicted and their ObjectIdMaps reused. \ No newline at end of file diff --git a/website/docs/dev/tsavorite/reviv.md b/website/docs/dev/tsavorite/reviv.md index b8ad03e7ae7..39b4f679e36 100644 --- a/website/docs/dev/tsavorite/reviv.md +++ b/website/docs/dev/tsavorite/reviv.md @@ -129,16 +129,6 @@ FreeList revivification functions as follows: - Clearing the extra value length and filler and calls `DisposeForRevivification` as described in [Maintaining Extra Value Length](#maintaining-extra-value-length). - Unsealing the record; epoch management guarantees nobody is still executing who saw this record before it went into the free record pool. -### Concurrency Control Considerations for FreeList Revivification -FreeList Revivification requires `ConcurrencyControlMode` not be `ConcurrencyControlMode.None`; otherwise the tag chain can be unstable, with the first record in the tag chain potentially removed and reused by revivification while a thread is tracing back from it: - - Thread1: Get the first record address from the `HashBucketEntry` - - Thread2: Delete and elide the first record, putting it on the revivification FreeList - - Thread3: Revivify that record with a different key, setting its .PreviousAddress - - Thread1: Follows the *former* .PreviousAddress and is now on an entirely different tag chain. -This is *only* possible for the first record in the tag chain (the tail-most record, in the `HashBucketEntry`); we do not elide records in the middle of the tag chain. - -For `ConcurrencyControlMode.LockTable`, because the only current LockTable implementation is via the `HashBucket`s and locking at the `HashBucket` level is higher than the tag chain, we get a stable tag chain *almost* for free. The cost is that the `HashBucket` must be locked *before* calling TracebackForKeyMatch. - ### `FreeRecordPool` Design The FreeList hierarchy consists of: - The `FreeRecordPool`, which maintains the bins, deciding which bin should be used for Enqueue and Dequeue. diff --git a/website/docs/dev/tsavorite/storefunctions.md b/website/docs/dev/tsavorite/storefunctions.md index d22f9b9d1c3..88eefea61ad 100644 --- a/website/docs/dev/tsavorite/storefunctions.md +++ b/website/docs/dev/tsavorite/storefunctions.md @@ -8,7 +8,7 @@ title: StoreFunctions and Allocator Wrapper This section discusses both of these because they were part of a change to add two additional type args, `TStoreFunctions` and `TAllocator`, to `TsavoriteKV` as well as the various sessions and `*Context` (e.g. `BasicContext`). The purpose of both of these is to provide better performance by inlining calls. StoreFunctions also provides better logical design for the location of the operations that are store-level rather than session-level, as described below. -From the caller point of view, we have two new type parameters on `TsavoriteKV`. The `TStoreFunctions` and `TAllocator` are also on `*.Context` (e.g. `BasicContext`) as well. C# allows the 'using' alias only as the first lines of a namespace declaration, and the alias is file-local and recognized by subsequent 'using' aliases, so the "Api" aliases such as `BasicGarnetApi` in multiple files are much longer now. +From the caller point of view, we have two new type parameters on `TsavoriteKV`. The `TStoreFunctions` and `TAllocator` are also on `*.Context` (e.g. `BasicContext`) as well. C# allows the 'using' alias only as the first lines of a namespace declaration, and the alias is file-local and recognized by subsequent 'using' aliases, so the "Api" aliases such as `BasicGarnetApi` in multiple files are much longer now. `TsavoriteKV` constructor has been changed to take 3 parameters: - `KVSettings`. This replaces the previous long list of parameters. `LogSettings`, `ReadCacheSettings`, and `CheckpointSettings` have become internal classes, used only by `TsavoriteKV` (created from `TsavoriteKVSettings`) when instantiating the Allocators (e.g. the new `AllocatorSettings` has a `LogSettings` member). `SerializerSettings` has been removed in favor of methods on `IStoreFunctions`. @@ -22,7 +22,7 @@ These are described in more detail below. Because `IStoreFunctions` is intended to provide maximum inlining, Tsavorite does not provide a `StoreFunctionsBase`. Instead, Tsavorite provides a `StoreFunctions` struct implementation, with optional implementations passed in, for: - Key Comparison (previously passed as an `ITsavoriteKeyComparer` interface, which is not inlined) -- Key and Value Serializers. Due to limitations on type arguments, these must be passed as `Func<>` which creates the implementation instance, and because serialization is an expensive operation, we stay with the `IObjectSerializer` and `IObjectSerializer` interfaces rather than clutter the `IStoreFunctions` interface with the Key and Value Serializer type args. +- Serializers for Value objects. Due to limitations on type arguments, these must be passed as `Func<>` which creates the implementation instance, and because serialization is an expensive operation, we stay with the `IObjectSerializer` interfaces rather than clutter the `IStoreFunctions` interface with the Key and Value Serializer type args. - Record disposal (previously on `ISessionFunctions` as multiple methods, and now only a single method with a "reason" parameter). - Checkpoint completion callback (previously on `ISessionFunctions`). @@ -33,8 +33,8 @@ Of course, because `TsavoriteKV` has the `TStoreFunctions` type parameter, this As with `StoreFunctions`, the Allocator Wrapper is intended to provide maximal inlining. As noted above, type parameters implemented by classes do not generate inlined code; the JITted code is general, for a single `IntPtr`-sized reference. For structs, however, the JITter generates code specific to that specific struct type, in part because the size can vary (e.g. when pushed on the stack as a parameter). There is a hack that allows a type parameter implemented by a class to be inlined: the generic type must be for a struct that wraps the class type and makes calls on that class type in a non-generic way. This is the approach the Allocator Wrapper takes: -- The `BlittableAllocator`, `GenericAllocator`, and `SpanByteAllocator` classes are now the wrapper structs, with `Key`, `Value`, and `TStoreFunctions` type args. These implement the `IAllocator` interface. -- There are new `BlittableAllocatorImpl`, `GenericAllocatorImpl`, and `SpanByteAllocatorImpl` classes that implement most of the functionality as previously, including inheriting from `AllocatorBase`. These also have `Key`, `Value`, and `TStoreFunctions` type args; the `TAllocator` is not needed as a type arg because it is known to be the `XxxAllocator` Wrapper struct. The wrapper structs contain an instance of the `XxxAllocatorImpl` class. +- The `SpanByteAllocator` and `ObjectAllocator` classes are now the wrapper structs, with a `TStoreFunctions` type arg. These implement the `IAllocator` interface. +- There are new `SpanByteAllocatorImpl` and `ObjectAllocatorImpl` classes that implement most of the functionality as previously, including inheriting from `AllocatorBase`. These also have a `TStoreFunctions` type arg; the `TAllocator` is not needed as a type arg because it is known to be the `XxxAllocator` Wrapper struct. The wrapper structs contain an instance of the `XxxAllocatorImpl` class. - `AllocatorBase` itself now contains a `_wrapper` field that is a struct-wrapper instance (which contains the instance pointer of the fully-derived allocator class) that is constrained to the `IAllocator` interface. `AllocatorBase` itself is templated on `TStoreFunctions` and `TAllocator`. The new Allocator definition supports two interfaces: diff --git a/website/docs/dev/vector-sets.md b/website/docs/dev/vector-sets.md index 8e0893cd02a..2bfc4dae47c 100644 --- a/website/docs/dev/vector-sets.md +++ b/website/docs/dev/vector-sets.md @@ -6,29 +6,24 @@ title: Vector Sets # Overview -Garnet has partial support for Vector Sets, implemented on top of the [DiskANN project](https://www.nuget.org/packages/diskann-garnet/). - -This data type is very strange when compared to others Garnet supports. - -> [!IMPORTANT] -> The DiskANN link needs to be updated once OSS'd. +Garnet has partial support for Vector Sets, implemented on top of the [DiskANN project](https://github.com/microsoft/DiskANN). # Design -Vector Sets are a combination of one "index" key, which stores metadata and a pointer to the DiskANN data structure, and many "element" keys, which store vectors/quantized vectors/attributes/etc. All Vector Set keys are kept in the main store, but only the index key is visible - this is accomplished by putting all element keys in different namespaces. +Vector Sets are a combination of one "index" key, which stores metadata and a pointer to the DiskANN data structure, and many "element" keys, which store vectors/quantized vectors/attributes/etc. All Vector Set keys are kept in the store as binary (ie. non-object) values, but only the index key is visible - this is accomplished by putting all element keys in different namespaces. ## Global Metadata -In order to track allocated Vector Sets (and their respective hash slots), in progress cleanups, in progress migrations - we keep a single `ContextMetadata` struct under the empty key in namespace 0. +In order to track allocated Vector Sets (and their respective hash slots), in progress cleanups, in progress migrations - we keep a single `ContextMetadata` struct under the empty key in namespace `VectorManager.MetadataNamespace` (which is `1`). This is loaded and cached on startup, and updated (both in memory and in Tsavorite) whenever a Vector Set is created or deleted. Simple locking (on the `VectorManager` instance) is used to serialize these updates as they should be rare. > [!IMPORTANT] > Today `ContextMetadata` can track only 64 Vector Sets in some state of creation or cleanup. > -> The practical limit is actually 31, because context must be < 256, divisible by 8, and not 0 (which is reserved). +> The practical limit is actually 15, because context must be < 128, divisible by 8, and not 0 (which is reserved). > -> This limitation will be lifted eventually, perhaps after Store V2 lands. +> This limitation will be lifted eventually. ## Indexes @@ -46,17 +41,11 @@ The index key (represented by the `Index` struct) contains the following data: > We have an extension here, `XPREQ8` which is not from Redis. > This is a quantizier for data sets which have already been 8-bit quantized or are otherwise naturally small byte vectors, and is extremely optimized for reducing reads during queries. > It forbids the `REDUCE` option and requires 4-byte element ids. - * > [!IMPORTANT] - > Today only `XPREQ` is actually implemented, eventually DiskANN will provide reasonable versions of all the Redis builtin quantizers. - - `Guid ProcessInstanceId` - an identifier which is used distinguish the current process from previous instances, this is used after [recovery](#recovery) or [replication](#replication) to detect if `IndexPtr` is dangling - -The index key is in the main store alongside other binary values like strings, hyperloglogs, and so on. It is distinguished for `WRONGTYPE` purposes with the `VectorSet` bit on `RecordInfo`. -> [!IMPORTANT] -> `RecordInfo.VectorSet` is checked in a few places to correctly produce `WRONGTYPE` responses, but we need more coverage for all commands. Probably something akin to how ACLs required per-command tests. +The index key is in the store alongside other binary values like strings, hyperloglogs, and so on. It is distinguished for `WRONGTYPE` purposes with `RecordType` field on `ISourceLogRecord` logs set to `VectorManager.RecordType` (which is `1`). > [!IMPORTANT] -> A generalization of the `VectorSet`-bit should be used for all data types, this can happen once we have Store V2. +> `RecordType` is checked in a few places to correctly produce `WRONGTYPE` responses, but we need more coverage for all commands. Probably something akin to how ACLs required per-command tests. ## Elements @@ -83,11 +72,12 @@ Implemented commands: - [ ] VCARD - [x] VDIM - [x] VEMB - - [ ] VGETATTR - - [ ] VINFO + - [x] VGETATTR + - [x] VINFO - [ ] VISMEMBER - [ ] VLINKS - [ ] VRANDMEMBER + - [ ] VRANGE - [x] VREM - [ ] VSETATTR - [x] VSIM @@ -139,33 +129,21 @@ Metadata is handled purely on the Garnet side by reading out the [`Index`](#inde > We _may_ return more details of our own implementation. What those are need to be documented, and why, > when we implement `VINFO`. -## Deletion (via `DEL` and `UNLINK`) - -`DEL` (and its equivalent `UNLINK`) is only non-Vector Set command to be routinely expected on a Vector Set key. It is complicated by not knowing we're operating on a Vector Set until we get rather far into deletion. - -We cope with this by _cancelling_ the Tsavorite delete operation once we have a `RecordInfo` with the `VectorSet`-bit set and a value which is not all zeros, detecting that cancellation in `MainStoreOps`, and shunting the delete attempt to `VectorManager`. - -`VectorManager` performs the delete in five steps: - - Acquire exclusive locks covering the Vector Set ([more locking details](#locking)) - - Add the key to an `InProgressDeletes` key (namespace 0, key=0x01) - - If the index was initialized in the current process ([see recovery for more details](#recovery)), call DiskANN's `drop_index` function - - Perform a write to zero out the index key in Tsavorite - - Reattempt the Tsavorite delete - - Cleanup ancillary metadata and schedule element data for cleanup ([more details below](#cleanup)) - - Remove the key from the `InProgressDeletes` key +## Deletion (via `DEL`, `UNLINK`, `FLUSHDB`, `FLUSHALL`) -The `InProgressDeletes` key is necessary to recover from interrupted deletes. At process start, `VectorManager` consults the `InProgressDeletes` key and completes any deletes that got as far as zero-ing out the index key. +Deletion of Vector Sets is detected in the `GarnetTriggers.OnDispose` callback, which calls `VectorManager.RequestDeletion` to begin the process of deletion. -> [!IMPORTANT] Interrupted deletes are expected only during process exits, but if they occur without the process exiting they will leave the Vector Set in a partially deleted state. We detect that and return a new `GarnetStatus.BADSTATE` which returns an explanatory error. -> -> We _could_ resume the delete on `GarnetStatus.BADSTATE`, but like `GarnetStatus.WRONGTYPE` that needs to be done for _all_ commands not just Vector Set commands. This work is likewise left for the future. - -## FlushDB +This takes place in four steps: + 1. The Vector Set context is marked for deletion from `GarnetTriggers.OnDispose` + * A background task does this, as we do not have a usable storage session in the `GarnetTriggers` callback + * We _block_ on that background task, if an error occurs an exception is raised and the delete aborted + 2. `GarnetTriggers.OnDispose` returns, deleting the index key + 3. A background cleanup task scans the Tsavorite log for element keys, [see Cleanup](#cleanup) for more detail + 4. The Vector Set context is marked available -`FLUSHDB` (and it's relative `FLUSHALL`) require special handling. +During recovery partially deleted Vector Sets are found by checking [`ContextMetadata`](#global-metadata). Those whose index keys no longer are exist are rescheduled for cleanup, and those who still have an index key have their "mark for deletion"-bit cleared. -> [!IMPORTANT] -> This is not currently implemented. +`FLUSHDB` and `FLUSHALL` acquire _all_ exclusive locks on `VectorManager` before beginning a flush, and resets context metadata before releasing those locks. In combination with `GarnetTriggers.OnEvict` dropping DiskANN indexes this cleanly removes all index keys and element data. # Locking @@ -214,7 +192,7 @@ Replicating Vector Sets is tricky because of the unusual "writes are actually re As noted above, inserts (via `VADD`) and deletes (via `VREM`) are reads from Tsavorite's perspective. As a consequence, normal replication (which is triggered via `MainSessionFunctions.WriteLog(Delete|RMW|Upsert)`) does not happen on those operations. -To fix that, synthetic writes against related keys are made after an insert or remove. These writes are against the same Vector Set key, but in namespace 0. See `VectorManager.ReplicateVectorSetAdd` and `VectorManager.ReplicateVectorSetRemove` for details. +To fix that, synthetic writes against related keys are made after an insert or remove. These writes are against the same Vector Set key, without any namespace information. See `VectorManager.ReplicateVectorSetAdd` and `VectorManager.ReplicateVectorSetRemove` for details. > [!IMPORTANT] > There is a failure case here where we crash between the insert operation completing and the replication operation completing. @@ -222,9 +200,6 @@ To fix that, synthetic writes against related keys are made after an insert or r > This appears to simply extend a window that already existed between when a Tsavorite operation completed and an entry was written to the AOF. > This needs to confirmed - if it is not the case, handling this failure needs to be figured out. -> [!IMPORTANT] -> This code assumes a Vector Set under the empty string is illegal. That does not seem to be true with Redis - so we will need to move these keys elsewhere. For now, we just forbid the empty key for VADDs. - > [!NOTE] > These synthetic writes might appear to double write volume, but that is not the case. Actual inserts and deletes have extreme write amplification (that is, each cause DiskANN to perform many writes against the Main Store), whereas the synthetic writes cause a single (no-op) modification to the Main Store plus an AOF entry. @@ -271,7 +246,7 @@ At a high level, migration between the originating primary a destination primary # Cleanup -Deleting a Vector Set only drops the DiskANN index and removes the top-level keys (ie. the visible key and related hidden keys for replication). This leaves all element, attribute, neighbor lists, etc. still in the Main Store. +Deleting a Vector Set only drops the DiskANN index and removes the top-level keys (ie. the index key). This leaves all element, attribute, neighbor lists, etc. still in the Main Store. To clean up the remaining data we record the deleted index context value in `ContextMetadata` and then schedule a full sweep of the Main Store looking for any keys under namespaces related to that context. When we find those keys we delete them, see `VectorManager.RunCleanupTaskAsync()` and `VectorManager.PostDropCleanupFunctions` for details. @@ -281,7 +256,7 @@ To clean up the remaining data we record the deleted index context value in `Con > If we wanted to explore better options, we'd need to build something that can drop whole namespaces at once in Tsavorite. > [!IMPORTANT] -> Today because we only have ~30 available Vector Set contexts, it is quite likely that deleting a Vector Set and then immediately creating a new one will fail if you're near the limit. +> Today because we only have ~15 available Vector Set contexts, it is quite likely that deleting a Vector Set and then immediately creating a new one will fail if you're near the limit. > > This will be fixed once we have arbitrarily long namespaces in Store V2, and have updated `ContextMetadata` to track those. @@ -295,18 +270,12 @@ During startup we read any old `ContextMetadata` out of the Main Store, cache it ## Vector Sets -While reading out [`Index`](#indexes) before performing a DiskANN function call, we check the stored `ProcessInstanceId` against the (randomly generated) one in our `VectorManager` instance. If they do not match, we know that the DiskANN `IndexPtr` is dangling and we need to recreate the index. +While reading out [`Index`](#indexes) before performing a DiskANN function call, we check the stored `IndexPtr`. If it is null, we know that the DiskANN side needs to be recreated. To recreate, we acquire exclusive locks (in the same way we would for `VADD` or `DEL`) and invoke `create_index` again. From DiskANN's perspective, there's no difference between creating a new empty index and recreating an old one which has existing data. This means we recreate indexes lazily after recovery. Consequently the _first_ command (regardless of if it's a `VADD`, a `VSIM`, or whatever) against an index after recovery will be slower since it needs to do extra work, and will block other commands since it needs exclusive locking. -> [!NOTE] -> Today `ProcessInstanceId` is a `GUID`, which means we're paying for a 16-byte comparison on every command. -> -> This comparison is highly predictable, but we could try and remove the comparison (with caching, as mentioned for `Index` above). -> We could also make it cheaper by using a random `ulong` instead, but would need to do some math to convince ourselves collisions aren't possible in realistic scenarios. - # DiskANN Integration Almost all of how Vector Sets actually function is handled by DiskANN. Garnet just embeds it, translates between RESP commands and DiskANN functions, and manages storage. @@ -316,9 +285,9 @@ In order for DiskANN to access and store data in Garnet, we provide a set of cal All callbacks take a `ulong context` parameter which identifies the Vector Set involved (the high 61-bits of the context) and the associated namespace (the low 3-bits of the context). On the Garnet side, the whole `context` is effectively a namespace, but from DiskANN's perspective the top 61-bits are an opaque identifier. > [!IMPORTANT] -> As noted elsewhere, we only have a byte's worth of namespaces today - so although `context` could handle quintillions of Vector Sets, today we're limited to just 31. +> As noted elsewhere, we only have a byte's worth of namespaces today - so although `context` could handle quintillions of Vector Sets, today we're limited to ~15. > -> This restriction will go away with Store V2, but we expect "lower" Vector Sets to out perform "higher" ones due to the need for intermediate data copies with longer namespaces. +> This restriction will go away later, but we expect "lower" Vector Sets to out perform "higher" ones due to the need for intermediate data copies with longer namespaces. ## Read Callback @@ -329,10 +298,10 @@ void ReadCallbackUnmanaged(ulong context, uint numKeys, nint keysData, nuint key `context` identifies which Vector Set is being operated on AND the associated namespace, `numKeys` tells us how many keys have been encoded into `keysData`, `keysData` and `keysLength` define a `Span` of length prefixied keys, `dataCallback` is a `delegate* unmanaged[Cdecl, SuppressGCTransition]` used to push found keys back into DiskANN, and `dataCallbackContext` is passed back unaltered to `dataCallback`. -In the `Span` defined by `keysData` and `keysLength` the keys are length prefixed with a 4-byte little endian `int`. This is necessary to support variable length element ids, but also gives us some scratch space to store a namespace when we convert these to `SpanByte`s. This mangling is done as part of the `IReadArgBatch` implementation we use to read keys from Tsavorite. +In the `Span` defined by `keysData` and `keysLength` the keys are length prefixed with a 4-byte little endian `int`. > [!NOTE] -> Once variable sized namespaces are supported we'll have to handle the case where the namespace can't fit in 4 bytes. However, we expect that to be rare (4-bytes would give us ~53,000,000 Vector Sets) and the performance benefits of _not_ copying during querying are very large. +> Today we place the `context`-derived namespace byte in a field on `VectorElementKey`. In store v1 we kept namespace inline with key bytes (using the length prefixed bytes for storage) - it may be worth restoring that for performance. As we find keys, we invoke `dataCallback(index, dataCallbackContext, keyPointer, keyLength)`. If a key is not found, its index is simply skipped. The benefits of this is that we don't copy data out of the Tsavorite log as part of reads, DiskANN is able to do distance calculations and traversal over in-place data. @@ -355,7 +324,7 @@ byte WriteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength, nint w `context` identifies which Vector Set is being operated on AND the associated namespace, `keyData` and `keyLength` represent a `Span` of the key to write, and `writeData` and `writeLength` represent a `Span` of the value to write. -DiskANN guarantees an extra 4-bytes BEFORE `keyData` that we can safely modify. This is used to avoid copying the key value when we add a namespace to the `SpanByte` before invoking Tsavorite's `Upsert`. +DiskANN guarantees an extra 4-bytes BEFORE `keyData` that we can safely modify. This callback returns 1 if successful, and 0 otherwise. @@ -368,7 +337,7 @@ byte DeleteCallbackUnmanaged(ulong context, nint keyData, nuint keyLength) `context` identifies which Vector Set is being operated on AND the associated namespace, and `keyData` and `keyLength` represent a `Span` of the key to delete. -As with the write callback, DiskANN guarantees an extra 4-bytes BEFORE `keyData` that we use to store a namespace, and thus avoid copying the key value before invoking Tsavorite's `Delete`. +As with the write callback, DiskANN guarantees an extra 4-bytes BEFORE `keyData`. This callback returns 1 if the key was found and removed, and 0 otherwise. @@ -383,7 +352,7 @@ byte ReadModifyWriteCallbackUnmanaged(ulong context, nint keyData, nuint keyLeng `writeLength` is the desired number of bytes, this is only used used if we are creating a new key-value pair. -As with the write and delete callbacks, DiskANN guarantees an extra 4-bytes BEFORE `keyData` that we use to store a namespace, and thus avoid copying the key value before invoking Tsavorite's `RMW`. +As with the write and delete callbacks, DiskANN guarantees an extra 4-bytes BEFORE `keyData`. After we allocate a new key-value pair or find an existing one, `dataCallback(nint dataCallbackContext, nint dataPointer, nuint dataLength)` is called. Changes made to data in this callback are persisted. This needs to be _fast_ to prevent gumming up Tsavorite, as we are under epoch protection. @@ -395,7 +364,7 @@ The callback returns 1 if the key-value pair was found or created, and 0 if some Garnet calls into the following DiskANN functions: - - [x] `nint create_index(ulong context, uint dimensions, uint reduceDims, VectorQuantType quantType, uint buildExplorationFactor, uint numLinks, nint readCallback, nint writeCallback, nint deleteCallback, nint readModifyWriteCallback)` + - [x] `nint create_index(ulong context, uint dimensions, uint reduceDims, VectorQuantType quantType, uint buildExplorationFactor, uint numLinks, VectorDistanceMetricType distanceMetric, nint readCallback, nint writeCallback, nint deleteCallback, nint readModifyWriteCallback)` - [x] `void drop_index(ulong context, nint index)` - [x] `byte insert(ulong context, nint index, nint id_data, nuint id_len, VectorValueType vector_value_type, nint vector_data, nuint vector_len, nint attribute_data, nuint attribute_len)` - [x] `byte remove(ulong context, nint index, nint id_data, nuint id_len)` diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index b49c8f03b8e..1334cbdc064 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -98,7 +98,6 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **CheckpointDir** | ```-c```
    ```--checkpointdir``` | ```string``` | | Storage directory for checkpoints. Uses logdir if unspecified. | | **Recover** | ```-r```
    ```--recover``` | ```bool``` | | Recover from latest checkpoint and log, if present. | | **DisablePubSub** | ```--no-pubsub``` | ```bool``` | | Disable pub/sub feature on server. | -| **EnableIncrementalSnapshots** | ```--incsnap``` | ```bool``` | | Enable incremental snapshots. | | **PubSubPageSize** | ```--pubsub-pagesize``` | ```string``` | Memory size | Page size of log used for pub/sub (rounds down to power of 2) | | **DisableObjects** | ```--no-obj``` | ```bool``` | | Disable support for data structure objects. | | **EnableCluster** | ```--cluster``` | ```bool``` | | Enable cluster. | @@ -115,12 +114,13 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **EnableAOF** | ```--aof``` | ```bool``` | | Enable write ahead logging (append-only file). | | **AofMemorySize** | ```--aof-memory``` | ```string``` | Memory size | Total AOF memory buffer used in bytes (rounds down to power of 2) - spills to disk after this limit | | **AofPageSize** | ```--aof-page-size``` | ```string``` | Memory size | Size of each AOF page in bytes(rounds down to power of 2) | +| **AofSegmentSize** | ```--aof-segment-size``` | ```string``` | Memory size | Size of each AOF segment (file) in bytes on disk (rounds down to power of 2). This is the granularity at which AOF files are created and truncated. | | **CommitFrequencyMs** | ```--aof-commit-freq``` | ```int``` | Integer in range:
    [-1, MaxValue] | Write ahead logging (append-only file) commit issue frequency in milliseconds. 0 = issue an immediate commit per operation, -1 = manually issue commits using COMMITAOF command | | **WaitForCommit** | ```--aof-commit-wait``` | ```bool``` | | Wait for AOF to flush the commit before returning results to client. Warning: will greatly increase operation latency. | | **AofSizeLimit** | ```--aof-size-limit``` | ```string``` | Memory size | Maximum size of AOF (rounds down to power of 2) after which unsafe truncation will be applied. Left empty AOF will grow without bound unless a checkpoint is taken | | **CompactionFrequencySecs** | ```--compaction-freq``` | ```int``` | Integer in range:
    [0, MaxValue] | Background hybrid log compaction frequency in seconds. 0 = disabled (compaction performed before checkpointing instead) | | **ExpiredObjectCollectionFrequencySecs** | ```--expired-object-collection-freq``` | ```int``` | Integer in range:
    [0, MaxValue] | Frequency in seconds for the background task to perform object collection which removes expired members within object from memory. 0 = disabled. Use the HCOLLECT and ZCOLLECT API to collect on-demand. | -| **CompactionType** | ```--compaction-type``` | ```LogCompactionType``` | None, Shift, Scan, Lookup | Hybrid log compaction type. Value options: None - No compaction, Shift - shift begin address without compaction (data loss), Scan - scan old pages and move live records to tail (no data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss) | +| **CompactionType** | ```--compaction-type``` | ```LogCompactionType``` | None, Shift, Lookup, Scan | Hybrid log compaction type. Value options: None - no compaction, Shift - shift begin address without compaction (data loss), Lookup - lookup each record in compaction range, for record liveness checking using hash chain (no data loss; recommended for production use), Scan - scan old pages and move live records to tail (no data loss; NOT RECOMMENDED — builds a temporary parallel KV index proportional to the keyspace, causing significant transient memory use; prefer Lookup) | | **CompactionForceDelete** | ```--compaction-force-delete``` | ```bool``` | | Forcefully delete the inactive segments immediately after the compaction strategy (type) is applied. If false, take a checkpoint to actually delete the older data files from disk. | | **CompactionMaxSegments** | ```--compaction-max-segments``` | ```int``` | Integer in range:
    [0, MaxValue] | Number of log segments created on disk before compaction triggers. | | **ObjectStoreCompactionMaxSegments** | ```--obj-compaction-max-segments``` | ```int``` | Integer in range:
    [0, MaxValue] | Number of object store log segments created on disk before compaction triggers. | @@ -149,7 +149,6 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **AzureStorageServiceUri** | ```--storage-service-uri``` | ```string``` | | The service URI to use when establishing connection to Azure Blobs Storage. | | **AzureStorageManagedIdentity** | ```--storage-managed-identity``` | ```string``` | | The managed identity to use when establishing connection to Azure Blobs Storage. | | **CheckpointThrottleFlushDelayMs** | ```--checkpoint-throttle-delay``` | ```int``` | Integer in range:
    [-1, MaxValue] | Whether and by how much should we throttle the disk IO for checkpoints: -1 - disable throttling; >= 0 - run checkpoint flush in separate task, sleep for specified time after each WriteAsync | -| **EnableFastCommit** | ```--fast-commit``` | ```bool``` | | Use FastCommit when writing AOF. | | **FastCommitThrottleFreq** | ```--fast-commit-throttle``` | ```int``` | Integer in range:
    [0, MaxValue] | Throttle FastCommit to write metadata once every K commits. | | **NetworkSendThrottleMax** | ```--network-send-throttle``` | ```int``` | Integer in range:
    [0, MaxValue] | Throttle the maximum outstanding network sends per session. | | **EnableScatterGatherGet** | ```--sg-get``` | ```bool``` | | Whether we use scatter gather IO for MGET or a batch of contiguous GET operations - useful to saturate disk random read IO. | diff --git a/website/sidebars.js b/website/sidebars.js index 125613c27f0..b47f235f95c 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -20,16 +20,16 @@ const sidebars = { {type: 'category', label: 'Welcome', collapsed: false, items: ["welcome/intro", "welcome/news", "welcome/features", "welcome/releases", "welcome/compatibility", "welcome/roadmap", "welcome/faq", "welcome/about-us"]}, {type: 'category', label: 'Getting Started', items: ["getting-started/build", "getting-started/configuration", "getting-started/memory", "getting-started/security", "getting-started/compaction"]}, {type: 'category', label: 'Benchmarking', items: ["benchmarking/overview", "benchmarking/results-resp-bench", "benchmarking/resp-bench", {type: 'link', label: 'BDN Charts', href: 'https://microsoft.github.io/garnet/charts/'}]}, - {type: 'category', label: 'Commands', items: ["commands/overview", "commands/api-compatibility", "commands/raw-string", "commands/generic-commands", "commands/analytics-commands", "commands/data-structures", "commands/server-commands", "commands/client-commands", "commands/checkpoint-commands", "commands/transactions-commands", "commands/cluster", "commands/acl-commands", "commands/scripting-commands", "commands/garnet-specific-commands", "commands/json"]}, + {type: 'category', label: 'Commands', items: ["commands/overview", "commands/api-compatibility", "commands/raw-string", "commands/generic-commands", "commands/analytics-commands", "commands/data-structures", "commands/range-index", "commands/server-commands", "commands/client-commands", "commands/checkpoint-commands", "commands/transactions-commands", "commands/cluster", "commands/acl-commands", "commands/scripting-commands", "commands/garnet-specific-commands", "commands/json"]}, {type: 'category', label: 'Server Extensions', items: ["extensions/overview", "extensions/raw-strings", "extensions/objects", "extensions/transactions", "extensions/procedure", "extensions/module"]}, {type: 'category', label: 'Cluster Mode', items: ["cluster/overview", "cluster/replication", "cluster/key-migration"]}, {type: 'category', label: 'Developer Guide', items: ["dev/onboarding", "dev/code-structure", "dev/configuration", "dev/network", "dev/processing", "dev/garnet-api", - {type: 'category', label: 'Tsavorite - Storage Layer', collapsed: true, items: ["dev/tsavorite/intro", "dev/tsavorite/reviv", "dev/tsavorite/locking", "dev/tsavorite/storefunctions", "dev/tsavorite/epoch"]}, + {type: 'category', label: 'Tsavorite - Storage Layer', collapsed: true, items: ["dev/tsavorite/intro", "dev/tsavorite/reviv", "dev/tsavorite/locking", "dev/tsavorite/storefunctions", "dev/tsavorite/epochprotection", "dev/tsavorite/logrecord", "dev/tsavorite/object-allocator"]}, "dev/transactions", "dev/custom-commands", "dev/multi-db", "dev/collection-broker", - {type: 'category', label: 'Cluster', collapsed: true, items: ["dev/cluster/overview","dev/cluster/sharding", "dev/cluster/slot-migration"]}, + {type: 'category', label: 'Cluster', collapsed: true, items: ["dev/cluster/overview","dev/cluster/sharding", "dev/cluster/slot-migration", "dev/cluster/replication-dev"]}, "dev/contributing"]}, /* {type: 'category', label: 'Command Reference', items: ["commands", "pubsub", "transactions"]}, diff --git a/website/yarn.lock b/website/yarn.lock index f12fbe85882..b8d27271fce 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -669,9 +669,9 @@ "@babel/helper-plugin-utils" "^7.28.6" "@babel/plugin-transform-modules-systemjs@^7.29.0": - version "7.29.0" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.29.0.tgz#e458a95a17807c415924106a3ff188a3b8dee964" - integrity sha512-PrujnVFbOdUpw4UHiVwKvKRLMMic8+eC0CuNlxjsyZUiBjhFdPsewdXCkveh2KqBA9/waD0W1b4hXSOBQJezpQ== + version "7.29.4" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-modules-systemjs/-/plugin-transform-modules-systemjs-7.29.4.tgz#f621105da99919c15cf4bde6fcc7346ef95e7b20" + integrity sha512-N7QmZ0xRZfjHOfZeQLJjwgX2zS9pdGHSVl/cjSGlo4dXMqvurfxXDMKY4RqEKzPozV78VMcd0lxyG13mlbKc4w== dependencies: "@babel/helper-module-transforms" "^7.28.6" "@babel/helper-plugin-utils" "^7.28.6" @@ -1080,35 +1080,10 @@ resolved "https://registry.yarnpkg.com/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz#ca2035b0fefe956a8676ff0c69af73e605fcd81f" integrity sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA== -"@chevrotain/cst-dts-gen@12.0.0": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@chevrotain/cst-dts-gen/-/cst-dts-gen-12.0.0.tgz#ec068e1e83c5fdad69d81773556cae97f0b5dcdb" - integrity sha512-fSL4KXjTl7cDgf0B5Rip9Q05BOrYvkJV/RrBTE/bKDN096E4hN/ySpcBK5B24T76dlQ2i32Zc3PAE27jFnFrKg== - dependencies: - "@chevrotain/gast" "12.0.0" - "@chevrotain/types" "12.0.0" - -"@chevrotain/gast@12.0.0": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@chevrotain/gast/-/gast-12.0.0.tgz#0e0cbf8eee01c7a4449b9caf19e5f3834dba2c35" - integrity sha512-1ne/m3XsIT8aEdrvT33so0GUC+wkctpUPK6zU9IlOyJLUbR0rg4G7ZiApiJbggpgPir9ERy3FRjT6T7lpgetnQ== - dependencies: - "@chevrotain/types" "12.0.0" - -"@chevrotain/regexp-to-ast@12.0.0", "@chevrotain/regexp-to-ast@~12.0.0": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@chevrotain/regexp-to-ast/-/regexp-to-ast-12.0.0.tgz#a90bc4b4f5337a883a88dddd0cca7c38cfe66a7a" - integrity sha512-p+EW9MaJwgaHguhoqwOtx/FwuGr+DnNn857sXWOi/mClXIkPGl3rn7hGNWvo31HA3vyeQxjqe+H36yZJwYU8cA== - -"@chevrotain/types@12.0.0": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@chevrotain/types/-/types-12.0.0.tgz#a762b5c2b4f07496b56c93c30ce224b3637cc2c8" - integrity sha512-S+04vjFQKeuYw0/eW3U52LkAHQsB1ASxsPGsLPUyQgrZ2iNNibQrsidruDzjEX2JYfespXMG0eZmXlhA6z7nWA== - -"@chevrotain/utils@12.0.0": - version "12.0.0" - resolved "https://registry.yarnpkg.com/@chevrotain/utils/-/utils-12.0.0.tgz#9aab2055df43d0bb55919eaca76a9cda45e52b89" - integrity sha512-lB59uJoaGIfOOL9knQqQRfhl9g7x8/wqFkp13zTdkRu1huG9kg6IJs1O8hqj9rs6h7orGxHJUKb+mX3rPbWGhA== +"@chevrotain/types@~11.1.1": + version "11.1.2" + resolved "https://registry.yarnpkg.com/@chevrotain/types/-/types-11.1.2.tgz#e83a1a2704f0c5e49e7592b214031a0f4a34d7e5" + integrity sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw== "@colors/colors@1.5.0": version "1.5.0" @@ -2369,12 +2344,12 @@ dependencies: "@types/mdx" "^2.0.0" -"@mermaid-js/parser@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@mermaid-js/parser/-/parser-1.1.0.tgz#8f96c35ddab34a1b12af58f2c59f5abb7d4743fc" - integrity sha512-gxK9ZX2+Fex5zu8LhRQoMeMPEHbc73UKZ0FQ54YrQtUxE1VVhMwzeNtKRPAu5aXks4FasbMe4xB4bWrmq6Jlxw== +"@mermaid-js/parser@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@mermaid-js/parser/-/parser-1.1.1.tgz#30f3ab68d816912e43f245a72a0d4081bf69d966" + integrity sha512-VuHdsYMK1bT6X2JbcAaWAhugTRvRBRyuZgd+c22swUeI9g/ntaxF7CY7dYarhZovofCbUNO0G7JesfmNtjYOCw== dependencies: - langium "^4.0.0" + "@chevrotain/types" "~11.1.1" "@napi-rs/wasm-runtime@^0.2.3": version "0.2.12" @@ -4077,24 +4052,6 @@ cheerio@^1.0.0: undici "^7.19.0" whatwg-mimetype "^4.0.0" -chevrotain-allstar@~0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/chevrotain-allstar/-/chevrotain-allstar-0.4.1.tgz#04e1429faca94a14d4572e0107c4865beac36298" - integrity sha512-PvVJm3oGqrveUVW2Vt/eZGeiAIsJszYweUcYwcskg9e+IubNYKKD+rHHem7A6XVO22eDAL+inxNIGAzZ/VIWlA== - dependencies: - lodash-es "^4.17.21" - -chevrotain@~12.0.0: - version "12.0.0" - resolved "https://registry.yarnpkg.com/chevrotain/-/chevrotain-12.0.0.tgz#8ebefe0a0516b1b314a8d9c7f4e948a509098d1c" - integrity sha512-csJvb+6kEiQaqo1woTdSAuOWdN0WTLIydkKrBnS+V5gZz0oqBrp4kQ35519QgK6TpBThiG3V1vNSHlIkv4AglQ== - dependencies: - "@chevrotain/cst-dts-gen" "12.0.0" - "@chevrotain/gast" "12.0.0" - "@chevrotain/regexp-to-ast" "12.0.0" - "@chevrotain/types" "12.0.0" - "@chevrotain/utils" "12.0.0" - chokidar@^3.5.3, chokidar@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" @@ -5249,6 +5206,11 @@ es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: dependencies: es-errors "^1.3.0" +es-toolkit@^1.45.1: + version "1.46.1" + resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.46.1.tgz#38ca27191a98a867fc544b81cf1477a68947fb06" + integrity sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ== + esast-util-from-estree@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/esast-util-from-estree/-/esast-util-from-estree-2.0.0.tgz#8d1cfb51ad534d2f159dc250e604f3478a79f1ad" @@ -5504,9 +5466,9 @@ fast-json-stable-stringify@^2.0.0: integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== fast-uri@^3.0.1: - version "3.1.0" - resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.0.tgz#66eecff6c764c0df9b762e62ca7edcfb53b4edfa" - integrity sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA== + version "3.1.2" + resolved "https://registry.yarnpkg.com/fast-uri/-/fast-uri-3.1.2.tgz#8af3d4fc9d3e71b11572cc2673b514a7d1a8c8ec" + integrity sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ== fastq@^1.6.0: version "1.20.1" @@ -6582,18 +6544,6 @@ kleur@^3.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -langium@^4.0.0: - version "4.2.2" - resolved "https://registry.yarnpkg.com/langium/-/langium-4.2.2.tgz#d7409c23475d591ed6fc7d123c396e4fa4134e60" - integrity sha512-JUshTRAfHI4/MF9dH2WupvjSXyn8JBuUEWazB8ZVJUtXutT0doDlAv1XKbZ1Pb5sMexa8FF4CFBc0iiul7gbUQ== - dependencies: - "@chevrotain/regexp-to-ast" "~12.0.0" - chevrotain "~12.0.0" - chevrotain-allstar "~0.4.1" - vscode-languageserver "~9.0.1" - vscode-languageserver-textdocument "~1.0.11" - vscode-uri "~3.1.0" - latest-version@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-7.0.0.tgz#843201591ea81a4d404932eeb61240fe04e9e5da" @@ -6655,7 +6605,7 @@ locate-path@^7.1.0: dependencies: p-locate "^6.0.0" -lodash-es@4.18.1, lodash-es@^4.17.21, lodash-es@^4.17.23: +lodash-es@4.18.1, lodash-es@^4.17.21: version "4.18.1" resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.18.1.tgz#b962eeb80d9d983a900bf342961fb7418ca10b1d" integrity sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A== @@ -7015,13 +6965,13 @@ merge2@^1.3.0, merge2@^1.4.1: integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== mermaid@>=11.6.0: - version "11.14.0" - resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-11.14.0.tgz#ce81b22bc10f3117ef7737406ef2d10ee1741769" - integrity sha512-GSGloRsBs+JINmmhl0JDwjpuezCsHB4WGI4NASHxL3fHo3o/BRXTxhDLKnln8/Q0lRFRyDdEjmk1/d5Sn1Xz8g== + version "11.15.0" + resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-11.15.0.tgz#b485c13ea5e1e74f3328c4bb00427bda87fa1c1e" + integrity sha512-pTMbcf3rWdtLiYGpmoTjHEpeY8seiy6sR+9nD7LOs8KfUbHE4lOUAprTRqRAcWSQ6MQpdX+YEsxShtGsINtPtw== dependencies: "@braintree/sanitize-url" "^7.1.1" "@iconify/utils" "^3.0.2" - "@mermaid-js/parser" "^1.1.0" + "@mermaid-js/parser" "^1.1.1" "@types/d3" "^7.4.3" "@upsetjs/venn.js" "^2.0.0" cytoscape "^3.33.1" @@ -7032,14 +6982,14 @@ mermaid@>=11.6.0: dagre-d3-es "7.0.14" dayjs "^1.11.19" dompurify "^3.3.1" + es-toolkit "^1.45.1" katex "^0.16.25" khroma "^2.1.0" - lodash-es "^4.17.23" marked "^16.3.0" roughjs "^4.6.6" stylis "^4.3.6" ts-dedent "^2.2.0" - uuid "^11.1.0" + uuid "^11.1.0 || ^12 || ^13 || ^14.0.0" methods@~1.1.2: version "1.1.2" @@ -9955,7 +9905,7 @@ utils-merge@1.0.1: resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA== -uuid@14.0.0, uuid@^11.1.0, uuid@^8.3.2: +uuid@14.0.0, "uuid@^11.1.0 || ^12 || ^13 || ^14.0.0", uuid@^8.3.2: version "14.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-14.0.0.tgz#0af883220163d264ffe0c084f6b8a89b9666966d" integrity sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg== @@ -9994,41 +9944,6 @@ vfile@^6.0.0, vfile@^6.0.1: "@types/unist" "^3.0.0" vfile-message "^4.0.0" -vscode-jsonrpc@8.2.0: - version "8.2.0" - resolved "https://registry.yarnpkg.com/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz#f43dfa35fb51e763d17cd94dcca0c9458f35abf9" - integrity sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA== - -vscode-languageserver-protocol@3.17.5: - version "3.17.5" - resolved "https://registry.yarnpkg.com/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz#864a8b8f390835572f4e13bd9f8313d0e3ac4bea" - integrity sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg== - dependencies: - vscode-jsonrpc "8.2.0" - vscode-languageserver-types "3.17.5" - -vscode-languageserver-textdocument@~1.0.11: - version "1.0.12" - resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz#457ee04271ab38998a093c68c2342f53f6e4a631" - integrity sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA== - -vscode-languageserver-types@3.17.5: - version "3.17.5" - resolved "https://registry.yarnpkg.com/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz#3273676f0cf2eab40b3f44d085acbb7f08a39d8a" - integrity sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg== - -vscode-languageserver@~9.0.1: - version "9.0.1" - resolved "https://registry.yarnpkg.com/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz#500aef82097eb94df90d008678b0b6b5f474015b" - integrity sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g== - dependencies: - vscode-languageserver-protocol "3.17.5" - -vscode-uri@~3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.1.0.tgz#dd09ec5a66a38b5c3fffc774015713496d14e09c" - integrity sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ== - watchpack@^2.5.1: version "2.5.1" resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-2.5.1.tgz#dd38b601f669e0cbf567cb802e75cead82cde102" @@ -10080,9 +9995,9 @@ webpack-dev-middleware@^7.4.2: schema-utils "^4.0.0" webpack-dev-server@^5.2.2: - version "5.2.3" - resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-5.2.3.tgz#7f36a78be7ac88833fd87757edee31469a9e47d3" - integrity sha512-9Gyu2F7+bg4Vv+pjbovuYDhHX+mqdqITykfzdM9UyKqKHlsE5aAjRhR+oOEfXW5vBeu8tarzlJFIZva4ZjAdrQ== + version "5.2.4" + resolved "https://registry.yarnpkg.com/webpack-dev-server/-/webpack-dev-server-5.2.4.tgz#6e6306ce59848ed322c235e48b326632b1eed6d6" + integrity sha512-GqDPGZN9bRqKBTkp4aWkobDDHMsrXKoGSdOH56smIri8qR0JG8gfL8/v/f/OZR3/OKXjG8uwJbFVhKm/FNU/UA== dependencies: "@types/bonjour" "^3.5.13" "@types/connect-history-api-fallback" "^1.5.4"